NodeJS Script to prepare the dataset for ImageAI Training
The goal of this script is to simply prepare the field to copy the appropriate files for the training. The tool used for the training is : https://github.com/OlafenwaMoses/ImageAI/blob/master/imageai/Classification/CUSTOMTRAINING.md
The code is available here (And mostly more up-to-date than here): https://github.com/webuxlab/image-ai
The NodeJS Script
There is no dependencies. It only scans the directory and do some symlinks to prepare the file structure for us.
index.js
// Studio Webux 2022
const { resolve } = require("path");
const {
readdirSync,
symlinkSync,
mkdirSync,
existsSync,
unlinkSync,
} = require("fs");
process.env.DEBUG &&
process.argv.forEach((val, index) => {
console.log(`${index}: ${val}`);
});
function getFiles(workingDirectory) {
return readdirSync(resolve(process.cwd(), workingDirectory));
}
function listPrefixes(files) {
return new Set(files.map((file) => file.split("_")[0]));
}
async function getFileCountPerPrefixes(prefixes, files) {
return Promise.all(prefixes.map((prefix) => getCount(prefix, files)));
}
function getCount(prefix, files) {
const count = files.filter((file) => file.includes(prefix)).length;
return {
prefix,
count,
p10_test: (count * 0.1).toFixed(0),
p20_test: (count * 0.2).toFixed(0),
p10_training: (count * 0.9).toFixed(0),
p20_training: (count * 0.8).toFixed(0),
trainable: count > 500,
};
}
function verifyImageFormat(files) {
return files.filter((file) => file.includes(".jpg"));
}
/**
* image AI seems to require the same amount of picture per categories to have a equivalent probabilities.
* @param {*} prefixes
* @param {*} meta
* @returns
*/
function getMinimumCount(prefixes, meta) {
const values = prefixes.split(",").map((prefix) => ({
prefix,
count: meta.filter((info) => info.prefix === prefix)[0].count,
}));
return Math.min.apply(
Math,
values.map((i) => i.count)
);
}
async function symlinkHandler(prefixes, source, dest, files, meta, limit) {
return Promise.all(
prefixes.split(",").map(
(prefix) =>
symlink(
prefix,
source,
dest,
files.filter((file) => file.includes(prefix)),
meta,
limit
),
meta
)
);
}
/**
* Placeholder in case we need to handle the dup differently.
* @param {*} file
*/
function fileExist(file) {
// Currently this approach doesn't support continuing a training because the files can changed;
// So we might have duplicated file after adding new stuff.
// I think a better approach is having a mapping to removed the already linked files in
// either test or train folders before starting the symlink process
// unlinkSync(file);
process.env.DEBUG &&
console.debug(
`File [${file}] is already in either test or train, skipping...`
);
return true; // Will NOT symlink the file again.
}
/**
* Allows improving the dataset over time without moving files everywhere and cause more damage.
* @param {*} dest
* @param {*} prefix
* @param {*} file
* @returns
*/
function checkDup(dest, prefix, file) {
return (
existsSync(resolve(dest, "test", prefix, file)) ||
existsSync(resolve(dest, "train", prefix, file))
);
}
function symlink(prefix, source, dest, files, meta, limit = null) {
const testCount = limit
? (limit * 0.1).toFixed(0)
: meta.filter((info) => info.prefix === prefix)[0].p10_test;
const trainCount = limit
? (limit * 0.9).toFixed(0)
: meta.filter((info) => info.prefix === prefix)[0].p10_training;
const isTrainable = meta.filter((info) => info.prefix === prefix)[0]
.trainable;
if (!isTrainable) {
throw new Error(`[WARNING] The prefix [${prefix}] is not trainable.`);
}
mkdirSync(resolve(dest, "test", prefix), { recursive: true });
mkdirSync(resolve(dest, "train", prefix), { recursive: true });
// test structure
files.slice(0, testCount).forEach((file) => {
(checkDup(dest, prefix, file) &&
fileExist(resolve(dest, "test", prefix, file))) ||
symlinkSync(
resolve(source, file),
resolve(dest, "test", prefix, file),
"file"
);
});
// training structure
files
.slice(testCount, parseInt(testCount) + parseInt(trainCount))
.forEach((file) => {
(checkDup(dest, prefix, file) &&
fileExist(resolve(dest, "train", prefix, file))) ||
symlinkSync(
resolve(source, file),
resolve(dest, "train", prefix, file),
"file"
);
});
// Check
const trainFileRead = readdirSync(resolve(dest, "train", prefix)).length;
const testFileRead = readdirSync(resolve(dest, "test", prefix)).length;
if (trainFileRead != trainCount || testFileRead != testCount) {
throw new Error(
`Something went wrong. ${prefix}, training: ${trainFileRead}/${trainCount}; test: ${testFileRead}/${testCount}`
);
}
return {
prefix,
source,
dest,
count: files.length,
meta: meta.filter((info) => info.prefix === prefix)[0],
limit,
};
}
(async () => {
try {
const target = process.argv.slice(2)[0];
if (!target) {
throw new Error("Missing target.");
}
let files = getFiles(target);
files = [...verifyImageFormat(files)];
console.debug("Valid File(s) " + files.length);
const prefixes = listPrefixes(files);
const mapping = await getFileCountPerPrefixes([...prefixes], files);
const minimum = getMinimumCount(process.argv.slice(3)[0], mapping);
if (process.argv.slice(3)[0] && process.argv.slice(4)[0]) {
const output = await symlinkHandler(
process.argv.slice(3)[0], //prefixes
process.argv.slice(2)[0], //source
process.argv.slice(4)[0], //dest
files, // files to process
mapping, // metadata to build the test/training
minimum || null
);
console.debug(output);
} else {
console.log(`Prefixes found: ${[...prefixes].length}`);
console.log(`Available Prefixes: ${[...prefixes]}`);
console.log(`Mapping found: ${JSON.stringify(mapping, null, 2)}`);
}
} catch (e) {
console.error(`[ERROR] ${e.message}`);
console.error(e);
process.exit(1);
}
})();
Usage
Get all prefixes
node ./index.js ./images/
Where images/
contains all your images
Something like:
images/
cats_123.png
cats_124.png
cats_125.png
cats_126.png
dogs_0.png
dogs_1.png
dogs_2.png
snakes_0.png
snakes_1.png
snakes_2.png
Écureuil_1.png
Écureuil_2.png
Écureuil_3.png
Écureuil_4.png
Image AI requires a structure to perform the training correctly. I need that script because I use a crawler to gather all images and the crawler use the naming convention mentionned above.
Symlink and file preparation
The first script will list what prefixes are available and if it is trainable (which means, having more than 500 images)
The next script is to create symbolic links in order to have the correct file structure.
node move.js ./pins "Étiquette,Kitchen design,wood" ./images_to_train
images_to_train/
test/
Étiquette/
Kitchen design/
wood/
train/
Étiquette/
Kitchen design/
wood/
From here you have the correct structure to start a training.
The actual training
I’ll not really document how to install all that, because if it changes I’ll forget to update it… Please follow the official documentation.
Some notes (Copy Pasted stuffs) Ubuntu:
apt update && apt install python3 python3-pip -y
pip install tensorflow==2.4.0
pip install tensorflow-gpu==2.4.0
pip install keras==2.4.3 numpy==1.19.3 pillow==7.0.0 scipy==1.4.1 h5py==2.10.0 matplotlib==3.3.2 opencv-python keras-resnet==0.2.0
pip install imageai --upgrade
Using WSL (don’t loose you time with that approach, see below to install it natively on Windows 10)
pip install tensorflow-gpu==2.4.0
pip install keras==2.4.3 numpy==1.19.3 pillow==7.0.0 scipy==1.4.1 h5py==2.10.0 matplotlib==3.3.2 opencv-python keras-resnet==0.2.0
pip install imageai --upgrade
WSL Bundle:
pip --version
pip 20.0.2 from /usr/lib/python3/dist-packages/pip (python 3.8)
python --version
Python 3.8.10
Forget it… WSL crashes after 1 minute…
Based on the documentation, you only need to do :
from imageai.Classification.Custom import ClassificationModelTrainer
model_trainer = ClassificationModelTrainer()
model_trainer.setModelTypeAsResNet50()
model_trainer.setDataDirectory("images_to_train")
model_trainer.trainModel(num_objects=3, num_experiments=100, enhance_data=True, batch_size=32, show_network_summary=True)
Voila !
Windows 10
How to start the training on windows 10 with tensorflow-gpu
First you must install python 3.8, ~~I used the windows store to do so : https://www.microsoft.com/store/productId/9MSSZTT1N39L~~; I used this link to download and install Python: https://www.python.org/downloads/release/python-380/
Then install the prerequisites:
New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force
pip install tensorflow-gpu==2.4.0
pip install keras==2.4.3 numpy==1.19.3 pillow==7.0.0 scipy==1.4.1 h5py==2.10.0 matplotlib==3.3.2 opencv-python keras-resnet==0.2.0
pip install imageai --upgrade
Download and install CUDA 11.5 : https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_version=11&target_type=exe_local
Download and install cuDNN 8.3.3.40 : https://developer.nvidia.com/rdp/cudnn-download that matches your cuda version installed. (For this step you need an nvidia account)
Add these to your PATH variable:
https://www.tensorflow.org/install/gpu?hl=fr#windows_setup https://ourcodeworld.com/articles/read/1433/how-to-fix-tensorflow-warning-could-not-load-dynamic-library-cudart64-110dll-dlerror-cudart64-110dll-not-found
C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.5\extras\CUPTI\lib64
C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.5\bin
C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.5\include
C:\Program Files\NVIDIA\CUDNN\v8.3\bin
C:\zlib\dll_x64
Create a symlink for: New-Item -ItemType SymbolicLink -Path "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.5\bin\cupti64_110.dll" -Target "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.5\bin\cupti64_2022.1.1.dll"
Create another symlink: New-Item -ItemType SymbolicLink -Path "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.5\bin\cusolver64_10.dll" -Target "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.5\bin\cusolver64_11.dll"
Install Zlib : https://docs.nvidia.com/deeplearning/cudnn/install-guide/index.html#install-zlib-windows; Download and extract the zip file and add the path in your PATH variable.
As for now I’ve successfully started the training on windows, it wasn’t easy to setup !
Errors
- Cuda 11.6 (cuda_11.6.2_511.65_windows) didn’t work. neither cuDNN (cudnn_8.3.3.40_windows)
- Will try cuDNN (cudnn_8.3.3.40_windows) with CUDA 11.5.2 (cuda_11.5.2_windows_network)
- DO NOT INSTALL Python 3.8 from the windows store.
- Ran out of memory:
setx TF_FORCE_GPU_ALLOW_GROWTH true
W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 2.30GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available.
How to test your custom trained model
from imageai.Classification.Custom import CustomImageClassification
import os
execution_path = os.getcwd()
prediction = CustomImageClassification()
prediction.setModelTypeAsResNet50()
prediction.setModelPath(os.path.join(execution_path, "architecture\\models\\model_ex-028_acc-0.922763.h5"))
prediction.setJsonPath(os.path.join(execution_path, "architecture\\json\\model_class.json"))
prediction.loadModel(num_objects=10)
predictions, probabilities = prediction.classifyImage(os.path.join(execution_path, "architecture\\test\\Bathroom\\Bathroom_1.png"), result_count=5)
for eachPrediction, eachProbability in zip(predictions, probabilities):
print(eachPrediction , " : " , eachProbability)
Launch this python script to see the prediction made by the algorithm
Another attempt
Based on the documentation, I wasn’t using the correct version. So I’ve started again with these:
My results, using 18K images were always returning 100% of the wrong object. There is few issues that have been opened for a while on github without solution.
Python and dependencies:
python --version
Python 3.7.6
pip install tensorflow-gpu==2.4.0
pip install keras==2.4.3 numpy==1.19.3 pillow==7.0.0 scipy==1.4.1 h5py==2.10.0 matplotlib==3.3.2 opencv-python==4.2.0.32 keras-resnet==0.2.0
pip install imageai --upgrade
Nvidia:
nvidia-smi.exe
Wed Mar 30 19:59:51 2022
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 451.48 Driver Version: 451.48 CUDA Version: 11.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name TCC/WDDM | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
|===============================+======================+======================|
| 0 GeForce GTX 166... WDDM | 00000000:06:00.0 On | N/A |
| 36% 48C P2 27W / 130W | 5804MiB / 6144MiB | 29% Default |
+-------------------------------+----------------------+----------------------+
I’ve installed this Python 3.7.6 from here : https://www.python.org/downloads/release/python-376/
For Cuda and cuDNN : cuda_11.0.2_win10_network / cudnn-11.0-windows-x64-v8.0.4.30
For the training, I’m still using 10 objects, but this time they all have the same amount of pictures and they are all JPG.
Found 8870 images belonging to 10 classes.
Found 990 images belonging to 10 classes.
It takes 1h30min per 8 epochs with a batch_size of 8. I know… not efficient at all.
Test
Gardening : 100.0 Stairs design : 0.0 Living room : 0.0 Kitchen design : 0.0 House exterior : 0.0
Still bad :/ with this trained model, I’m using ResNet50; some people were saying that it was possible to obtain better results with the dense algorithm. I’ll continue to train the dataset till I reach a better accuracy. And if it is still bad. I might try another algorithm. And meanwhile I’ll continue to build my dataset.
Epoch 6/6
1108/1108 [==============================] - 523s 472ms/step - loss: 0.5221 - accuracy: 0.8180 - val_loss: 0.9849 - val_accuracy: 0.7022
Epoch 00006: accuracy improved from 0.81596 to 0.82137, saving model to arch\models\model_ex-006_acc-0.821372.h5
I’m doing small batch of epoch; Because I use my computer for my day to day as well.
Extras
So far I perform few hours of training and the results are quite bad. I use 10 objects with a total of 18000 images. But I think (I didn’t read about it and I don’t have any knowledge in AI) That the dataset should have the same quantities of images per object to have better balance.
So I’ll try that approach; The prepare.js
script is already up-to-date to support that configuration.