I need to edit 34 000 photos on a pixel scale, so I wrote a program that replaces pixels in an image if they are over a specific RBG value (240, 240, 240), basically I want to isolate only white pixels and set everything else to black. This is my code:
const jimp = require('jimp')
var fs = require('fs');
const PNG = require('pngjs').PNG;
const frames = fs.readdirSync("./out/")
for (let i = 0; i < frames.length; i++) {
function work(frames, i) {
if (frames[i] !== ".DS_Store") {
jimp.read('./out/' + frames[i], (err, image) => {
const bitmap = new Uint8ClampedArray(image.bitmap.data)
let texterHighlit = 240
let nextD = 4;
let items = [];
for (i = 0; i < bitmap.length + 1; i++) {
if (i === nextD) {
if (bitmap[i - 4] > texterHighlit && bitmap[i - 3] > texterHighlit && bitmap[i - 2] > texterHighlit) {
items.push(255)
items.push(255)
items.push(255)
items.push(255)
} else {
items.push(0)
items.push(0)
items.push(0)
items.push(255)
}
nextD = nextD + 4;
}
}
var img_width = image.bitmap.width;
var img_height = image.bitmap.height;
var img_data = Uint8ClampedArray.from(items);
var img_png = new PNG({ width: img_width, height: img_height })
img_png.data = Buffer.from(img_data);
img_png.pack().pipe(fs.createWriteStream("./out/" + frames[i]))
return "done";
})
}
}
work(frames, i)
}
Now, this works great for a few images but the issue starts occurring when using it on large datasets like the 34 000 image folder I mentioned.
For reference, I am running this code on CentOS8 with PM2 and the --no-autorestart
flag. When not using PM2 and just using node <file>
, the process gets killed and outputs "killed" in the console, it does not show the "ran out of memory" one. I've been trying to debug this for quite a while but found no way to prevent this, I tried not using Jimp but a package called image-pixels which produced the same result.
Is there a way to prevent this from happening or will I have to manually do it on smaller folders (for example 1000 images each)?
Alright, so asynchronous done correctly was the answer in the end, credit to NeonSurvivor#2046
for this answer given to me on Discord!
const jimp = require('jimp')
var fs = require('fs');
(async() => {
async function work(frame) {
if (frame === '.DS_Store') return;
const image = await jimp.read('./out/' + frame);
const texterHighlit = 240;
image.scan(0, 0, image.bitmap.width, image.bitmap.height, (x, y, idx) => {
const { data } = image.bitmap;
data[idx] = data[idx + 1] = data[idx + 2] = 255 * [0, 1, 2].every(offset => data[idx + offset] > texterHighlit);
data[idx + 3] = 255;
});
image.write('./out/' + frame);
return 'done';
}
for (const frame of fs.readdirSync('./out/')) {
await work(frame);
await new Promise(res => setTimeout(res, 500));
}
})()