javascriptnode.jsjimp

NodeJS peaking at 6Gb when manipulating images with Jimp


I need to edit 34 000 photos on a pixel scale, so I wrote a program that replaces pixels in an image if they are over a specific RBG value (240, 240, 240), basically I want to isolate only white pixels and set everything else to black. This is my code:

const jimp = require('jimp')
var fs = require('fs');
const PNG = require('pngjs').PNG;


const frames = fs.readdirSync("./out/")
for (let i = 0; i < frames.length; i++) {
    function work(frames, i) {
        if (frames[i] !== ".DS_Store") { 
            jimp.read('./out/' + frames[i], (err, image) => {
                const bitmap = new Uint8ClampedArray(image.bitmap.data)
                let texterHighlit = 240
                let nextD = 4;
                let items = [];
                for (i = 0; i < bitmap.length + 1; i++) {
                    if (i === nextD) {
                        if (bitmap[i - 4] > texterHighlit && bitmap[i - 3] > texterHighlit && bitmap[i - 2] > texterHighlit) {
                            items.push(255)
                            items.push(255)
                            items.push(255)
                            items.push(255)
                        } else {
                            items.push(0)
                            items.push(0)
                            items.push(0)
                            items.push(255)
                        }
                        nextD = nextD + 4;
                    }
                }

                var img_width = image.bitmap.width;
                var img_height = image.bitmap.height;
                var img_data = Uint8ClampedArray.from(items);
                var img_png = new PNG({ width: img_width, height: img_height })
                img_png.data = Buffer.from(img_data);
                img_png.pack().pipe(fs.createWriteStream("./out/" + frames[i])) 
                return "done";

            })
        }
    }
    work(frames, i)

}

Now, this works great for a few images but the issue starts occurring when using it on large datasets like the 34 000 image folder I mentioned.

For reference, I am running this code on CentOS8 with PM2 and the --no-autorestart flag. When not using PM2 and just using node <file>, the process gets killed and outputs "killed" in the console, it does not show the "ran out of memory" one. I've been trying to debug this for quite a while but found no way to prevent this, I tried not using Jimp but a package called image-pixels which produced the same result.

Is there a way to prevent this from happening or will I have to manually do it on smaller folders (for example 1000 images each)?


Solution

  • Alright, so asynchronous done correctly was the answer in the end, credit to NeonSurvivor#2046 for this answer given to me on Discord!

    const jimp = require('jimp')
    var fs = require('fs');
    
    (async() => {
        async function work(frame) {
        if (frame === '.DS_Store') return;
        const image = await jimp.read('./out/' + frame);
        const texterHighlit = 240;
        image.scan(0, 0, image.bitmap.width, image.bitmap.height, (x, y, idx) => {
            const { data } = image.bitmap;
            data[idx] = data[idx + 1] = data[idx + 2] = 255 *  [0, 1, 2].every(offset => data[idx + offset] > texterHighlit);
            data[idx + 3] = 255;
        });
        image.write('./out/' + frame);
        return 'done';
    }
    
    for (const frame of fs.readdirSync('./out/')) {
        await work(frame);
        await new Promise(res => setTimeout(res, 500));
    }
    })()