Some requests that can be made to my nodejs server require heavy processing (for example: 5000 files). Since the request will take a while to process, I want to display progresses in the browser. For that, I'm using io-socket. Regularly, the server sends the progress advance to the client with for example ioSocket.emit("log", "progress 24%).
However, if progress sending to the client usually works, when there is a big amount of files, it doesn't. Nothing is sent to browser.
I'm sure the process is going fine since I log progress to node terminal, and there it appears as expected.
I'm wondering what I can do to have ioSocket.emit event work in heavy load case, because it's where it's most useful to see progress.
The files processing function looks like this:
var child_process = require("child_process");
function (ioSocket) {
ioSocket.emit("log", "start")
var ratingCounts = 0;
var listOfFilesRatings = []
_.each(listOfFilesPaths, function(path, i){
child_process.exec("exiftool -b -Rating "+ path, function(err, stdout){
if (err) console.log(err)
else {
listOfFilesRatings.push(stdout);
ratingCounts++;
ioSocket.emit("log", "rating test progress "+ ratingCounts)
};
});
ioSocket.emit("log", "each progress "+ i)
});
}
In this example, only the first "start" emit will be fired to the browser.
However if I do the following:
function (ioSocket) {
ioSocket.emit("log", "start")
for (i=0; i<1000; i++) {
ioSocket.emit("log", "each progress "+ i)
};
}
everything works fine, and I get the "start" and all "each progress" sent to browser.
If you are processing 5000 files, your scheme with _.each() and child_process.exec() will launch 5000 exiftool processes at once. That will likely bring any computer, except for perhaps some big iron to its knees. You should probably be launching no more than N of those at a time where you run some performance tests on your particular hardware to determine what N should be (probably under 10).
Here's one way to do that:
var child_process = require("child_process");
function processFiles(ioSocket) {
return new Promise((resolve, reject) => {
ioSocket.emit("log", "start")
let ratingCounts = 0;
let listOfFilesRatings = [];
const maxInFlight = 10;
let inFlightCntr = 0;
let fileIndex = 0;
function run() {
// while room to run more, run them
while (inFlightCntr < maxInFlight && fileIndex < listOfFilesPaths.length) {
let index = fileIndex++;
++inFlightCntr;
ioSocket.emit("log", "each progress " + index)
child_process.exec("exiftool -b -Rating " + path, function(err, stdout) {
++ratingCounts;
--inFlightCntr;
if (err) {
console.log(err);
listOfFilesRatings[index] = 0;
} else {
listOfFilesRatings[index] = stdout;
ioSocket.emit("log", "rating test progress " + ratingCounts)
}
run();
});
}
if (inFlightCntr === 0 && fileIndex >= listOfFilesPaths.length) {
// all done here
console.log(listOfFilesRatings);
resolve(listOfFilesRatings);
}
}
run();
});
}
processFiles().then(results => {
console.log(results);
});