javascriptnode.jsgulpasync-awaitthrough2

Node JS async/await with multiple fs.writeFile using through2 (Gulp/Vinyl)


I'm using through2 to generate multiple files from a Gulp stream. I'm using NodeJS 10.6.0 so thought I'd make full use of async/await, but am not fully understanding the mechanics yet. Currently the through2 done() callback is being fired before all files have been written.

Here's what I have (simplified) - note that I'm not returning the stream at the end as there is no need to.

async function createDirectory(pathDir) {
    return new Promise((resolve, reject) => {
        mkdirp(pathDir, (err) => {
            if (err) reject(err);
            else resolve();
        });
    });
}

async function writeFile(outputFilePath, outputFileContent) {
    return new Promise((resolve, reject) => {
        fs.writeFile(outputFilePath, outputFileContent, (err) => {
            if (err) reject(err);
            else resolve();
        });
    });
}

async function doWriteFile(outputFolderPath, outputFilePath, outputContent) {
    await createDirectory(outputFolderPath);
    await writeFile(outputFilePath, outputContent, outputContent);
}

async function doGenerateVariant(data, variantArr) {
    for (const variant of variantArr) {

        /* Do a load of stuff */

        const variantOutputFolderPath = blah;
        const variantOutputFilePath = blah;
        const variantOutputContent = blah;

        await doWriteFile(variantOutputFolderPath, variantOutputFilePath, variantOutputContent);
    }
}

const generateVariant = () => {
    return through.obj((file, enc, done) => {
        const data = JSON.parse(file.contents.toString());

        */ Do a load of stuff */

        const { variant } = data;
        const variantArr = Object.values(variant);
        doGenerateVariant(data, variantArr);
        return done();
    });
};

This doesn't work as done() gets returned before all files have been written. I'm guessing I'm missing a return or two but nothing I do seems to be working.

If I pass done() into doGenerateVariant and call it after doWriteFile everything works as expected but I know this isn't correct.


Solution

  • You need to wait for doGenerateVariant to do its job before calling done. Remember async function always returns a Promise. So you could do it this way

    const generateVariant = () => {
        return through.obj((file, enc, done) => {
            const data = JSON.parse(file.contents.toString());
    
            */ Do a load of stuff */
    
            const { variant } = data;
            const variantArr = Object.values(variant);
            doGenerateVariant(data, variantArr).then(() => done());
        });
    };
    

    or using async/await

    const generateVariant = () => {
        return through.obj(async (file, enc, done) => {
            const data = JSON.parse(file.contents.toString());
    
            */ Do a load of stuff */
    
            const { variant } = data;
            const variantArr = Object.values(variant);
            await doGenerateVariant(data, variantArr);
            done();
        });
    };