node.jsexpressbusboy

How to combine video upload chunks Node.js


I'm trying to upload a large (8.3GB) video to my Node.js (Express) server by chunking using busboy. How to I receive each chunk (busboy is doing this part) and piece it together as one whole video?

I have been looking into readable and writable streams but I'm not ever getting the whole video. I keep overwriting parts of it, resulting in about 1 GB.

Here's my code:

req.busboy.on('file', (fieldname, file, filename) => {
    logger.info(`Upload of '${filename}' started`);

    const video = fs.createReadStream(path.join(`${process.cwd()}/uploads`, filename));
    const fstream = fs.createWriteStream(path.join(`${process.cwd()}/uploads`, filename));

    if (video) {
        video.pipe(fstream);
    }

    file.pipe(fstream);

    fstream.on('close', () => {
        logger.info(`Upload of '${filename}' finished`);
        res.status(200).send(`Upload of '${filename}' finished`);
    }); 
});

Solution

  • After 12+ hours, I got it figured out using pieces from this article that was given to me. I came up with this code:

    //busboy is middleware on my index.js
    const fs = require('fs-extra');
    const streamToBuffer = require('fast-stream-to-buffer');
    
    //API function called first
    uploadVideoChunks(req, res) {
        req.pipe(req.busboy);
    
        req.busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
            const fileNameBase = filename.replace(/\.[^/.]+$/, '');
    
            //save all the chunks to a temp folder with .tmp extensions
            streamToBuffer(file, function (error, buffer) {
                const chunkDir = `${process.cwd()}/uploads/${fileNameBase}`;
                fs.outputFileSync(path.join(chunkDir, `${Date.now()}-${fileNameBase}.tmp`), buffer);
            });
        });
    
        req.busboy.on('finish', () => {
            res.status(200).send(`Finshed uploading chunk`);
        });
    }
    
    //API function called once all chunks are uploaded
    saveToFile(req, res) {
        const { filename, profileId, movieId } = req.body;
    
        const uploadDir = `${process.cwd()}/uploads`;
        const fileNameBase = filename.replace(/\.[^/.]+$/, '');
        const chunkDir = `${uploadDir}/${fileNameBase}`;
        let outputFile = fs.createWriteStream(path.join(uploadDir, filename));
    
        fs.readdir(chunkDir, function(error, filenames) {
           if (error) {
               throw new Error('Cannot get upload chunks!');
           }
    
           //loop through the temp dir and write to the stream to create a new file
           filenames.forEach(function(tempName) {
               const data = fs.readFileSync(`${chunkDir}/${tempName}`);
                    outputFile.write(data);
                    //delete the chunk we just handled
                    fs.removeSync(`${chunkDir}/${tempName}`);
               });
    
                outputFile.end();
            });
    
            outputFile.on('finish', async function () {
                //delete the temp folder once the file is written
                fs.removeSync(chunkDir);
            }
        });
    }