javascriptnode.jsasync-awaitnode-modulesrequestjs

How to download files one by one in node js?


I'm trying to download multiple file using request library, I need to download them one by one and also show a progress bar, the files links are stored in an array that passes them to a function to start the download

const request = require('request')
const fs = require('fs')
const ProgressBar = require('progress')

async function downloadFiles(links) {
    for (let link of links) {
        let file = request(link)
        file.on('response', (res) => {
            var len = parseInt(res.headers['content-length'], 10);
            console.log();
            bar = new ProgressBar('  Downloading [:bar] :rate/bps :percent :etas', {
                complete: '=',
                incomplete: ' ',
                width: 20,
                total: len
            });
            file.on('data', (chunk) => {
                bar.tick(chunk.length);
            })
            file.on('end', () => {
                console.log('\n');
            })
        })
        file.pipe(fs.createWriteStream('./downloads/' + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)))
    }
}


let links = ['https://speed.hetzner.de/100MB.bin', 'https://speed.hetzner.de/100MB.bin', 'https://speed.hetzner.de/100MB.bin', 'https://speed.hetzner.de/100MB.bin']
downloadFiles(links)

This is what I've got so far, the problem is that the request is asynchronous, I tried to use async/await but that way I couldn't get the progress bar to work. How can make it so that the files are downloaded one at the time and also have a progress bar?


Solution

  • Based on my comment about async.queue, this is how I would write that up. You can call dl.downloadFiles([]) as often as you want and it will just fetch everything that you have added to the queue one after another.

    const request = require('request')
    const async = require('async')
    const fs = require('fs')
    const ProgressBar = require('progress')
    
    class Downloader {
        constructor() {
            this.q = async.queue(this.singleFile, 1);
    
            // assign a callback
            this.q.drain(function() {
                console.log('all items have been processed');
            });
    
            // assign an error callback
            this.q.error(function(err, task) {
                console.error('task experienced an error', task);
            });
        }
    
        downloadFiles(links) {
            for (let link of links) {
                this.q.push(link);
            }
        }
    
        singleFile(link, cb) {
            let file = request(link);
            let bar;
            file.on('response', (res) => {
                const len = parseInt(res.headers['content-length'], 10);
                console.log();
                bar = new ProgressBar('  Downloading [:bar] :rate/bps :percent :etas', {
                    complete: '=',
                    incomplete: ' ',
                    width: 20,
                    total: len
                });
                file.on('data', (chunk) => {
                    bar.tick(chunk.length);
                })
                file.on('end', () => {
                    console.log('\n');
                    cb();
                })
            })
            file.pipe(fs.createWriteStream('./downloads/' + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)))
        }
    }
    
    const dl = new Downloader();
    
    dl.downloadFiles([
        'https://speed.hetzner.de/100MB.bin',
        'https://speed.hetzner.de/100MB.bin'
    ]);