node.js - streams with percentage complete

Question

I need to stream a file in base64 to an http endpoint using something like request or superagent. What is the best way to figure out what percentage of the file has been uploaded?

I assume I can create the read stream using something like:

fs.createReadStream('/tmp/cats.jpg', {encoding: 'base64'})

Any examples using one out of above libraries would be greatly appreciated.

Answer 1

I think you can use progress-stream.

Here is an example from the package:

var progress = require('progress-stream');
var fs = require('fs');
 
var stat = fs.statSync(filename);
var str = progress({
    length: stat.size,
    time: 100 /* ms */
});
 
str.on('progress', function(progress) {
    console.log(progress);
 
    /*
    {
        percentage: 9.05,
        transferred: 949624,
        length: 10485760,
        remaining: 9536136,
        eta: 42,
        runtime: 3,
        delta: 295396,
        speed: 949624
    }
    */
});
 
fs.createReadStream(filename)
    .pipe(str)
    .pipe(fs.createWriteStream(output));

Answer 2

I was looking for an answer to a similar issue and thanks to Alberto Zaccagni's answer, I was able to get some code working.

So for the people who don't want to piece the puzzle themselves, here is the code (edited for Stackoverflow):

var zipfile = "my_large_archive.zip";

// Get the size of the file
fs.stat(zipfile, function (err, stats) {
    var zipSize         = stats.size;
    var uploadedSize    = 0; // Incremented by on('data') to keep track of the amount of data we've uploaded

    // Create a new read stream so we can plug events on it, and get the upload progress
    var zipReadStream   = fs.createReadStream(zipfile);

    zipReadStream.on('data', function(buffer) {
        var segmentLength   = buffer.length;

        // Increment the uploaded data counter
        uploadedSize        += segmentLength;

        // Display the upload percentage
        console.log("Progress:\t",((uploadedSize/zipSize*100).toFixed(2)+"%"));
    });

    // Some other events you might want for your code
    zipReadStream.on('end', function() {
        console.log("Event: end");
    });
    zipReadStream.on('close', function() {
        console.log("Event: close");
    });


    var formData    = require('form-data');
    var form        = new formData();

    form.append('apikey', 'f4sd5f4sdf6ds456');  // Just some post parameters I need to send to the upload endpoint
    form.append('file', zipReadStream);         // The zip file, passed as a fs.createReadStream instance

    // Submit the form and the file
    form.submit('http://www.someserver.com/upload', function(err, res) {
        if (err) {
            console.log("Oups! We encountered an error :(\n\n", err);
            return false;
        }
        console.log("Your file has been uploaded.");
        res.resume();   // Fix is you use that code for a CLI, so that the execution will stop and let users enter new commands
    });
});

Answer 3

In nodejs we have the Readable stream, it emits the data event when it receives a chunk of data, by knowing the file size you could easily keep track of how much data passes through the data event receiver and then update the percentage.

Get the file dimension with

require('fs').watchFile('yourfile', function () {
    fs.stat('yourfile', function (err, stats) {
        console.log(stats.size);
    });
});