node.js - Nodejs streaming upload multiple files

Question

I am working on a command line application that is supposed to take an array of file names, do transform operations ( text files, spreadsheets, etc payloads have to be rewritten into JSON objects ), and send results to an endpoint API ( api.example.com ). I am considering a sequential read, and pipe result to an instance of -http, or -request, But has no idea of where to start from. Is there any alternatives or strategy have you used to solve a similar problem?

Any algorithm, or point to an article or a similar question here on SO will be highly appreciated. Thanks.

Update1. I found a link that may help in this google group https://groups.google.com/forum/#!topic/nodejs/_42VJGc9xJ4

To keep track of the final solution:

var request = require('request');
var file = fs.createReadStream(path)
      .pipe(request.put({url: url, headers:{'Content-Length': fileSize}}, function(err, res, body){
        if(err) {
          console.log('error', err);
        } else {
          console.log('status', res.statusCode);
          if(res.statusCode === 200) {
            console.log('success'); 
          }
        }
      }));

The remaining problem is how to make this work for "n" files, in case "n" is high - 100 text files or more.

Answer 1

After try and err, I solved this issue with Events, and I paste the answer just in case someone else is struggling to solve a similar problem.

var EventEmitter = require('events').EventEmitter;
var request        = require('request');
var util          = require('util');


function Tester(){
  EventEmitter.call(this);
  this.files = ['http://google.ca', 'http://google.com', 'http://google.us'];



}

util.inherits( Tester, EventEmitter );

Tester.prototype.run = function(){
  //referencing this to be used with the kids down-here  
  var self = this;
  if( !this.files.length ) { console.log("Cannot run again .... "); return false; }
  request({ url : this.files.shift()}, function( err, res, body ) {
    console.log( err, res.statusCode, body.length, " --- Running the test --- remaining files ", self.files );
    if( !self.files.length ) self.emit( "stop" );
    else self.emit( "next" , self.files );
  });
};

//creating a new instance of the tester class
var tester = new Tester();
  tester.on("next", function(data){
       //@todo --- wait a couple of ms not to overload the server.
       //re-run each time I got this event
       tester.run();
  });
  tester.on("stop", function(data){
    console.log("Got stop command --- Good bye!");
  });
  //initialize the first run
  tester.run();
  //graceful shutdown -- supporting windows as well 
  //@link http://stackoverflow.com/questions/10021373/what-is-the-windows-equivalent-of-process-onsigint-in-node-js
  if (process.platform === "win32") {
    require("readline").createInterface({
        input: process.stdin,
        output: process.stdout
    }).on("SIGINT", function () {
        process.emit("SIGINT");
    });
}

process.on("SIGINT", function () {
    // graceful shutdown
    process.exit();
});



console.log('Started the application ... ');

NB:

• For a quick test, here is the runnable

• I used get for a quick test, but post/put may as well. I hope it help, feel free to leave a comment. Thanks.