mysql - Node js从SQL Server读取100k条记录需要很长时间

Question

我正在编写节点 js 应用程序，它从 SQL 服务器数据库中获取 100 000 条记录并将它们插入到 mysql 数据库中。我已经完成了代码，但我的应用程序没有给我预期的性能。

我选择 10 万条记录需要 9-10 秒，批量插入过程需要 3-4 秒。我希望节点加快选择过程（需要 3-4 秒从 sql server 中选择记录）。我尝试使用 2 种方法，但仍然无法达到预期的时间。

/*First Way*/

var sql       = require('mssql');
var mysql     = require("mysql");
var sizeof    = require('sizeof');
var moment    = require('moment');

var config = {

        user: '**', 
        password: '*****',
        server: '******', 
        database: '*******'
}

var connection1 = mysql.createConnection({
			  host     : 'localhost',
			  user     : 'root',
			  password : 'root',
			  database : 'test'
		});

 connection1.connect();


  sql.connect(config, function(err) {
       
       if(err)
       	console.log(err);

      console.log('Connecting to Sql Server');
      console.time('Overall Time-Taken');
   
      var request = new sql.Request();

      console.time('Time-Taken to fetch');

      request.query('select TOP 100000 * from ShipmentAuditLog WITH (NOLOCK)', function(err, recordset) {
          
          if(err)
            console.log(err);
            else{
                 //console.log(sizeof(recordset));
                 console.timeEnd('Time-Taken to fetch');
                 syncing(recordset);
             } 
      });

  });


  function mapping(row) {
  
      var formatedString = '';

      var CreatedDate = new moment(row.CreatedDate).format('YYYY-MM-DD HH:mm:ss');

      row.Process = row.Process.replace(/'/g,"\\'");

      if(row.Comment != null){
           formatedString = row.Comment;
           formatedString = formatedString.replace(/'/g,"\\'");
           row.Comment    = formatedString;    
      }
       return "('" + row.ShippingID + "','" + row.BagNo + "','" + row.ProcessLocation + "','" + row.Process + "','" + row.Comment + "','" + CreatedDate + "','" + row.CreatedBy + "','" + row.LastModifiedDate + "','" + row.LastModifiedBy + "','" + row.DestinationLocation + "','" + row.VenderLostShipmentsDebitId + "', NOW())";
       //return "('" + row.ShippingID + "','" + row.BagNo + "','" + row.ProcessLocation + "','" + row.Process + "','" + row.Comment + "','" + CreatedDate + "','" + row.CreatedBy + "','" + row.LastModifiedDate + "','" + row.LastModifiedBy + "','" + row.DestinationLocation + "','" + row.VenderLostShipmentsDebitId + "')";
  }

    var syncing  = function(recordset){

      //var columns = [];
      var values  = [];

      /*for(col in recordset[0]){
          columns.push(col);
      }*/

      /*for(var i=0,len = recordset.length;i<len;i+=1){
         values.push(mapping(recordset[i]));
      }*/

      recordset.forEach(function(record){
          values.push(mapping(record));
      })
      //console.log(values);

      console.time('Time-Taken to Insert');

      var QRY = connection1.query('INSERT INTO shipmentauditlog VALUES'+ values.join(','), function(err, result) {
            if (err) {
                console.log(err);
            }
            else {
    
                console.timeEnd('Time-Taken to Insert');
                console.timeEnd('Overall Time-Taken');
               
            }
        });
        //console.log(QRY.sql); 
  }

    
    /*Second Way*/
    
    
    var sql       = require('mssql');
var mysql     = require("mysql");
var moment    = require('moment');

var config = {

        user: '**', 
        password: '***',
        server: '****', 
        database: '****',
        stream: true 
}

var connection1 = mysql.createConnection({
			  host     : 'localhost',
			  user     : 'root',
			  password : 'root',
			  database : 'test'
		});

connection1.connect();

 var values = [];

 function mapResult(row){

    var formatedString = '';

    var CreatedDate = new moment(row.CreatedDate).format('YYYY-MM-DD HH:mm:ss');

    row.Process = row.Process.replace(/'/g,"\\'");

      if(row.Comment != null){
           formatedString = row.Comment;
           formatedString = formatedString.replace(/'/g,"\\'");
           row.Comment    = formatedString;    
      }

     return "('" + row.ShippingID + "','" + row.BagNo + "','" + row.ProcessLocation + "','" + row.Process + "','" + row.Comment + "','" + CreatedDate + "','" + row.CreatedBy + "','" + row.LastModifiedDate + "','" + row.LastModifiedBy + "','" + row.DestinationLocation + "','" + row.VenderLostShipmentsDebitId + "', NOW())";
 }

   var connection = new sql.Connection(config, function(err) {

    if(err)
        console.log(err);

    console.log('Connecting to Sql Server');
    console.time('Overall Time-Taken');
 
    var request = new sql.Request(connection);

    //request.stream = true;
    
    request.query('select TOP 100000 * FROM ShipmentAuditLog WITH (NOLOCK)'); // or request.execute(procedure); 
       
    request.on('recordset',function(col){
         console.time('Time-Taken to fetch');
     });
    
    // Emitted for each row 
    request.on('row', function(row) {
        //Build array with resultset for bulk insert
         values.push(mapResult(row));              
    });
 
    request.on('error', function(err) {
        console.log(err);
    });
 
    // Emitted for the last one
    request.on('done', function(returnValue) {
        console.timeEnd('Time-Taken to fetch');
        // Function which perform bulk insert into mysql
        syncing(values);
        connection.close();
    });

});


 var syncing = function(values){

         /*var startTime = new Date();
         console.log("Start Time :"+ startTime);*/

         console.time('Time-Taken to Insert');

      var qry =  connection1.query('INSERT INTO shipmentauditlog VALUES'+values.join(','),function(err,res){

            if(err)
                console.log(err);
            else{
                /*var endTime = new Date();
                console.log('End Time:'+ endTime);*/
                console.timeEnd('Time-Taken to Insert');
                console.timeEnd('Overall Time-Taken');
            }

        });
 }