2

我正在开发一个应用程序，该应用程序可以抓取网站并将数据公开为休息 api。我正在使用 Node、Express、request、cheerio。我似乎在数组中得到了正确的值..但是在返回之前..数组仍然是空的（在请求函数范围之外）

我无法锻炼我所缺少的东西......你能看看我的代码并告诉我吗？

在 app.js 我指定了路线，

    app.use('/timetable', timetable.timetable(url));

在 timetable.js 中

var classes = require('../lib/classes');
var schedule = require('../lib/schedule');
exports.timetable = function(timeTableURL) {

return function(req, res) {

    request( timeTableURL, function srapeWebsite(error, response, html ) {
        var webHtml = '';
        var moreInfo = []; 
        if( !error && response.statusCode === 200 ) {
            webHtml = cheerio.load(html);
        }
        // schedule.getInfo returns an array of json objects where
        // json = {
        //    day,
        //    number,
        //    url
        // }
        var info = schedule.getInfo(webHtml);

        for (var index = 0; index < info.length; index++) {     
            var json = info[index];
            classes.getMoreInfo(json, function (moreInfoJson) {
                //console.log shows correct information here
                moreInfo.push(moreInfoJson);
            };
        }
        // however moreInfo is empty here..hence not getting anything
        res.json(moreInfo);
    } );
  };
};

思路是，对于数组中的每一个json对象，从指定的url中获取更多信息。

所以最终的结果是，

    finalJson = {
       day : ''
       json : []
    }

在 classes.js 中

exports.getMoreInfo =  function (info, callback) {

var infoDay = info.day;
var infoNumber = parseInt(info.number);
var moreInfoURL = info.moreInfoUrl;
var stuff = [];

var moreInfo = {};
moreInfo.day = infoDay;
moreInfo.json = [];

if (infoNumber > 0 ) {
    request(moreInfoURL, function(error, response,html) {
        var moreInfoHtml = '';
        if( !error && response.statusCode === 200 ) {
            moreInfoHtml = cheerio.load(html) ;

        }
        var $ = moreInfoHtml;

        $('table tbody tr').each ( function getWhatisNeeded () {
            var json = getJson ( $(this) );
            stuff.push(json);

        });

        moreInfo.json = stuff;
        return callback(moreInfo);
    });
  }
}
4

2 回答 2

3

您对 classes.getMoreInfo 的调用是异步的。

发生的情况是您不等待调用返回，而是使用空数组调用 res.json。

编辑：如何解决这个问题？一种方法是使用承诺（例如参见https://github.com/malko/D.js）。

getMoreInfo 应该返回一个延迟对象：

exports.getMoreInfo =  function (info) {
    var deferred = D();
    ...
    request(..., function(){
        deferred.resolve(moreInfo);
    });
    ...
    return deferred.promise;
}

和 timetable.js 应该或多或少像这样：

var promises = [];
for (var index = 0; index < info.length; index++) {     
    var json = info[index];
    promises.push(classes.getMoreInfo(json));
}
D.all(promises).this(function(array_of_results){
    res.json(array_of_results);
}

我没有测试代码，所以我可能在这里有一个错误，但这是关于如何解决 node.js 和一般情况下的异步问题的一个很好的指南。让你自己熟悉 Promise，这样解决问题会容易得多。

于 2014-08-26T12:21:49.927 回答
0

Thanks for the tip 'jonjon'..I used Async.map to fix it ( sorry was going to post it yesterday..but you beat me to it)..I'll also try out Promises as well..I might as well learn both techniques..

this is what I did to fix it..

In app.js

app.get('/timetable', timetable.timetable);

In timetable.js

var url = '...';
exports.timetable = function ( request, response ) {

    //scrape html to get info
    scrape.getHtml( url, function ( error, html ) {
    // schedule.getInfo returns an array of json objects where
    // json = {
    //    day,
    //    number,
    //    url
    // }
    var info = schedule.getInfo(cheerio.load(html));

    //getmoreinfo
    async.map(info, classes.getMoreInfo, function( error,moreInfo ) {
        if(!error) {
            response.json(moreInfo);
        }else {
            response.send("error encountered");
        }
    });
  });
}

In classes.js

   exports.getMoreInfo =  function (info, callback) {

   var infoDay = info.day;
   var infoNumber = parseInt(info.number);
   var moreInfoURL = info.moreInfoUrl;
   var stuff = [];

   var moreInfo = {};
   moreInfo.day = infoDay;
   moreInfo.json = [];

   if (infoNumber > 0 ) {
   request(moreInfoURL, function(error, response,html) {
    var moreInfoHtml = '';
    if( !error && response.statusCode === 200 ) {
        moreInfoHtml = cheerio.load(html) ;

    }
    var $ = moreInfoHtml;

    $('table tbody tr').each ( function getWhatisNeeded () {
        var json = getJson ( $(this) );
        stuff.push(json);

    });

    moreInfo.json = stuff;
    return callback(null, moreInfo);
  });
 }
}

I'm getting what I'm after now..however it seems to take 5 seconds..will need to test more anyway..

于 2014-08-29T00:57:23.400 回答