nodejs利用superagent爬取数据的简单例子

爬取世界银行统计数据

安装：npm install superagent

/**
 * Created by zh on 16-9-7.
 */
var request = require('superagent')
    , fs = require('fs');
var xlsx = require("node-xlsx");

function sleep(milliSeconds) {
    var startTime = new Date().getTime();
    while (new Date().getTime() < startTime + milliSeconds);
}

var list = xlsx.parse("/home/zh/桌面/API_BTN_DS2_zh_excel_v2.xlsx");
list[2]['data'].forEach(function(row){
    url='http://api.worldbank.org/v2/zh/indicator/'+row[0]+'?downloadformat=excel';
    console.log(row[1],url);
    //console.log();
    sleep(5000);
    var stream = fs.createWriteStream('/home/zh/project/cloudmap/worldbank/'+row[1]+'.xls');
    var req = request.get(url);
    req.pipe(stream);
    sleep(5000);
});

View Code