【发布时间】:2016-09-02 07:08:56
【问题描述】:
我还有另一个“发送后无法设置标题”。问题。我创建了一个 post 路由器,它接收一个大的 url 对象,我需要对其进行处理。
处理过程包括抓取对象包含的所有 url。我设置了一个瓶颈,最多 10 个并发爬取和 800 毫秒的等待时间。
无论哪种方式,数据都有可能到达并记录下来,但是当我调用我的函数时,我的函数应该处理数据,我的应用程序会抛出一个异常。
发布路由器
router.post('/crawledUrls', function (req, res, err) {
crawler = new crawlerClass();
var urlArray = JSON.parse(req.body);
crawler.crawlArrayUrls(urlArray);
res.send({message: "Array sent successfully!"});
});
crawler.crawlArrayUrls
crawlerClass.prototype.crawlArrayUrls = function (arrayObject) {
var itemsProcessed = 0;
var emptyUrlArray = [];
var newArray = [];
for(keys in arrayObject){
newArray.push(arrayObject[keys]);
}
var limiter = new Bottleneck(10, 800);
newArray.forEach(function (listItem, indexArray, err) {
limiter.submit(err, function () {
if(err) throw err;
request({
encoding: null,
method: "GET",
headers: {
'User-Agent': 'request'
},
uri: listItem
}, function (error, response, body, err) {
if(err) throw err;
$ = cheerio.load(body);
if($('div.alert.alert-danger').html() == undefined) {
console.log(listItem);
emptyUrlArray.push(listItem);
} else {
console.log("Else");
//Not done.
}
});
});
});
};
App.js
var express = require('express');
var session = require('express-session');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
//var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var scheduleObject = require('./UtilityLogic/scheduleObject');
var scheduleBidding = require("./GoogleAPILogic/scheduleBidding");
var dbModules = require("./DBLogic/dbModules");
var routes = require('./routes/index');
var app = express();
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');
app.enable('trust proxy');
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(session({
secret: 'secret',
resave: false,
saveUninitialized: true,
cookie: {
httpOnly: true,
secure: true
}
}));
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', routes);
// catch 404 and forward to error handler
app.use(function(req, res, next) {
var err = new Error('Not Found');
err.status = 404;
next(err);
});
// error handlers
// development error handler
// will print stacktrace
if (app.get('env') === 'development') {
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
throw err;
});
}
// production error handler
// no stacktraces leaked to user
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
});
var scheduler = new scheduleObject();
var dbObject = new dbModules("./somePath");
var scheduleBidObject = new scheduleBidding("someKey");
//Test connection to DB
dbObject.testConnectionToDB();
scheduler.scheduleFunction('59 59 23 * * *',function (){
scheduleBidObject.getBiddingHeaders('/SomePath/');
});
scheduler.scheduleFunction('01 e 10 00 * * *',function (){
scheduleBidObject.getBiddingHeaders('/SomePath/');
});
module.exports = app;
抛出异常
Error: Can't set headers after they are sent.
www-2 at ServerResponse.OutgoingMessage.setHeader (http.js:691:11)
www-2 at ServerResponse.header (/somePath/node_modules/express/lib/response.js:719:10)
www-2 at ServerResponse.send (/somePath/node_modules/express/lib/response.js:164:12)
www-2 at done (/somePath/node_modules/express/lib/response.js:956:10)
www-2 at Object.exports.renderFile (/somePath/node_modules/jade/lib/index.js:374:12)
www-2 at View.exports.__express [as engine] (/somePath/node_modules/jade/lib/index.js:417:11)
www-2 at View.render (/somePath/node_modules/express/lib/view.js:126:8)
www-2 at tryRender (/somePath/node_modules/express/lib/application.js:639:10)
www-2 at EventEmitter.render (/somePath/node_modules/express/lib/application.js:591:3)
www-2 at ServerResponse.render (/somePath/node_modules/express/lib/response.js:960:7)
【问题讨论】:
-
//Do something with the body的代码是什么? -
现在更新了代码。我正在关注给定网页上的特定元素。
-
我想过,但在另一种情况下,他在 req.res 之前调用了 res.redirect,如果我是正确的,这会导致错误。我应该在函数执行后首先调用 res.send 。我可能错了,但我看不到解决方案。
-
这通常发生在您调用 res.send 两次时。这是您在 router.post 中真正拥有的所有代码吗?
标签: node.js express post header web-crawler