NodeJS中来自MongoDB的Json数据

Question

我正在尝试使用 nodejs rest 服务从 mongodb 集合中获取大量 json 数据。但是在获取数据时出现此错误：

[8744:00000000003A9550] 164892 ms：Mark-sweep 1407.6 (1468.2) -> 1407.5 (1437.2) MB，2510.7 / 0.0 ms 最后手段 GC 在旧空间请求
[8744:00000000003A9550] 167480 ms：Mark-sweep 1407.5 (1437.2) -> 1407.5 (1436.7) MB，2587.7 / 0.0 ms 请求旧空间中的最后手段 GC

==== JS 堆栈跟踪 ========================================== =

安全上下文：000000E816AA5879
1: _writeGeneric [net.js:755] [bytecode=000001F3DEF03FC1 offset=351](this=000003C99D221C61 ,writev=000001EC53F02371 ,data=0000010D3E5FBCA9 ,encoding=000001EC53F02431 ,cb=0000 2: _writev [net.js:778] [字节码=0000000C48D59649 偏移量=23](this=000003C99D221C...

FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory
 1: node_module_register
 2: v8::internal::FatalProcessOutOfMemory
 3: v8::internal::FatalProcessOutOfMemory
 4: v8::internal::Factory::NewRawOneByteString
 5: v8::internal::Smi::SmiPrint
 6: v8::internal::StackGuard::HandleInterrupts
 7: v8::String::WriteUtf8
 8: v8::internal::wasm::AsmType::Extern
 9: v8::internal::wasm::AsmType::Extern
10: v8::internal::wasm::AsmType::Extern
11: v8::internal::wasm::SignatureMap::Find
12: v8::internal::Builtins::CallableFor
13: v8::internal::Builtins::CallableFor
14: v8::internal::Builtins::CallableFor
15: 000000BC8E7843C1

D:\new>node --max_old_space_size=2000 service.js
23-10-2018 07-08-46.1046 - info: log information extraData=somve_value

节点服务：

var getFiletredTimestamp = function(alarm , fromDate , toDate){
    var resultData = [];
    for(var i=0;i<alarm.length;i++){
        var obj = alarm[i];
        var x;
        for( x in obj['alarms'] ){
             if(new Date(obj['alarms'][x]['timestamp']).getTime()>=new Date(fromDate).getTime() && new Date(obj['alarms'][x]['timestamp']).getTime()<=new Date(toDate).getTime()){
                 resultData.push(obj);
                 break;
             }
       }
    }
    //console.log(resultData);
    console.log(resultData.length);
    return resultData;
};

function getAlarmHistory(req,res){
   var queryObject = url.parse(req.url,true).query;
   db.collection("HISTORY").find({},{"no":1,"type":1,"ID":1,"id":1,"ptype":1,"alarms":1}).toArray(function (err , result){
       console.log(result.length);
       var index = 0 , resultset = [];
       var docs = getFiletredTimestamp(result , queryObject.fromdate, queryObject.todate);
        var count = docs.length;
        console.log("count " , count);
        if(count==0){
                console.log("resulttttttttt",resultset);

                res.writeHead(200, {
                    'Content-Type': 'application/json'
                });
                res.write('');
                res.end();  
            }else{
       docs.forEach(function(doc){
           console.log()
               if (doc!=null){
                var valdata=doc.alarms;
                var fields = [];
                var queryString ="SELECT datafield1,LOC_NAME FROM details c inner join ldetails l on c.loc_id=l.loc_id where no='"+doc.no+"' limit 1;";
               var dtfield1 = null;
               var dtfield2 = null;
                connection.query(queryString, function(err,result){
                           index++;
                           console.log("result query" , result);
                        if(err){
                            console.log("err",err);
                        }else{
                            if(result.length>0)
                                {
                                     dtfield1 = result[0].datafield1;
                                     dtfield2 = result[0].LOC_NAME;
                                     if(dtfield1!=null){
                                         for (var x in valdata) {
                                            resultset.push({"Name":dtfield1,"LName":dtfield2,"no":doc.no,"type":doc.type,"ID":doc.ID,"aType":x,"aValue":valdata[x].value,"occurTimestamp":valdata[x].occurTimestamp,"Timestamp":valdata[x].timestamp});
                                         }} 

                                         }
                                          if(index == count){
                                                console.log("resulttttttttt",resultset);

                                                res.writeHead(200, {
                                                    'Content-Type': 'application/json'
                                                });
                                                res.write(JSON.stringify(resultset));
                                                res.end();
                                            }
                                         }
                    }); 
   }
       });}

   })

}

如何处理错误。请帮忙。

Answer 1

使用分页方法，一次只处理小块数据，或流式处理结果： http://mongodb.github.io/node-mongodb-native/2.0/tutorials/streams/