node.js - mongodb中数据find出来，提示process out of memory，数据大小并没有超过NodeJS默认的512MB

问题描述

使用mongoose从数据库中一次性find出数据，提示：

但是我查了default the memory limit of Node.js is 512 mb，我的数据集合大小只有127MB，并没有超过这个大小，也不需要设置--max_old_space_size吧，求解

代码如下：

//model.jsvar LagouInfo = require(’./schema.js’)var DesData = require(’./des.js’);var async = require(’async’);LagouInfo.find({}, function(res){ res.forEach(function(item){if(item.content.trim()){ var sumSalary = 0; item.salary.forEach(function(sitem){var num;if(sitem.indexOf(’-’) == -1){ num = sitem.replace(/D+/, ’’);}else{ num = parseInt(sitem.trim().split(’-’)[1]);}// console.log(sitem, num);sumSalary += num; }) var tags = item.tag.trim().split(’,’); tags.forEach(function(tag){DesData.update({’tag’: tag}, {$push: {’content’: item.content}});DesData.update({’tag’: tag}, {$inc: {’total’: item.total}});DesData.update({’tag’: tag}, {$inc:{’salary’: sumSalary}}); })} })})

//des.jsvar mongoose = require(’./db.js’), Schema = mongoose.Schema;var InfoSchema = new Schema({ tag: {type: String}, content: {type: Array}, total: {type: Number}, salary: {type: Number}, });var Data = mongoose.model(’desdata’, InfoSchema, ’desdata’);function insert(obj, callback){ var data = new Data(obj); data.save(function(err, res){if(err) console.log(’Error:’ + err);else callback(null, res); })}function update(conditions, updateStr){ Data.update(conditions, updateStr, function(err, res){if(err) console.log(’Error:’ + err);else console.log(’Update Success’);// else callback(null, res); })}function find(conditions, callback){ Data.find(conditions, function(err, res){if(err) console.log(’Error:’ + err);else callback(res); }) // return Data.find(conditions).exec();}module.exports = { insert: insert, update: update, find: find}

//schema.jsvar mongoose = require(’./db.js’), Schema = mongoose.Schema;var LagouSchema = new Schema({ name: {type: String}, cid: {type: Number}, process: {type: String}, content: {type: String}, url: {type: String}, tag: {type: String}, total: {type: Number}, salary: {type: Array}});var Lagou = mongoose.model(’lagou’, LagouSchema, ’lagou’);function update(conditions, update){ Lagou.update(conditions, update, function(err, res){if(err) console.log(’Error:’ + err);else console.log(’Res:’ + res); })}function del(conditions){ Lagou.remove(conditions, function(err, res){if(err) console.log(’Error:’ + err);else console.log(’Res:’ + res); })}function find(conditions, callback){ Lagou.find(conditions, function(err, res){if(err) console.log(’Error:’ + err);else callback(res); })}module.exports = { find: find, del: del, update: update}

//db.jsvar mongoose = require(’mongoose’), DB_URL = ’mongodb://localhost:27017/result’;mongoose.Promise = global.Promise;mongoose.connect(DB_URL);//连接成功mongoose.connection.on(’connected’, function(){ console.log(’Mongoose connection open to ’ + DB_URL);})//连接异常mongoose.connection.on(’error’, function(err){ console.log(’Mongoose connection error: ’ + err);})//连接断开mongoose.connection.on(’disconnected’, function(){ console.log(’Mongoose connection disconnected’);})module.exports = mongoose;

============更新2017-01-12===================

尝试设置了--max_old_space_size还是报错

报错内容：

$ node --max_old_space_size=2000 models/test.jsMongoose connection open to mongodb://localhost:27017/result<--- Last few GCs ---> 251717 ms: Mark-sweep 1989.2 (2067.0) -> 1989.3 (2067.0) MB, 1564.0 / 0 ms [allocation failure] [scavenge might not succeed]. 253271 ms: Mark-sweep 1989.3 (2067.0) -> 1989.2 (2067.0) MB, 1554.2 / 0 ms [allocation failure] [scavenge might not succeed]. 254868 ms: Mark-sweep 1989.2 (2067.0) -> 1989.3 (2067.0) MB, 1597.0 / 0 ms [last resort gc]. 256434 ms: Mark-sweep 1989.3 (2067.0) -> 1989.2 (2067.0) MB, 1566.2 / 0 ms [last resort gc].<--- JS stacktrace --->==== JS stack trace =========================================Security context: 00000394003B4639 <JS Object> 2: /* anonymous */(aka /* anonymous */) [D:softwareself_learnPaperpreprocessnode_modulesmongodb-corelibconnectionpool.js:~1096] [pc=000002BA3DA6C022] (this=00000394003041B9 <undefined>) 3: waitForAuth(aka waitForAuth) [D:softwareself_learnPaperpreprocessnode_modulesmongodb-corelibconnectionpool.js:1088] [pc=000002BA41DE1FBC] (this=00000394003041B9 <undefined>,cb=0000...FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory

问题解答

感觉这个报错和mongodb没有关系了 是node程序的内存问题吧

试试这样子可以吗

node --max_old_space_size=2000 server.js

FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory

首先，数据大小在数据库里面看到的和程序读到内存里的体积是不同的。建议加些条件，缩小数据量，看是否有问题？然后加多点数据，看是否有问题？看看有问题的时候记录条数是多少。第二，我怎么觉得你这函数写的有点问题？DesData.update是异步方法，你用的是同步写法，然后我看你的tags.foreach方法应该是循环多次的，多少次我不清楚，但应该挺大的，导致你oom问题的很可能出现在这里----堆积太多异步方法等待执行了。而且你这样写异步真的好吗。建议用async包来改一下你的代码，最好用forEachLimit，手机打字就不直接帮你改代码了，如果还不行，我上电脑的时候帮你改。

能否贴出您的部分代码，看看代码是否有环节存在潜在Issue。