来自多个集合的 $lookup 和嵌套输出答案

【问题标题】：$lookup from Multiple Collections, and nested output来自多个集合的 $lookup 和嵌套输出
【发布时间】：2018-02-20 08:06:52
【问题描述】：

我有多个集合，我使用了单独的集合和外键的方法，我想加入这个集合来构建一个嵌套的集合。这是我的集合模式：

const SurveySchema = new Schema({
_id:{ type: Schema.ObjectId, auto: true },
name: String,
enabled: {type: Boolean, Default: true},
created_date:{type: Date, Default: Date.now},
company: {type: Schema.Types.ObjectId, ref: 'Company'},});

const GroupSchema = new Schema({
  _id:{ type: Schema.ObjectId, auto: true },
  name: String,
  order: String,
  created_date:{type: Date, Default: Date.now},
  questions: [{type: Schema.Types.ObjectId, ref: 'Question'}],
  survey: {type: Schema.Types.ObjectId, ref: 'Survey'}
});

const ResponseSchema = new Schema({
  _id:{ type: Schema.ObjectId, auto: true },
  response_text: String,
  order: String,
  created_date:{type: Date, Default: Date.now},
  question:{type: Schema.Types.ObjectId, ref: 'Question'}
});

这是我构建这个嵌套对象的代码：

Survey.aggregate([
  { $match: {} },
  { $lookup: {
    from: 'groups',
    localField: '_id',
    foreignField: 'survey',
    as: 'groupsofquestions',
  }},
  { $unwind: {
    path: "$groupsofquestions",
    preserveNullAndEmptyArrays: true
  }},
  { $lookup: {
    from: 'questions',
    localField: 'groupsofquestions._id',
    foreignField: 'group',
    as: 'questionsofgroup',
  }},
  { $lookup: {
    from: 'response',
    localField: 'questionsofgroup._id',
    foreignField: 'question',
    as: 'responses',
  }},
  { $group: {
    _id: "$_id",
    name: {$first: "$name"},
    groups: {$push: {
      id: "$groupsofquestions._id",
      name: "$groupsofquestions.name",
      questions: "$questionsofgroup",
      reponses: "$responses"
    }}
  }}
])

我想如下结构，（也有外部链接）：

http://jsoneditoronline.org/?id=d7d1779b3b95e3acb28f8a2be0785423

[
  {
    "__v": 0,
    "_id": "59b6715725dcd2060da7f591",
    "company": "59b6715725dcd2060da7f58f",
    "created_date": "2017-09-11T11:19:51.709Z",
    "enabled": true,
    "name": "function String() { [native code] }",
    "groups": [
      {
        "_id": "59b6715725dcd2060da7f592",
        "name": "groupe 1 des question",
        "order": "1",
        "created_date": "2017-09-11T11:19:51.709Z",
        "survey": "59b6715725dcd2060da7f591",
        "__v": 0,
        "questions": [
          {
            "_id": "59b6715725dcd2060da7f594",
            "question_text": "question 1 group 1",
            "order": "1",
            "created_date": "2017-09-11T11:19:51.709Z",
            "group": "59b6715725dcd2060da7f592",
            "__v": 0,
            "responses": [
              {
                "_id": "59b6715725dcd2060da7f598",
                "response_text": "reponse 1 question 1 group 1",
                "order": "1",
                "created_date": "2017-09-11T11:19:51.710Z",
                "question": "59b6715725dcd2060da7f594",
                "__v": 0
              },
              {
                "_id": "59b6715725dcd2060da7f599",
                "response_text": "reponse 2 question 1 group 1",
                "order": "2",
                "created_date": "2017-09-11T11:19:51.710Z",
                "question": "59b6715725dcd2060da7f594",
                "__v": 0
              }
            ]
          },
          {
            "_id": "59b6715725dcd2060da7f595",
            "question_text": "question 2 group 1",
            "order": "2",
            "created_date": "2017-09-11T11:19:51.710Z",
            "group": "59b6715725dcd2060da7f592",
            "__v": 0,
            "responses": [
              {
                "_id": "59b6715725dcd2060da7f59a",
                "response_text": "reponse 1 question 2 group 1",
                "order": "1",
                "created_date": "2017-09-11T11:19:51.710Z",
                "question": "59b6715725dcd2060da7f595",
                "__v": 0
              },
              {
                "_id": "59b6715725dcd2060da7f59b",
                "response_text": "reponse 2 question 2 group 1",
                "order": "2",
                "created_date": "2017-09-11T11:19:51.710Z",
                "question": "59b6715725dcd2060da7f595",
                "__v": 0
              }
            ]
          }
        ]
      },
      {
        "_id": "59b6715725dcd2060da7f593",
        "name": "groupe 2 des question",
        "order": "2",
        "created_date": "2017-09-11T11:19:51.709Z",
        "survey": "59b6715725dcd2060da7f591",
        "__v": 0,
        "questions": [
          {
            "_id": "59b6715725dcd2060da7f596",
            "question_text": "question 1 group 1",
            "order": "1",
            "created_date": "2017-09-11T11:19:51.710Z",
            "group": "59b6715725dcd2060da7f592",
            "__v": 0,
            "responses": [
              {
                "_id": "59b6715725dcd2060da7f59c",
                "response_text": "reponse 1 question 1 group 2",
                "order": "1",
                "created_date": "2017-09-11T11:19:51.710Z",
                "question": "59b6715725dcd2060da7f596",
                "__v": 0
              },
              {
                "_id": "59b6715725dcd2060da7f59d",
                "response_text": "reponse 2 question 1 group 2",
                "order": "2",
                "created_date": "2017-09-11T11:19:51.710Z",
                "question": "59b6715725dcd2060da7f596",
                "__v": 0
              }
            ]
          },
          {
            "_id": "59b6715725dcd2060da7f597",
            "question_text": "question 2 group 1",
            "order": "2",
            "created_date": "2017-09-11T11:19:51.710Z",
            "group": "59b6715725dcd2060da7f592",
            "__v": 0,
            "responses": [
              {
                "_id": "59b6715725dcd2060da7f59e",
                "response_text": "reponse 1 question 2 group 2",
                "order": "1",
                "created_date": "2017-09-11T11:19:51.710Z",
                "question": "59b6715725dcd2060da7f597",
                "__v": 0
              },
              {
                "_id": "59b6715725dcd2060da7f59f",
                "response_text": "reponse 2 question 2 group 2",
                "order": "2",
                "created_date": "2017-09-11T11:19:51.710Z",
                "question": "59b6715725dcd2060da7f597",
                "__v": 0
              }
            ]
          }
        ]
      }
    ]
  }
]

有人可以帮我按照示例中所示的方式构建响应吗？

【问题讨论】：

您使用preserveNullAndEmptyArrays 有什么原因，还是您只是从某个地方复制了示例？在 $lookup 与外部集合中的任何结果不匹配的情况下，它确实会有所不同。然而，如果在关系总是期望匹配某些东西时不需要这样做，这是一个合理的性能改进。您还错过了 Question 架构定义，但我们可以合理地假设它与示例输出匹配，因为其他架构也匹配。
我不确定你在问什么，因为你似乎正在做你需要做的一切。

标签： javascript node.js mongodb mongoose aggregation-framework

【解决方案1】：

大多数情况下，您需要在使用$unwind 处理后使用$group 来“重建”，以便再次嵌套您的数组输出。还有一些提示：

   Survey.aggregate([
      { "$lookup": {
        "from": Group.collection.name,
        "localField": "_id",
        "foreignField": "survey",
        "as": "groups"
      }},
      { "$unwind": "$groups" },
      { "$lookup": {
        "from": Question.collection.name,
        "localField": "groups.questions",
        "foreignField": "_id",
        "as": "groups.questions"
      }},
      { "$unwind": "$groups.questions" },
      { "$lookup": {
        "from": Response.collection.name,
        "localField": "groups.questions._id",
        "foreignField": "question",
        "as": "groups.questions.responses"
      }},
      { "$group": {
        "_id": {
          "_id": "$_id",
          "company": "$company",
          "created_date": "$created_date",
          "enabled": "$enabled",
          "name": "$name",
          "groups": {
            "_id": "$groups._id",
            "name": "$groups.name",
            "order": "$groups.order",
            "created_date": "$groups.created_date",
            "survey": "$groups.survey"
          }
        },
        "questions": { "$push": "$groups.questions" }
      }},
      { "$sort": { "_id": 1 } },
      { "$group": {
        "_id": "$_id._id",
        "company": { "$first": "$_id.company" },
        "created_date": { "$first": "$_id.created_date" },
        "enabled": { "$first": "$_id.enabled" },
        "name": { "$first": "$_id.name" },
        "groups": {
          "$push": {
            "_id": "$_id.groups._id",
            "name": "$_id.groups.name",
            "order": "$_id.groups.order",
            "created_date": "$_id.groups.created_date",
            "survey": "$_id.groups.survey",
            "questions": "$questions"
          }
        }
      }},
      { "$sort": { "_id": 1 } }
    ]);

这就是重建数组的方法，您一次只做一步，而不是试图一次性完成所有工作。这可能是最难理解的概念，但“管道”意味着您实际上可以“多次”执行操作，将一个操作链接到另一个操作的输出。

所以第一个$group 是在“组”详细信息级别完成的，因为您想要$push 数组中的"questions" 项目，这是$unwind 的最后一个“解构”。请注意，"responses" 仍然是一个数组，作为最后一个$lookup 阶段的结果。但除了数组内容之外，其他所有内容都在_id“分组键”中。

在“第二个”$group 上，您实际上使用$first 之类的运算符来构造Survey 级别的特定字段属性。 "groups" 数组又是用$push 构造的，因此上一阶段的“分组键”中的每个属性都以_id 为前缀，所以这里都是引用它们的方式。

此外，从技术角度来看，如果您有预期的订单，您应该在每次调用 $group 之后始终 $sort。分组键的集合不保证以任何特定顺序（尽管通常它是反向堆栈顺序）。如果您期望一个顺序，请指定它，尤其是在应用 $push 以在 $group 之后重构数组时。

之所以没有$sort之前最初的$group是因为前面的管道阶段实际上对现有订单没有任何影响。因此，发现的顺序始终保持不变。

几个提示：

像Group.collection.name 这样的东西实际上使用猫鼬模型上定义的属性来做像“获取集合名称”这样的事情。这使您免于对 $lookup 本身进行硬编码，并与运行代码时模型上注册的任何内容保持一致。
除非你是认真的，否则不要使用preserveNullAndEmptyArrays 之类的选项。有一种“特殊方式”，$lookup + $unwind 的组合实际上被处理，并且真正在“单个阶段”中执行，而不是在“展开”之前检索所有结果。您可以在聚合管道的“解释”输出中看到这一点。简而言之，如果您总是有关系匹配，则不要使用该选项。最好不要。

演示

作为一个完整的清单和概念证明，我们可以加载您的源 JSON，将其存储在数据库中的单独集合中，然后使用聚合语句来检索和重建所需的结构：

const fs = require('fs'),
      mongoose = require('mongoose'),
      Schema = mongoose.Schema;

mongoose.Promise = global.Promise;
mongoose.set('debug',true);

const uri = 'mongodb://localhost/nested',
      options = { useMongoClient: true };

const responseSchema = new Schema({
  response_text: String,
  order: String,
  created_date: Date,
  question: { type: Schema.Types.ObjectId, ref: 'Question' }
});

const questionSchema = new Schema({
  question_text: String,
  order: String,
  created_date: Date,
  group: { type: Schema.Types.ObjectId, ref: 'Group' }
});

const groupSchema = new Schema({
  name: String,
  order: String,
  created_date: Date,
  survey: { type: Schema.Types.ObjectId, ref: 'Survey' },
  questions: [{ type: Schema.Types.ObjectId, ref: 'Question' }]
});

const surveySchema = new Schema({
  company: { type: Schema.Types.ObjectId, ref: 'Company' },
  created_date: Date,
  enabled: Boolean,
  name: String
});

const companySchema = new Schema({

});

const Company = mongoose.model('Company', companySchema);
const Survey = mongoose.model('Survey', surveySchema);
const Group = mongoose.model('Group', groupSchema);
const Question = mongoose.model('Question', questionSchema);
const Response = mongoose.model('Response', responseSchema);


function log(data) {
  console.log(JSON.stringify(data,undefined,2))
}

(async function() {

  try {

    const conn = await mongoose.connect(uri,options);

    await Promise.all(
      Object.keys(conn.models).map( m => conn.models[m].remove() )
    );

    // Initialize data
    let content = JSON.parse(fs.readFileSync('./jsonSurveys.json'));
    //log(content);

    for ( let item of content ) {

      let survey = await Survey.create(item);
      let company = await Company.create({ _id: survey.company });

      for ( let group of item.groups ) {
        await Group.create(group);
        for ( let question of group.questions ) {
          await Question.create(question);
          for ( let response of question.responses ) {
            await Response.create(response);
          }
        }
      }

    }

    // Run aggregation

    let results = await Survey.aggregate([
      { "$lookup": {
        "from": Group.collection.name,
        "localField": "_id",
        "foreignField": "survey",
        "as": "groups"
      }},
      { "$unwind": "$groups" },
      { "$lookup": {
        "from": Question.collection.name,
        "localField": "groups.questions",
        "foreignField": "_id",
        "as": "groups.questions"
      }},
      { "$unwind": "$groups.questions" },
      { "$lookup": {
        "from": Response.collection.name,
        "localField": "groups.questions._id",
        "foreignField": "question",
        "as": "groups.questions.responses"
      }},
      { "$group": {
        "_id": {
          "_id": "$_id",
          "company": "$company",
          "created_date": "$created_date",
          "enabled": "$enabled",
          "name": "$name",
          "groups": {
            "_id": "$groups._id",
            "name": "$groups.name",
            "order": "$groups.order",
            "created_date": "$groups.created_date",
            "survey": "$groups.survey"
          }
        },
        "questions": { "$push": "$groups.questions" }
      }},
      { "$sort": { "_id": 1 } },
      { "$group": {
        "_id": "$_id._id",
        "company": { "$first": "$_id.company" },
        "created_date": { "$first": "$_id.created_date" },
        "enabled": { "$first": "$_id.enabled" },
        "name": { "$first": "$_id.name" },
        "groups": {
          "$push": {
            "_id": "$_id.groups._id",
            "name": "$_id.groups.name",
            "order": "$_id.groups.order",
            "created_date": "$_id.groups.created_date",
            "survey": "$_id.groups.survey",
            "questions": "$questions"
          }
        }
      }},
      { "$sort": { "_id": 1 } }
    ]);

    log(results);

  } catch(e) {
    console.error(e);
  } finally {
    mongoose.disconnect();
  }


})();

另一种情况

另外值得注意的是，通过一些小的架构更改，可以通过使用对.populate() 的嵌套调用来实现相同的结果：

  let alternate = await Survey.find().populate({
      path: 'groups',
      populate: {
        path: 'questions',
        populate: {
          path: 'responses'
        }
      }
    });

虽然它看起来更简单，但实际上引入了更多负载，因为这会向数据库发出多个查询以检索数据，而不是一次调用：

Mongoose: groups.find({ survey: { '$in': [ ObjectId("59b6715725dcd2060da7f591") ] } }, { fields: {} })
Mongoose: questions.find({ _id: { '$in': [ ObjectId("59b6715725dcd2060da7f594"), ObjectId("59b6715725dcd2060da7f595"), ObjectId("59b6715725dcd2060da7f596"), ObjectId("59b6715725dcd2060da7f597") ] } }, { fields: {} })
Mongoose: responses.find({ question: { '$in': [ ObjectId("59b6715725dcd2060da7f594"), ObjectId("59b6715725dcd2060da7f595"), ObjectId("59b6715725dcd2060da7f596"), ObjectId("59b6715725dcd2060da7f597") ] } }, { fields: {} })

您可以在修改后的清单中看到架构更改（只是为连接添加了虚拟字段）以及正在运行的代码：

const fs = require('fs'),
      mongoose = require('mongoose'),
      Schema = mongoose.Schema;

mongoose.Promise = global.Promise;
mongoose.set('debug',true);

const uri = 'mongodb://localhost/nested',
      options = { useMongoClient: true };

const responseSchema = new Schema({
  response_text: String,
  order: String,
  created_date: Date,
  question: { type: Schema.Types.ObjectId, ref: 'Question' }
});

const questionSchema = new Schema({
  question_text: String,
  order: String,
  created_date: Date,
  group: { type: Schema.Types.ObjectId, ref: 'Group' }
},{
  toJSON: {
    virtuals: true,
    transform: function(doc,obj) {
      delete obj.id;
      return obj;
    }
  }
});

questionSchema.virtual('responses',{
  ref: 'Response',
  localField: '_id',
  foreignField: 'question'
});

const groupSchema = new Schema({
  name: String,
  order: String,
  created_date: Date,
  survey: { type: Schema.Types.ObjectId, ref: 'Survey' },
  questions: [{ type: Schema.Types.ObjectId, ref: 'Question' }]
});

const surveySchema = new Schema({
  company: { type: Schema.Types.ObjectId, ref: 'Company' },
  created_date: Date,
  enabled: Boolean,
  name: String
},{
  toJSON: {
    virtuals: true,
    transform: function(doc,obj) {
      delete obj.id;
      return obj;
    }
  }
});

surveySchema.virtual('groups',{
  ref: 'Group',
  localField: '_id',
  foreignField: 'survey'
});

const companySchema = new Schema({

});

const Company = mongoose.model('Company', companySchema);
const Survey = mongoose.model('Survey', surveySchema);
const Group = mongoose.model('Group', groupSchema);
const Question = mongoose.model('Question', questionSchema);
const Response = mongoose.model('Response', responseSchema);


function log(data) {
  console.log(JSON.stringify(data,undefined,2))
}

(async function() {

  try {

    const conn = await mongoose.connect(uri,options);

    await Promise.all(
      Object.keys(conn.models).map( m => conn.models[m].remove() )
    );

    // Initialize data
    let content = JSON.parse(fs.readFileSync('./jsonSurveys.json'));
    //log(content);

    for ( let item of content ) {

      let survey = await Survey.create(item);
      let company = await Company.create({ _id: survey.company });

      for ( let group of item.groups ) {
        await Group.create(group);
        for ( let question of group.questions ) {
          await Question.create(question);
          for ( let response of question.responses ) {
            await Response.create(response);
          }
        }
      }

    }

    // Run aggregation

    let results = await Survey.aggregate([
      { "$lookup": {
        "from": Group.collection.name,
        "localField": "_id",
        "foreignField": "survey",
        "as": "groups"
      }},
      { "$unwind": "$groups" },
      { "$lookup": {
        "from": Question.collection.name,
        "localField": "groups.questions",
        "foreignField": "_id",
        "as": "groups.questions"
      }},
      { "$unwind": "$groups.questions" },
      { "$lookup": {
        "from": Response.collection.name,
        "localField": "groups.questions._id",
        "foreignField": "question",
        "as": "groups.questions.responses"
      }},
      { "$group": {
        "_id": {
          "_id": "$_id",
          "company": "$company",
          "created_date": "$created_date",
          "enabled": "$enabled",
          "name": "$name",
          "groups": {
            "_id": "$groups._id",
            "name": "$groups.name",
            "order": "$groups.order",
            "created_date": "$groups.created_date",
            "survey": "$groups.survey"
          }
        },
        "questions": { "$push": "$groups.questions" }
      }},
      { "$sort": { "_id": 1 } },
      { "$group": {
        "_id": "$_id._id",
        "company": { "$first": "$_id.company" },
        "created_date": { "$first": "$_id.created_date" },
        "enabled": { "$first": "$_id.enabled" },
        "name": { "$first": "$_id.name" },
        "groups": {
          "$push": {
            "_id": "$_id.groups._id",
            "name": "$_id.groups.name",
            "order": "$_id.groups.order",
            "created_date": "$_id.groups.created_date",
            "survey": "$_id.groups.survey",
            "questions": "$questions"
          }
        }
      }},
      { "$sort": { "_id": 1 } }
    ]);

    log(results);

    let alternate = await Survey.find().populate({
      path: 'groups',
      populate: {
        path: 'questions',
        populate: {
          path: 'responses'
        }
      }
    });

    log(alternate);

  } catch(e) {
    console.error(e);
  } finally {
    mongoose.disconnect();
  }


})();

【讨论】：

嗨，这是最完整的冻糕答案，非常感谢:)，
我还有一个问题，我会在前面使用 angular，我想分别创建和更新每个集合（调查、组、问题）并使用 $lookup 来显示调查，是这是可能的和正确的。对不起我的英语不好:(。
@SoufianeLam 当您有新问题时Ask a new Question。花时间写和解释，有人会尝试回答。可能吗。是的。我一直都这样做。这是一个相当广泛的问题。最好看看How do I ask a good question? 和What types of questions should I avoid asking?。只要你说的足够清楚你的意图，英语的准确性就没有那么重要了。其他人可以修改。
谢谢你的回答，好的，这是我在 Stackoverflow 中的第一个 ASK，下次我会解释并花时间写一个问题:)。