Mongoose: populate match not working

[paidon]

What is wrong about my code?
Tried to filter using match in populate but only getting null for b_id:

my code is as follows:

exports.search = function (req, res) {
    var b = req.body.b;
    var a = req.body.a;
    J.find()
        .populate({
            path: 'b_id',
            match: {name: b},
            model: B
        })
        .populate({
            path: 'w_id',
            model: W
        })
        .exec(function (err, j) {
            W.populate(j, {
                path: 'w_id.a_id',
                model: A
            }, function (err, j) {
                console.log(j);
            });
        });
}

everything works fine I leave out the match and vars a and b are also filled correctly.

Any suggestions?

Many thanks!

EDIT:

These are my schemas:

var B = new Schema({
    _id             : Number,
    name            : String
}, { collection: 'b' });

var A = new Schema({
    _id             : Number,
    name            : String
}, { collection: 'a' });

var W = new Schema({
    _id             : Number,
    a_id            : { type: Number, ref: 'a' },
    bez     : String
}, { collection: 'w' });

var J = new Schema({
    _id             : Number,
    b_id    : { type: Number, ref: 'b' },
    w_id        : { type: Number, ref: 'w' },
    s1          : String,
    e1          : String,
    s2          : String,
    e2          : String,
    bdh : String
}, { collection: 'j' });

[ebensing]

I'm unable to reproduce this error. Can you provide code to reproduce it?

[lefnire]

Same issue, following code:

Group
        .findOne({type: 'party', members: {'$in': [user._id]}})
        .populate({
          path: 'members',
          match: {_id: {$ne: user._id}}, //fixme this causes it to hang??
          select: 'profile preferences items stats achievements party backer ' + usernameFields
        })
        .exec(cb);

causes the query to hang. Remove the match directive and everything works fine. S'ok in this case, I can take the duplicate personal user.

[aheckmann]

Not having any luck reproducing either of these two issues. Closing for now. If you are able to post some working code that reproduces the error, please open a new ticket.

[MangeshPimpalkar]

match does not work.
When i test it with mocha it works fine but the same with angular fails.

If i remove match then it works in angular.

Program.find({'isActive': true}).populate({
path: 'network',
match: { isVisible: true },
select: '_id name'
}).exec(function (err, programs) {
res.json(programs);
});

[vkarpov15]

Hmm how are you using angular to use populate and match? Can you provide a code sample that demonstrates your issue?

[YoleYu]

var ruleSchema = new Schema({
    name: {type: String, unique: true},
    request: {type:Schema.Types.ObjectId, ref: 'Request'},
    response: {type:Schema.Types.ObjectId, ref: 'Response'},

    comment: "String",
    createTime: {type: Date, default: Date.now}
},{ strict: false });

var requestSchema = new Schema({
    //id: {type: String, unique: true},
    method: "String",
    scheme: "String",
    host: "String",
    port: "Number",
    //path: "path",
    firstPath: {type: String, trim: true },
    path: [String],// something like:  /product/abc/123
    queryString: "String", // stringfied parms
    fragment: "String", //Anchor

    headers: [String],

    comment: "String",
    createTime: {type: Date, default: Date.now}
},{ strict: false });

var queryRule = function(req, callback){
  Rule.find().populate({path:'request' , match:{method: 'POST'} }).exec(function(error, rules){
  //Rule.find().exec(function(error, rules){
    if(error){
      callback(error)
    }else{
      callback(null, rules)
    }
  });
}

I have inserted 4 records in Rule schema with only one record have method equals POST, but with the code, I can query out 4 records.

[vkarpov15]

So for me, the following code:

var mongoose = require('mongoose');
var Schema = mongoose.Schema;

mongoose.connect('mongodb://localhost:27017/gh-1548');

var ruleSchema = new Schema({
    name: {type: String, unique: true},
    request: {type:Schema.Types.ObjectId, ref: 'Request'},

    comment: "String",
    createTime: {type: Date, default: Date.now}
},{ strict: false });

var requestSchema = new Schema({
    //id: {type: String, unique: true},
    method: "String",
    scheme: "String",
    host: "String",
    port: "Number",
    //path: "path",
    firstPath: {type: String, trim: true },
    path: [String],// something like:  /product/abc/123
    queryString: "String", // stringfied parms
    fragment: "String", //Anchor

    headers: [String],

    comment: "String",
    createTime: {type: Date, default: Date.now}
},{ strict: false });

var Rule = mongoose.model('Rule', ruleSchema);
var Request = mongoose.model('Request', requestSchema);

var post = { method: 'POST' };
var get = { method: 'GET' };
Request.create(post, get, get, get, function(err, p, g, g, g) {
  if (err) {
    throw err;
  }

  Rule.remove({}, function() {
    Rule.create({ name: 'a', request: p._id }, { name: 'b', request: g._id }, function(err, r1, r2) {
      if (err) {
        throw err;
      }
      Rule.find().populate({path:'request' , match:{method: 'POST'} }).exec(function(err, rules) {
        if (err) {
          throw err;
        }

        console.log(JSON.stringify(rules, null, '  '));
      });
    });
  });
});

Outputs

[
  {
    "_id": "54c12276fcb2488d300795e4",
    "name": "a",
    "request": {
      "_id": "54c12276fcb2488d300795e0",
      "method": "POST",
      "__v": 0,
      "createTime": "2015-01-22T16:16:54.942Z",
      "headers": [],
      "path": []
    },
    "__v": 0,
    "createTime": "2015-01-22T16:16:54.999Z"
  },
  {
    "_id": "54c12277fcb2488d300795e5",
    "name": "b",
    "request": null,
    "__v": 0,
    "createTime": "2015-01-22T16:16:55.000Z"
  }
]

Which appears to the correct behavior: returns all rules with populated request iff the request has method = 'post'. Is this the same behavior you're seeing?

[YoleYu]

In my opinion, I think it should only return the rule with name = "a". Maybe I lost something about mongodb. I think it's some kind of similar with the JOIN in relational database, so I think it should only return only the rule with name = "a"

[vkarpov15]

Yeah there is an argument for that behavior, but IMO that would be too much mongoose magic under the hood in order to make the query behave properly with limits. MongoDB itself doesn't support anything like joins. The way populate() works is that mongoose does a query for the Rules and then a separate query for all the Requests associated with those rules, with optional filters.

[ChristianRich]

+1

Populate match does not filter. It just returns all the records.

[d4goxn]

So to clarify, when a populate clause contains a match statement, that doesn't affect the number of results? It just determines whether or not each individual result has its populate.path populated?

Given the following documents:

// Story collection
[{
  title: "A family story",
  fans: [ObjectId(123), ObjectId(456)],
}, {
  title: "A kid's story",
  fans: [ObjectId(123)],
}];

// Fans collection
[{
  _id: ObjectId(123),
  age: 10,
  name: "Some Kid"
}, {
  _id: ObjectId(456),
  age: 35,
  name: "Dad"
}];

and the following query:

Story
.find(...)
.populate({
  path: 'fans',
  match: { age: { $gte: 21 }},
  options: { limit: 5 }
})
.exec()

Would this filter stories, like so:

 // Result
[{
  title: "A family story",
  fans: [{
    _id: ObjectId(123),
    age: 10,
    name: "Some Kid"
  }, {
    _id: ObjectId(456),
    age: 35,
    name: "Dad"
  }]
}]

or would it constrain the population of the fans on the stories, like so:

// Result
[{
  title: "A family story",
  fans: [
  ObjectId(123), // not populated
  { // populated
    _id: ObjectId(456),
    age: 35,
    name: "Dad"
  }]
}, {
  title: "A kid's story",
  fans: [ObjectId(123)],
}];

If the latter behaviour is what is supposed to happen, then may I respectfully ask what the intended use case for the populate.match clause is? And also, would you guys be open to a pull request that implemented the former behaviour, under a different name than "match"?

_update_ Corrected s/where/match/g, because I was thinking of populate.match as if it was equivalent to SQL's where clause.
_update_ Removed irrelevant populate.select from the example query.

[vkarpov15]

The latter behavior is correct: match will filter fans based on age. The issue in this particular thread was due to a misunderstanding of how populate.limit works, populate.match works as you expect.

[d4goxn]

Thanks for the clarification. Since I need the former behaviour, I have gone with a different solution: denormalize the fields that I need to sort and filter on, into the stories collection. Updates to the Fan model are a little more complex, requiring a denormalization step in a pre-save handler. I also need to periodically denormalize the entire collection, just in case someone or something runs a Mongo bulk update operation.

But, the query logic is simple and fast and updates are rare, so I'm ok with the solution. In my specific case, which is a little more complex than the example I copied from the docs above, the only alternative that I could see would have been to bring in a relational db to manage what really is a relationship between two types of documents.

[vkarpov15]

@d4goxn that's the tradeoff with mongodb. Reads are much faster because you're storing all the data you want to sort and filter on in one collection, but updates are more cumbersome. In many web and mobile apps this is great, because users read data much more often than they write it. Also, that's why mongoose has pre save hooks :)

[AbdulRehman3]

I have detailed question about this issue, if you can look at

#7754