Mongoose: populate option limit - not working

[keithics]

code below gives 2 friends instead of 1.

var opts = {
path: 'author.friends',
select: 'name',
options: { limit: 1 }
}

  var assert = require('assert')
  var mongoose = require('mongoose');
  var Schema = mongoose.Schema;
  var ObjectId = mongoose.Types.ObjectId;

  /**
   * Connect to the db
   */

  var dbname = 'testing_populateAdInfinitum_1'
  mongoose.connect('localhost', dbname);
  mongoose.connection.on('error', function() {
    console.error('connection error', arguments);
  });

  /**
   * Schemas
   */

  var user = new Schema({
    name: String,
    friends: [{
      type: Schema.ObjectId,
      ref: 'User'
    }]
  });
  var User = mongoose.model('User', user);

  var blogpost = Schema({
    title: String,
    tags: [String],
    author: {
      type: Schema.ObjectId,
      ref: 'User'
    }
  })
  var BlogPost = mongoose.model('BlogPost', blogpost);

  /**
   * example
   */

  mongoose.connection.on('open', function() {

    /**
     * Generate data
     */

    var userIds = [new ObjectId, new ObjectId, new ObjectId, new ObjectId];
    var users = [];

    users.push({
      _id: userIds[0],
      name: 'mary',
      friends: [userIds[1], userIds[2], userIds[3]]
    });
    users.push({
      _id: userIds[1],
      name: 'bob',
      friends: [userIds[0], userIds[2], userIds[3]]
    });
    users.push({
      _id: userIds[2],
      name: 'joe',
      friends: [userIds[0], userIds[1], userIds[3]]
    });
    users.push({
      _id: userIds[3],
      name: 'sally',
      friends: [userIds[0], userIds[1], userIds[2]]
    });

    User.create(users, function(err, docs) {
      assert.ifError(err);

      var blogposts = [];
      blogposts.push({
        title: 'blog 1',
        tags: ['fun', 'cool'],
        author: userIds[3]
      })
      blogposts.push({
        title: 'blog 2',
        tags: ['cool'],
        author: userIds[1]
      })
      blogposts.push({
        title: 'blog 3',
        tags: ['fun', 'odd'],
        author: userIds[2]
      })

      BlogPost.create(blogposts, function(err, docs) {
        assert.ifError(err);

        /**
         * Population
         */

        BlogPost
        .find({ tags: 'fun' })
        .lean()
        .populate('author')
        .exec(function(err, docs) {
          assert.ifError(err);

          /**
           * Populate the populated documents
           */

          var opts = {
            path: 'author.friends',
            select: 'name',
            options: { limit: 1 }
          }

          BlogPost.populate(docs, opts, function(err, docs) {
            assert.ifError(err);
            console.log('populated');
            var s = require('util').inspect(docs, { depth: null })
            console.log(s);
            done();
          })
        })
      })
    })
  });

  function done(err) {
    if (err) console.error(err.stack);
    mongoose.connection.db.dropDatabase(function() {
      mongoose.connection.close();
    });
  }

[wickY26]

I have exactly same issue... #1613 fixed a problem at limit on population but it multiply limit by length of result which is 2 in your example...

here is the code part

 +  // if a limit option is passed, we should have the limit apply to *each*
 +  // document, not apply in the aggregate
 +  if (options.options && options.options.limit) {
 +    options.options.limit = options.options.limit * len;
 +  }

I fixed it by finding results and populate referenced documents individually...
I think it is a bug to be fixed...

[evan-waters]

I think this issue is related to what I'm seeing.

Room
.find()
.populate({
  path: 'messages',
  select: 'message createdAt',
  options: {
    sort: { createdAt: -1 },
    limit: 1
  }
})
.exec(cb);

// Room results
[{ 
  _id: 53c04a605574680000405db2,
  messages: []
 },
{ 
  _id: 53c052dad8d226000045366f,
  messages: []
},
{
  _id: 53c05599d8d2260000453673,
  messages:
   [ { _id: 53c055b3d8d2260000453679,
       createdAt: Fri Jul 11 2014 15:22:59 GMT-0600 (MDT),
       message: 'whats up' },
     { _id: 53c055b2d8d2260000453678,
       createdAt: Fri Jul 11 2014 15:22:58 GMT-0600 (MDT),
       message: 'hello' },
     { _id: 53c055b1d8d2260000453677,
       createdAt: Fri Jul 11 2014 15:22:57 GMT-0600 (MDT),
       message: 'hey' } ]
 }]

When I try to populate a document with limit: 1, the last document in the array has seems to have set its limit equal to the number of returned Room documents and the rest of the documents' message arrays are empty.

Mongoose: 3.8.12
Mongodb: 2.6.3

[mathieumg]

I think this might not be completely fixed, I have the same issues as @evro above (empty arrays) if I populate a path that is nested.

For instance, Document.populate({ path: "a.b.c", options{ limit: 1 }, model: "C" }, cb), where a is an array of objects with each some properties and a b property pointing to a B document which contain a c property pointing to an array of C documents, will return something like:

{
  a: [ 
    { b: { _id: "ID of the populated B document", c: [  ] } },
    { b: { _id: "ID of the populated B document", c: [  ] } },
    // ...
    { b: 
      {
        _id: "ID of the populated B document", 
        c: {
          _id: "ID of the populated C document",
          cProp: "Value",
          otherCProp: 1234
        }
      }
    }
  ]
}

Of course, the initial query had populated the a field (A.populate("a.b").exec(cb);) and given the following result:

{
  a: [ 
    { b: { _id: "ID of the populated B document", c: [ "ID of a C document" ] } },
    { b: { _id: "ID of the populated B document", c: [ "ID of a C document" ] } },
    { b: { _id: "ID of the populated B document", c: [ "ID of a C document" ] } }
  ]
}

It's as though the options.limit applied at once to everything under a in "a.b.c" rather than the intended per-c.

Mongoose: 3.8.14
Mongo: 2.6.4

[johnCramer87]

+1 experiencing the same issue as @mathieumg and @evro currently with empty arrays when populating nested paths.

Mongoose: 3.8.15
Mongo: 2.6.3

[mathieumg]

@vkarpov15 Would you rather want me to open a new issue for this?

[vkarpov15]

I'll just reopen this one and investigate this when I get a chance. Thanks for pointing out your issue.

[mathieumg]

Thank you, knowing it's been acknowledged by the devs is perfect!

[vkarpov15]

@mathieumg I'd actually argue that this is expected behavior - the limit option on populate is pretty intentionally on a per-document basis. Implementing the limit on a per-element-of-array basis will be very tricky, and very janky performance-wise because it will require multiple queries to ensure that each element in the array gets at most one result. The way to implement this will be looping over a and populating each c with limit 1 - even if we implement this internally in mongoose, that's how we'd have to implement it. I'm gonna mark that particular suggestion as a "won't fix".

[codeHatcher]

@vkarpov15, was the final decision for this that it is the expected behaviour or was it adapted to @mathieumg and @johnCramer87 ? I only ask because I am experiencing the same issue.

[vkarpov15]

Final decision was 'expected behavior', because there would be no way to do this without multiple queries. As a general rule of thumb, if you're doing populate() + limit, you may want to rethink your schema design, because mongodb arrays should not grow without bound

[codeHatcher]

Ok sounds good. For me the limit 1 would have been nice because I need just 1 of
the populated documents returned (really just one populated) and thought it
might be more performant to limit 1 vs returning all 30 from the DB and
ultimately just returning 1 anyways.

Sent using CloudMagic [https://cloudmagic.com/k/d/mailapp?ct=pi&cv=6.0.64&pv=8.1.3]
On Thu, Jul 9, 2015 at 9:59 AM, Valeri Karpov [email protected] wrote:
Final decision was 'expected behavior', because there would be no way to do this
without multiple queries. As a general rule of thumb, if you're doing populate()

• limit, you may want to rethink your schema design, because mongodb arrays should not grow without bound
  [http://blog.mongodb.org/post/88473035333/6-rules-of-thumb-for-mongodb-schema-design-part-3]

—
Reply to this email directly or view it on GitHub
[https://github.com/Automattic/mongoose/issues/2151#issuecomment-119985596] .[https://github.com/notifications/beacon/AFHNVkGfcuyZSzwaH8d29reTYsa2CFRHks5obnWygaJpZM4CFqv8.gif]

[paglias]

@vkarpov15 I've just incurred in this and it seems that it only applies to Mongoose 4. it worked correctly for us in 3.x

[ghost]

@vkarpov15 This issue exist in the version 4.x, because when i do this query.....return all value, skip and limit don't work, when i use match, return all results but with null in the field population reference.

[vkarpov15]

@jaider2523 please open up a separate issue with code samples please

[micimize]

@vkarpov15 I think this is worth reconsidering for a few reasons:

• it does have a valid use case - this isn't an array that grows without bounds, it's a view into a different collection.
• it results in unexpected behavior unless you know exactly how many items will go in the array - you'll lose your "top comments per post", for instance.
• couldn't we implement this efficiently by using the aggregation framework and $group?

[vkarpov15]

There's definitely a valid use case, and plugins are most welcome, but this use case adds a lot of complexity to a feature (populate) that's already heavy.

[simuty]

Same issue

var path = 'video_list';
  var select = null;
  var match = '';
  var model = "Video";
  var options = {
    limit: 1,
    sort: { video_name: 1 }
  }

  CourseModel.find()
        .populate({path, select, match, model, options})
        .cursor()
        .on('data', function(doc){
          console.log(doc);
        })
        .on('error', function(err){
          console.error(err);;
        })
        .on('close', function(){

        });

open mongoose debug find this

videos.find({ _id: { '$in': [ ObjectId("59269b5d1dabfc0df3f51435"), ObjectId("59269b661dabfc0df3f51437") ] } },
 { limit: 16, sort: { video_name: 1 }, fields: {} })

my limit: 1 become ---> limit: 16

mongoose version 4.9.8
MongoDB version v3.4.0

[vkarpov15]

Expected behavior. If you use find(), you get multiple docs back, and use populate() mongoose needs to crank up the limit because otherwise the limit would mean you'd get back 15 documents with no video.

[mozi]

but it does crank up the limit to the point that it throws an exception

limit value must be non-negative

here is the options shown in the debug when streaming a large set of data whilst populating on the fly
{ limit: 9223372036854776000, fields: {} }

commenting out the following line in lib\model.js (line 3039) solves the issue

mod.options.options.limit = mod.options.options.limit * ids.length;

mongoose version 4.9.10

I think this issue needs to be reopened.

[vkarpov15]

Ouch yeah that's a bad problem. Can you open up a separate issue with code samples please?

[mozi]

raised 5468

[vkarpov15]

Confirmed, here's a repro:

const mongoose = require('mongoose');
const co = require('co');
mongoose.Promise = global.Promise;
const GITHUB_ISSUE = `gh-5468`;

mongoose.set('debug', true);

exec()
  .then(() => {
    console.log('successfully ran program');
    process.exit(0);
  })
  .catch(error => {
    console.error(`Error: ${ error }\n${ error.stack }`);
  });


function exec() {
  return co(function*() {
    mongoose.connect(`mongodb://localhost:27017/${ GITHUB_ISSUE }`, { useMongoClient: true })

    yield mongoose.connection.dropDatabase();

    const refSchema = new mongoose.Schema({
      _id: Number,
      name: String
    });
    const Ref = mongoose.model('Ref', refSchema);

    const testSchema = new mongoose.Schema({
      _id: Number,
      prevnxt: { type: [Number], ref: 'Ref' }
    });
    const Test = mongoose.model('Test', testSchema);

    for (let i = 0; i < 1000; ++i) {
      yield Ref.create([{ _id: 3 * i }, { _id: 3 * i + 1 }, { _id: 3 * i + 2 }]);
      yield Test.create({ _id: i, prevnxt: [3 * i, 3 * i + 1, 3 * i + 2] });
    }

    const cursor = Test.find().populate({ path: 'prevnxt', options: { limit: 2 } }).cursor();
    yield new Promise((resolve, reject) => {
      cursor.on('data', doc => console.log(doc));
      cursor.on('error', err => reject(err));
      cursor.on('end', () => resolve());
    });
    console.log('done');
    process.exit(0);
  });

[zargold]

Related to this issue:

I have a nested set of items (versions) and I'd like to have a limit of 30 versions for each item in the array...

[
document1 (versions: [long array of 500+ versions]),
document2 (versions: [long array of 500+ versions]),
document3 (versions: [long array of 500+ versions]),
...
]

Using populate I want to make sure I get the top 20 of each of the documents,
but it simply searches for $in( document1, document2, document3...) and then sets the limit to 20 * 3... isn't it possible that it will be all 60 from document1 or is there some deeper wisdom used.

Here's what the query says:

Mongoose: docversions.find({
 _id: { '$in': [ 
ObjectId("5c07d1e90feb71c693934ddc"), ObjectId("5c07d1e90feb71c693934d72") 

] } }, { limit: 20, sort: { createdDate: -1 }, projection: {} })

[vkarpov15]

@zargold it is possible, but unfortunately we don't have a good workaround with populate() yet. You'll have to use findOne() for each doc, or populate manually.

[furquankhan]

I think this issue is related to what I'm seeing.

Room
.find()
.populate({
  path: 'messages',
  select: 'message createdAt',
  options: {
    sort: { createdAt: -1 },
    limit: 1
  }
})
.exec(cb);

// Room results
[{ 
  _id: 53c04a605574680000405db2,
  messages: []
 },
{ 
  _id: 53c052dad8d226000045366f,
  messages: []
},
{
  _id: 53c05599d8d2260000453673,
  messages:
   [ { _id: 53c055b3d8d2260000453679,
       createdAt: Fri Jul 11 2014 15:22:59 GMT-0600 (MDT),
       message: 'whats up' },
     { _id: 53c055b2d8d2260000453678,
       createdAt: Fri Jul 11 2014 15:22:58 GMT-0600 (MDT),
       message: 'hello' },
     { _id: 53c055b1d8d2260000453677,
       createdAt: Fri Jul 11 2014 15:22:57 GMT-0600 (MDT),
       message: 'hey' } ]
 }]

When I try to populate a document with limit: 1, the last document in the array has seems to have set its limit equal to the number of returned Room documents and the rest of the documents' message arrays are empty.

Mongoose: 3.8.12
Mongodb: 2.6.3

Did you find the workaround for it?

[AbdelrahmanHafez]

@furquankhan

Yes, there's a solution for that now, instead of limit use perDocumentLimit.