Mongoose: [Bug] Populating an array containing null values leads to violation of array sequence.

[Gredys]

Do you want to request a feature or report a bug?
Bug

What is the current behavior?
Populating an array containing null values results in deletion of null values and violation of array sequence.

If the current behavior is a bug, please provide the steps to reproduce.

Small example with output:

let mongoose = require('mongoose');
let Schema = mongoose.Schema;

mongoose.Promise = global.Promise;

mongoose.set('debug', true);
mongoose.connect(`mongodb://localhost:27019/test`);

let UsersSchema = new Schema({
    id: Number,
    roles: [{type: Schema.Types.ObjectId, ref: 'roles'}]
}, {versionKey: false});

let Users = mongoose.model('users', UsersSchema);

let RolesSchema = new Schema({
    id: Number,
    name: String
}, {versionKey: false});

let Roles = mongoose.model('roles', RolesSchema);

setTimeout(async() => {
    await Roles.remove({})
               .exec();
    await Users.remove({})
               .exec();

    let role_1 = new Roles({id: 1, name: 'Role 1'});
    await role_1.save();

    let role_2 = new Roles({id: 2, name: 'Role 2'});
    await role_2.save();

    let user = new Users({id: 1, roles: [null, role_1, null, role_2, null, null]})
    await user.save();

    console.log('Before ==>', user);

    await Users.populate(user, [{path: 'roles'}]);

    console.log('After ==>', user);

    console.log('Selection ==>',
        await Users
            .find()
            .populate('roles')
            .exec()
    );
})

Output:
Mongoose without changes (console screenshot - https://i.imgur.com/MBkY5eT.png)

Mongoose: roles.remove({}, {})
Mongoose: users.remove({}, {})
Mongoose: roles.insertOne({ _id: ObjectId("5af5a89373668541b49094bf"), id: 1, name: 'Role 1' })
Mongoose: roles.insertOne({ _id: ObjectId("5af5a89373668541b49094c0"), id: 2, name: 'Role 2' })
Mongoose: users.insertOne({ roles: [ '\u001b[1mnull\u001b[22m', ObjectId("5af5a89373668541b49094bf"), '\u001b[1mnull\u001b[22m', ObjectId("5af5a89373668541b49094c0"), '\u001b[1mnull\u001b[22m', '\u001b[1mnull\u001b[22m' ], _id: ObjectId("5af5a89373668541b49094c1"), id: 1 })
Before ==> { roles: 
   [ null,
     5af5a89373668541b49094bf,
     null,
     5af5a89373668541b49094c0,
     null,
     null ],
  _id: 5af5a89373668541b49094c1,
  id: 1 }
Mongoose: roles.find({ _id: { '$in': [ '\u001b[1mnull\u001b[22m', ObjectId("5af5a89373668541b49094bf"), '\u001b[1mnull\u001b[22m', ObjectId("5af5a89373668541b49094c0"), '\u001b[1mnull\u001b[22m', '\u001b[1mnull\u001b[22m' ] } }, { fields: {} })
After ==> { roles: 
   [ { _id: 5af5a89373668541b49094bf, id: 1, name: 'Role 1' },
     { _id: 5af5a89373668541b49094c0, id: 2, name: 'Role 2' } ],
  _id: 5af5a89373668541b49094c1,
  id: 1 }
Mongoose: users.find({}, { fields: {} })
Mongoose: roles.find({ _id: { '$in': [ '\u001b[1mnull\u001b[22m', ObjectId("5af5a89373668541b49094bf"), '\u001b[1mnull\u001b[22m', ObjectId("5af5a89373668541b49094c0"), '\u001b[1mnull\u001b[22m', '\u001b[1mnull\u001b[22m' ] } }, { fields: {} })
Selection ==> [ { roles: [ [Object], [Object] ],
    _id: 5af5a89373668541b49094c1,
    id: 1 } ]

Mongoose with changes (console screenshot - https://i.imgur.com/j3tUzv7.png)

Mongoose: roles.remove({}, {})
Mongoose: users.remove({}, {})
Mongoose: roles.insert({ _id: ObjectId("5af5a8bab5a66042e4c894b5"), id: 1, name: 'Role 1' })
Mongoose: roles.insert({ _id: ObjectId("5af5a8bab5a66042e4c894b6"), id: 2, name: 'Role 2' })
Mongoose: users.insert({ roles: [ '\u001b[1mnull\u001b[22m', ObjectId("5af5a8bab5a66042e4c894b5"), '\u001b[1mnull\u001b[22m', ObjectId("5af5a8bab5a66042e4c894b6"), '\u001b[1mnull\u001b[22m', '\u001b[1mnull\u001b[22m' ], _id: ObjectId("5af5a8bab5a66042e4c894b7"), id: 1 })
Before ==> { roles: 
   [ null,
     5af5a8bab5a66042e4c894b5,
     null,
     5af5a8bab5a66042e4c894b6,
     null,
     null ],
  _id: 5af5a8bab5a66042e4c894b7,
  id: 1 }
Mongoose: roles.find({ _id: { '$in': [ ObjectId("5af5a8bab5a66042e4c894b5"), ObjectId("5af5a8bab5a66042e4c894b6") ] } }, { fields: {} })
After ==> { roles: 
   [ null,
     { _id: 5af5a8bab5a66042e4c894b5, id: 1, name: 'Role 1' },
     null,
     { _id: 5af5a8bab5a66042e4c894b6, id: 2, name: 'Role 2' },
     null,
     null ],
  _id: 5af5a8bab5a66042e4c894b7,
  id: 1 }
Mongoose: users.find({}, { fields: {} })
Mongoose: roles.find({ _id: { '$in': [ ObjectId("5af5a8bab5a66042e4c894b5"), ObjectId("5af5a8bab5a66042e4c894b6") ] } }, { fields: {} })
Selection ==> [ { roles: [ null, [Object], null, [Object], null, null ],
    _id: 5af5a8bab5a66042e4c894b7,
    id: 1 } ]

What is the expected behavior?
Null/invalid values in the array must not be removed. Array order must be same as before populating.

P.S. Why Mongoose includes null values when querying ObjectID to MongoDB?

Please mention your node.js, mongoose and MongoDB version.
NodeJS: 9.9.0
Mongoose: 5.0.17
MongoDB: 3.6.3

[Gredys]

Dunno if it will broke something else but I managed to 'fix'. Created local copy of Mongoose 5.0.17 for tests.

Changes in class: ./mongoose/lib/model.js (lines in comments)

function populate(model, docs, options, callback) {
    ...
    var ids = utils.array.flatten(mod.ids, flatten);
    // Filter out null values, no real use for null ObjectId documents
    // Line ~3081
    ids = utils.array.unique(ids).filter((id) => id !== null);
    ...
}

function valueFilter(val, assignmentOpts, justOne) {
  if (Array.isArray(val)) {
    // find logic
    var ret = [];
    var numValues = val.length;
    for (var i = 0; i < numValues; ++i) {
      var subdoc = val[i];
      // Add real value to returning array
      // Line ~3674
      if (isDoc(subdoc))
        maybeRemoveId(subdoc, assignmentOpts);
      ret.push(subdoc);
      if (assignmentOpts.originalLimit &&
          ret.length >= assignmentOpts.originalLimit) {
        break;
      }
    }
    ...
}

Tested on my local project. No errors or wrong data, except that null are converted to undefined. And no problem with wrong array order.

[vkarpov15]

Thanks for reporting, will investigate ASAP

[Gredys]

Addition to code above. Fixes two problems:

1. Null values converted to undefined.
2. Leading null values removed from array.

Additional changes in class: ./mongoose/lib/model.js (line ~3377-3379)

function assignRawDocsToIdStructure(...) {
  ...
  rawIds.length = 0;
  if (newOrder.length) {
    // reassign the documents based on corrected order

    // forEach skips over sparse entries in arrays so we
    // can safely use this to our advantage dealing with sorted
    // result sets too.
    newOrder.forEach(function(doc, i) {
      // Add real value to returning array
      //if (!doc) {
      //  return;
      //}
      rawIds[i] = doc;
    });
  ...
}

First message is updated with complete example and output from both version - modified and vanilla.

[vkarpov15]

Unfortunately this change would be backwards breaking because of issues like #3859, etc. We added an option retainNullValues to opt into keeping the null array entries for now.

yield BlogPost.
  findById(post._id).
  populate({ path: 'fans', options: { retainNullValues: true } });

[Gredys]

Thank you. Everything is working now.