Mongoose: Using $pull on nested array causes ValidationError

[krassx]

Do you want to request a feature or report a bug?
Bug

What is the current behavior?
When using $pull to remove item by ID from nested array, I get ValidationError stating that the fields, holding that array is required. I've checked various cases and each time ensured that nested array always have more than 1 item in it.

If the current behavior is a bug, please provide the steps to reproduce.

const RecordingSchema = new Schema({
    created_on: Date,
    submitted_on: Date,
    uploaded_on: Date,
    published_on: Date,
    updated_on: {
        type: Date,
        required: true,
        default: Date.now
    },
    srid: String,
    status: String
}, {
    usePushEach: true
});

const ItemSchema = new Schema({
    type: String,
    status: String,
    summary: String,
    description: String,
    created_on: Date,
    submitted_on: Date,
    published_on: Date,
    recordings: {
        type: [RecordingSchema],
        required: true
    }
}, {
    timestamps: {
        createdAt: 'created_on',
        updatedAt: 'updated_on'
    },
    usePushEach: true,
    runSettersOnQuery: true
});

// This is how I try to remove an item from nested array (from 'recordings')
itemsModel.findOneAndUpdate({
    _id: item._id
}, {
    $pull: {
        recordings: {
            _id: recId
        }
    }
}, {
    runValidators: true,
    context: 'query',
    fields: {
        _id: true
    }
}, err => console.log(err));

Finally, I get "Validation failed: recordings: Path `recordings` is required." in my console.

What is the expected behavior?
Item from nested array must be deleted (no ValidationError should be raised).

Please mention your node.js, mongoose and MongoDB version.
Node.JS: 8.9.4
Mongoose: 5.0.14
MongoDB: 3.6

[krassx]

I also tried to use { $in: [recId] }, like described in #6240, but that did not help.

[lineus]

Hi @krassx! The query options are telling mongoose to run the validators in the query context, but your schema doesn't have a custom validator setup. If you remove the query options or write a custom query validator, your query should work as expected. Update Validator docs are here.

contrived example:

#!/usr/bin/env node
'use strict';

const mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/test');
const Schema = mongoose.Schema;

const subSchema = new Schema({
  name: String
});

const schema = new Schema({
  arr: {
    type: [subSchema],
    required: true
  }
});

schema.path('arr').validate(function(val) {
  if (this.getUpdate().$)
})

const Test = mongoose.model('test', schema);

const test = new Test({
  arr: [
    { name: 'one' }, 
    { name: 'two' }, 
    { name: 'three' }
  ]
});

async function run () {
  await mongoose.connection.dropDatabase();
  await test.save();
  const cond = { _id: test._id };
  const update = { $pull: { arr: { _id: test.arr[1]._id } } };
  const opts = { 
    runValidators: true,
    context: 'query'
  };
  let docs = await Test.findByIdAndUpdate(cond, update, opts);
  console.log(docs);
  return mongoose.connection.close();
}

run().catch(console.error);

output with validator query option and no custom query validator

issues: ./6341.js
{ ValidationError: Validation failed: arr: Path `arr` is required.
    at ValidationError.inspect (/Users/lineus/dev/Help/mongoose5/node_modules/mongoose/lib/error/validation.js:56:24)
    at formatValue (util.js:430:38)
    at inspect (util.js:324:10)
    at format (util.js:191:12)
    at Console.warn (console.js:145:21)
    at <anonymous>
    at process._tickCallback (internal/process/next_tick.js:188:7)
  errors:
   { arr:
      { ValidatorError: Path `arr` is required.
    at new ValidatorError (/Users/lineus/dev/Help/mongoose5/node_modules/mongoose/lib/error/validator.js:25:11)
    at validate (/Users/lineus/dev/Help/mongoose5/node_modules/mongoose/lib/schematype.js:805:13)
    at /Users/lineus/dev/Help/mongoose5/node_modules/mongoose/lib/schematype.js:854:11
    at Array.forEach (<anonymous>)
    at DocumentArray.SchemaType.doValidate (/Users/lineus/dev/Help/mongoose5/node_modules/mongoose/lib/schematype.js:814:19)
    at DocumentArray.doValidate (/Users/lineus/dev/Help/mongoose5/node_modules/mongoose/lib/schema/documentarray.js:140:35)
    at /Users/lineus/dev/Help/mongoose5/node_modules/mongoose/lib/services/updateValidators.js:115:22
    at /Users/lineus/dev/Help/mongoose5/node_modules/async/internal/parallel.js:27:9
    at eachOfArrayLike (/Users/lineus/dev/Help/mongoose5/node_modules/async/eachOf.js:57:9)
    at exports.default (/Users/lineus/dev/Help/mongoose5/node_modules/async/eachOf.js:9:5)
    at _parallel (/Users/lineus/dev/Help/mongoose5/node_modules/async/internal/parallel.js:26:5)
    at parallelLimit (/Users/lineus/dev/Help/mongoose5/node_modules/async/parallel.js:85:26)
    at /Users/lineus/dev/Help/mongoose5/node_modules/mongoose/lib/services/updateValidators.js:176:5
    at model.Query.Query._findAndModify (/Users/lineus/dev/Help/mongoose5/node_modules/mongoose/lib/query.js:2464:7)
    at model.Query.Query._findOneAndUpdate (/Users/lineus/dev/Help/mongoose5/node_modules/mongoose/lib/query.js:2163:8)
    at process.nextTick (/Users/lineus/dev/Help/mongoose5/node_modules/kareem/index.js:311:33)
        message: 'Path `arr` is required.',
        name: 'ValidatorError',
        properties: [Object],
        kind: 'required',
        path: 'arr',
        value: [Object],
        reason: undefined,
        '$isValidatorError': true } },
  _message: 'Validation failed',
  name: 'ValidationError' }
^C

output after replacing current query opts with { new: true }

issues: ./6341.js
{ arr:
   [ { _id: 5ad0a64f2b7d0531a166c7ad, name: 'one' },
     { _id: 5ad0a64f2b7d0531a166c7ab, name: 'three' } ],
  _id: 5ad0a64f2b7d0531a166c7aa,
  __v: 0 }
issues:

[vkarpov15]

@lineus I think the issue here has nothing to do with custom validators, it's about $pull and required: true with update validators bugging out. I can see this being an issue, just need to see if I can repro it first.

[lineus]

I absolutely did not account for the fact that required triggers a validator. Add that to the fact that my code example must have changed between when I ran it and when I copied/pasted it here and you get a great big lineus shaped fail. Apologies @krassx.

here's a repro script that shows it failing with $pull, but working with $push:

6341.js

#!/usr/bin/env node
'use strict';

const mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/test');
const Schema = mongoose.Schema;

const subSchema = new Schema({
  name: String
});

const schema = new Schema({
  arr: {
    type: [subSchema],
    required: true
  }
});

const Test = mongoose.model('test', schema);

const test = new Test({
  arr: [
    { name: 'one' },
    { name: 'two' },
    { name: 'three' }
  ]
});

async function run () {
  await mongoose.connection.dropDatabase();
  await test.save();
  const cond = { _id: test._id };
  const pullUpdate = { $pull: { arr: { _id: test.arr[1]._id } } };
  const pushUpdate = { $push: { arr: { name: 'four' } } };
  const opts = {
    runValidators: true,
    context: 'query',
    new: true
  };

  await Test.findByIdAndUpdate(cond, pullUpdate, opts)
    .then(console.log)
    .catch((e) => { console.error(`pull: ${e.message}`); });

  await Test.findByIdAndUpdate(cond, pushUpdate, opts)
    .then(console.log)
    .catch((e) => { console.error(`push: ${e.message}`); });

  return mongoose.connection.close();
}

run();

output:

issues: ./6341.js
pull: Validation failed: arr: Path `arr` is required.
{ arr:
   [ { _id: 5ada45b008e4246407a8f8d0, name: 'one' },
     { _id: 5ada45b008e4246407a8f8cf, name: 'two' },
     { _id: 5ada45b008e4246407a8f8ce, name: 'three' },
     { _id: 5ada45b108e4246407a8f8d1, name: 'four' } ],
  _id: 5ada45b008e4246407a8f8cd,
  __v: 0 }
issues:

[krassx]

@lineus no worries. If any other info is required, just let me know.

[vkarpov15]

@lineus definitely a bit of a gotcha when working with mongoose internals. The fix ended up being pretty easy, I love it when fixing a bug involves just removing some dubiously useful code :tada: