Mongoose: Schema-level compound indexes not being saved to Mongo

[docksteaderluke]

Hi,

I'm trying to set a unique compound index on two fields in a schema. Here's a shortened version of my Schema file:

'use strict'

// Require
var mongoose = require('mongoose')
  , uniqueValidator = require('mongoose-unique-validator')
  , validate = require('mongoose-validator')

// Variables
var Schema = mongoose.Schema
  , ClientSchema = new Schema(
    {
      name : {
        type     : String,
        required : true,
        trim     : true,
        validate : [
          validate({
            validator : 'isLength',
            arguments : [ 1, 100 ],
            message   : 'Name must be 1-100 characters long.'
          })
        ]
      },
      organization : {
        type     : Schema.Types.ObjectId,
        ref      : 'Organization',
        required : true
      }
    }
  )

// Indexes
ClientSchema.index({ "name" : 1, "organization" : 1 }, { unique : true })

// Plugins
ClientSchema.plugin(uniqueValidator, {
  message : 'Name must be unique.'
})

module.exports = mongoose.model('Client', ClientSchema)

For some reason, even after completely dropping the collection, the compound index doesn't get added to mongo at all. However, if I create the index in the Mongo shell it works:

db.clients.ensureIndex({"name":1,"organization":1}, {unique:true})

I've been scouring the Internet and reading through the documentation and can't seem to figure out why this isn't working. Any ideas?

I'm using:

• Mongo v3.0.6
• Mongoose 4.1.7
• Node 0.12.7

[vkarpov15]

Try listening to the model's index event, Model.on('index', function(error) { /* log error */ });. There might be an error with your index build somewhere.

[docksteaderluke]

Thanks @vkarpov15,

I've added the index event listener to the end of the schema file like so:

var Client = module.exports = mongoose.model('Client', ClientSchema)
Client.on('index', function (error) {
    console.log("ON INDEX")
    if (error) console.log(error)
  })

The listener is working ("ON INDEX" gets printed to the console) but error is empty.

I've also added a generic error listener on the mongo connection like so:

...
mongoose.connect(process.env.MONGO_URL)
mongoose.connection.on('error', function (err) {
  console.log('MONGOOSE ERROR')
  console.log(err)
})

...and there aren't any errors being printed out.

Regardless of what I do, db.getCollection('clients').getIndexes() always seems to return:

{
    "0" : {
        "v" : 1,
        "key" : {
            "_id" : 1
        },
        "name" : "_id_",
        "ns" : "vpr.clients"
    }
}

...unless I create the compound index manually via the Mongo shell as I mentioned above.

Any other thoughts?

[docksteaderluke]

UPDATE:

I've changed the code at the end of the Schema file to the following (note the ensureIndex call):

var Client = module.exports = mongoose.model('Client', ClientSchema)

Client.ensureIndexes(function (err) {
  console.log('ENSURE INDEX')
  if (err) console.log(err)
})

Client.on('index', function (err) {
  console.log('ON INDEX')
  if (err) console.log(err)
})

...and now the indexes appear to be created properly! My understanding, from the documentation, is that ensureIndexes is called automatically unless autoIndex is false on either the schema or connection. I haven't set it to false in either of those places. Any thoughts why it's not auto-indexing?

[vkarpov15]

Yeah, mongoose should automatically call ensureIndex() for you unless you explicitly tell it not to. The below standalone script:

'use strict';

var mongoose = require('mongoose');
var Schema = mongoose.Schema;

var uniqueValidator = require('mongoose-unique-validator');

mongoose.set('debug', true);
mongoose.connect('mongodb://localhost:27017/gh3396');

// Variables
var ClientSchema = new Schema(
    {
      name : {
        type     : String,
        required : true,
        trim     : true
      },
      organization : {
        type     : Schema.Types.ObjectId,
        ref      : 'Organization',
        required : true
      }
    }
  )

// Indexes
ClientSchema.index({ "name" : 1, "organization" : 1 }, { unique : true })

// Plugins
ClientSchema.plugin(uniqueValidator, {
  message : 'Name must be unique.'
})

mongoose.model('Client', ClientSchema);

seems to output the correct result:

$ node gh-3393.js 
Mongoose: clients.ensureIndex({ name: 1, organization: 1 }) { unique: true, background: true, safe: undefined }  
^C

ensureIndex gets called normally. Can you try manipulating the above script and see if you can make it not call ensureIndex?

Also, just so you're aware, mongoose-unique-validator doesn't actually ensure uniqueness, it's only a best-effort approach that has a race condition.

[docksteaderluke]

Thanks @vkarpov15. I'll give that a shot and report back.

[docksteaderluke]

So, I figured out the issue.

I was running an import script that was dropping the collection then recreating it. ensureIndex wasn't getting called until I restarted the server. I wasn't aware of this. Is there a way to rebuild all of the indexes without having to restart the server?

[docksteaderluke]

Also, thanks for the heads up about mongoose-unique-validator. I was using it to build more user-friendly error messages. Any suggestions on a better way to do this (I need the field name and value parsed out) rather than outputting:
E11000 duplicate key error index: vpr.organizations.$name 1 dup key: { : \"Organization name" }

[vkarpov15]

@docksteaderluke call ensureIndexes() manually. Mongoose doesn't watch for indexes being dropped, so you're on your own there.

Re: the second point, the only way right now is to parse the error message. The best way off the top of my head would to write your own wrapper around save and attach it to schema.methods...

[docksteaderluke]

Thanks @vkarpov15. I ended up catching then parsing the error message with Regex for now. It's not an ideal solution but does the trick for the time being. If you're interested:
var parts = err.message.match(/^E11000.*?:\s*(.+?)\.(.+?)\.[$]?(.+?)[_\s].*?\"(.*?)\"/i)

• parts[1] -> database name
• parts[2] -> collection name
• parts[3] -> index name (I use this one to further parse out the field name)
• parts[4] -> value that violates the constraint

[vkarpov15]

Clever idea, thanks. Would be a good idea to have a plugin for this, re: #3401.

[suyash248]

I faced similar issue, I ended up using following code -

mongoose.connection.db.collection("your_collection").createIndex({name: 1, city: 1},null,function (err, results) { console.log(results); });

I used mongo native driver thorugh mongoose.

[GregoryNEUT]

UPDATE on the 03 january of 2020.

Using mongoose 5.7.0 or 5.8.4 and mongodb 4.2.1, I reproduce the problem. Had to call model.ensureIndexes() to force mongoose to create the indexes.

return new Promise((resolve, reject) => {
   const schema = ...
   const model = mongoose.model('Example', schema);

    // Wait for every index to finish building
    model.on('index', async (error) => {
        if (error) {
           // To debug
           console.error(`<debug> Index creation ERROR ${JSON.stringify(error.message, null, 2)}`);

           reject();

           return;
        }

        resolve();
    });

    // Array containing every indexes I want to create
    indexes.forEach((x) => {
       this.mongodbSchema.index(x);
    });

    // Must call this one in order to force mongoose creating the indexes
    // https://github.com/Automattic/mongoose/issues/3393
    model.ensureIndexes();
});

[vkarpov15]

@GregoryNEUT what is this.mongodbSchema in your code? Please open a new issue and follow the issue template