Mongoose: Document duplication even using unique index

[eduardonunesp]

This problem happened with my app, even with unique index at email, the mongoose permits that document complete the save action, for the same e-mail.

var UserSchema = mongoose.Schema({
  username: { type: String, required: true},
  email: {type: String, required: true, index: { unique: true }},
  password: { type: String, required: true },
});

module.exports = mongoose.model('User', UserSchema)

var user = new UserModel()
user.username = req.body.username;
user.email = req.body.email;
user.password = req.body.password;

user.save(function(err, result) {
    if (err) {
        var e = "";
        if (err.code == 11000) {
            e = "User already exists";
            req.logger.info('Err: %s', err);                    
        } else {
            e = err;
        }

        res.status(400).send(e);
    }   else {
        req.logger.info('Database sign-in %s', req.body.username);
        res.status(201).send('User created');
    }
});

• Mongo 2.6.3
• Mongoose 3.8.14

[Quentin01]

Can you check that your indexes are created in mongo ?

This may be linked to https://github.com/LearnBoost/mongoose/issues/2184

[mattcasey]

It looks like the schema is set up wrong... you can have index: true or unique: true (and both should create an index), but I've never seen index: { unique: true }.

[vkarpov15]

Ditto @mattcasey 's comments. Also, double check in the mongo shell if the index is there or not.

[codyhanson]

I noticed this too. Here is my schema definition.

var ApplicationSchema = new mongoose.Schema({
    _id: {type: String, required: true, index: true, unique: true},
    name: {type: String, required: true, index: true, unique: true},
    description: {type: String, default: ''},
    status: {type: String, required: true, index: true, default: 'enabled'},
    created_at: {type: Date},
    updated_at: {type: Date},
    deleted: {type: Boolean, index: true, default: false},
    deleted_at: {type: Date}
});

Here are the indexes for that collection.

> db.applications.getIndexes()
[
    {
        "v" : 1,
        "key" : {
            "_id" : 1
        },
        "name" : "_id_",
        "ns" : "mydb.applications"
    }
]

MongoDB v2.6.3, and Mongoose v3.8.14, Node v0.10.30, Mac OSX 10.9.4.

Works fine if I downgrade to v3.8.13.

[vkarpov15]

Hi @codyhanson, your issue is the options on your _id index don't match the index that's there by default. Try this:

  var ApplicationSchema = new mongoose.Schema({
    _id: {type: String, required: true},
    name: {type: String, required: true, index: true, unique: true},
    description: {type: String, default: ''},
    status: {type: String, required: true, index: true, default: 'enabled'},
    created_at: {type: Date},
    updated_at: {type: Date},
    deleted: {type: Boolean, index: true, default: false},
    deleted_at: {type: Date}
  });

MongoDB creates a non-unique index on _id by default whenever a collection is created. When you explicitly try to create a unique index on _id, there's a naming collision. Model emits an "index" event when its done building indexes (code) that contains any error that happened when building indexes, so if you're going to use mongoose for building indexes, I recommend you listen to that event and handle errors in a way that's appropriate for your application.

[callaars]

I got this problem as well:

var schema = new mongoose.Schema({
      username: {
        type: String,
        required: true,
        unique: true
      }
};

But no index is created, only the default one:

{
    "0" : {
        "v" : 1,
        "key" : {
            "_id" : 1
        },
        "name" : "_id_",
        "ns" : "test.users"
    }
}

Mongoose: 3.9.14, Mongo 2.6.3

Note: It seems to work on Mongo 2.6.4 so it might've been a bug with them.

[vkarpov15]

@bcallaars see my comment, I'm pretty sure it's the exact same issue. You might have to drop the _id index first before creating this new one.

[Santinell]

I have problem when trying use this sheme

var User = new mongoose.Schema({
_id: Number,
sid: {type: String, required: true, index: true, unique: true}
});

sid index is not created.

[vkarpov15]

Please check User.on('index', function(err) {}) event and see if any errors are being emitted.

[Santinell]

Sorry, I am forgot about plugin, what add index on _id

MongoError: Index with name: _id_ already exists with different options]
name: 'MongoError',
connectionId: 2134,
err: 'Index with name: _id_ already exists with different options',
code: 85

So, that means what no need to set unique index on _id, because its already by default has unique index?

[vkarpov15]

Yep there's no need to set an index on _id, mongodb server does that by default.

[codethejason]

@vkarpov15 Do you have more information on the on index handler? There doesn't seem to be much info in the documentation I have this:

coin = new Coin();
coin.save(function(err, coin) {
        if(err) {
          console.log(err);
          return res.status(500).send(err);
        }
        coin.on('index', function(err) {console.log(err);})
        return res.status(200).send({date: Date.now(), coin: coin, successful: true});
      });
    });

var coinSchema = new mongoose.Schema({
  name: { type: String, index: true, unique: true },
  address: {type: String, default: ""}
});

mongoose.model('Coin', coinSchema);

The code works if duplication is allowed, but mongodb throws an error when there's duplication, but I only see the error on the mongoDB log on my server and not in my application (background addExistingToIndex exception E11000 duplicate key error index: myDB.coins.$name_1 dup key: { : "CoinName" }). How come the on handler isn't picking up the error?

[vkarpov15]

@codethejason can you clarify the code you're using to listen to the 'index' event and your mongoose version? The below code works fine for me:

'use strict';

var assert = require('assert');
var mongoose = require('mongoose');
var Schema = mongoose.Schema;

mongoose.connect('mongodb://localhost/gh4410');
mongoose.set('debug', true);

var _schema = new Schema({ a: Number });
var _Model = mongoose.model('Test2', _schema, 'tests');

_Model.create([{ a: 1 }, { a: 1 }], function(error) {
  assert.ifError(error);
  var schema = new Schema({ a: { type: Number, unique: true } });
  var Model = mongoose.model('Test', schema, 'tests');
  Model.on('index', function(error) {
    console.log('Got error', error);
    process.exit(0);
  });
});

$ node gh-4410.js 
Mongoose: tests.insert({ a: 1, _id: ObjectId("57b33591cdf18c0a137ccf45"), __v: 0 })
Mongoose: tests.insert({ a: 1, _id: ObjectId("57b33591cdf18c0a137ccf46"), __v: 0 })
Mongoose: tests.ensureIndex({ a: 1 }, { unique: true, background: true })
Got error { MongoError: E11000 duplicate key error collection: gh4410.tests index: a_1 dup key: { : 1 }
    at Function.MongoError.create (/home/val/Workspace/10gen/mongoose/node_modules/mongodb-core/lib/error.js:31:11)
    at commandCallback (/home/val/Workspace/10gen/mongoose/node_modules/mongodb-core/lib/topologies/server.js:1187:66)
    at Callbacks.emit (/home/val/Workspace/10gen/mongoose/node_modules/mongodb-core/lib/topologies/server.js:119:3)
    at .messageHandler (/home/val/Workspace/10gen/mongoose/node_modules/mongodb-core/lib/topologies/server.js:358:23)
    at Socket.<anonymous> (/home/val/Workspace/10gen/mongoose/node_modules/mongodb-core/lib/connection/connection.js:292:22)
    at emitOne (events.js:96:13)
    at Socket.emit (events.js:188:7)
    at readableAddChunk (_stream_readable.js:172:18)
    at Socket.Readable.push (_stream_readable.js:130:10)
    at TCP.onread (net.js:542:20)
  name: 'MongoError',
  message: 'E11000 duplicate key error collection: gh4410.tests index: a_1 dup key: { : 1 }',
  ok: 0,
  errmsg: 'E11000 duplicate key error collection: gh4410.tests index: a_1 dup key: { : 1 }',
  code: 11000 }

[codethejason]

@vkarpov15 I'm using the latest version (4.5.9). I'm only listening to the 'index' event with coin.on('index', function(err) {console.log(err);}) and nothing else. I'm expecting it to print something but it doesn't print anything to the console.

An extended version of my code is here. Models are in models.js.

I can also confirm your code works. I also tested it with .save() and it also works with that.

[vkarpov15]

So first of all you're listening for the index event on a document, not the model itself. Secondly, you probably want to listen to the index event outside of a route handler, because 1) you probably have a memory leak there, and 2) the index event will have fired before that route handler executes

There's an easier way to do what you're trying to do, just set the emitIndexErrors option on your schema

[devarsh]

@vkarpov15
I'm facing the same issue, despite specifying unique index on username column duplicate records are allowed to be added, uniqueness is not enforced nor the index creation listener raises any error

Machine Info
Machine : Mac
Os : OS X EL Capitan
mongodb Version : 3.2.6
mongoose version : ^4.5.9

var userSchema = new mongoose.Schema({
    username : { type: String, required: true, index: true, unique: true },
    password : { type: String, required: true}
})
userSchema.pre('save',function(cb) {
    var user = this
    bcrypt.genSalt(5, function(err,salt) {
        if(err) return cb(err)
        bcrypt.hash(user.password, salt, null, function (err, hash) {
            if(err) return cb(err)
            user.password = hash
            cb()
        })
    })
})
mongoose.model('Users',userSchema)

var Users = require('../models/user.js')

exports.postUser = function (req,res) {
    var user = new Users({
        username : req.body.username,
        password : req.body.password
    })
    user.save(function(err) {
        if(err) return res.send(err)
        user.on('index',function(err) { 
            console.log(err)
        })
        res.json({message:`New user added`})
    })
}

exports.getUsers = (req,res) => {
    Users.find(function(err, users) {
    if (err)
      res.send(err)
    res.json(users)
  })
}

> db.users.getIndexes()
[
        {
            "v" : 1,
            "key" : {
                "_id" : 1
            },
            "name" : "_id_",
            "ns" : "beerlocker.users"
        }
]
> db.users.find()
{ "_id" : ObjectId("57b85aaf188b09ce2442aacb"), "username" : "Devarsh", "password" : "$2a$05$uj2qNtky3BenHtVxWQh.p.tXWqU.ucasf11mg/Wg7T6PqQmbtNBNy", "__v" : 0 }
> db.users.find()
{ "_id" : ObjectId("57b85aaf188b09ce2442aacb"), "username" : "Devarsh", "password" : "$2a$05$uj2qNtky3BenHtVxWQh.p.tXWqU.ucasf11mg/Wg7T6PqQmbtNBNy", "__v" : 0 }
{ "_id" : ObjectId("57b85b1a188b09ce2442aacc"), "username" : "Devarsh", "password" : "$2a$05$UxfjD23taQY/z9cmOxdq1eNd1i47wsgiKtr4rvnSJebAtbwQnHCKy", "__v" : 0 }
{ "_id" : ObjectId("57b85b1b188b09ce2442aacd"), "username" : "Devarsh", "password" : "$2a$05$nqgwxd28nT5isNF5jWe8Y.qOzFYu1Wvi0tdoIx8itv4eBSf6gFTEC", "__v" : 0 }
{ "_id" : ObjectId("57b85b1c188b09ce2442aace"), "username" : "Devarsh", "password" : "$2a$05$pFWQcGIVWMQhj8eCw2G17eYqcZKdc3bIaia8UdRfRWZjBbQ/CRKJe", "__v" : 0 }

[vkarpov15]

exports.postUser = function (req,res) {
    var user = new Users({
        username : req.body.username,
        password : req.body.password
    })
    user.save(function(err) {
        if(err) return res.send(err)
        user.on('index',function(err) { // <--- huh?
            console.log(err)
        })
        res.json({message:`New user added`})
    })
}

index event gets fired on the model, so you should do:

var Users = require('../models/user.js')

Users.on('index', error => console.error(error))

exports.postUser = function (req,res) { // ...
})

To listen for errors.

[cwysong85]

I was able to fix the issue.

Before the fix, I identified the issue by typing db.eidds.getIndexes() (eidds is my DB name, make sure to put your db name) into MongoDB shell:

> db.eidds.getIndexes()
[
        {
            "v" : 1,
            "key" : {
                "_id" : 1
            },
            "name" : "_id_",
            "ns" : "eidds.eidds"
        },
        {
            "v" : 1,
            "key" : {
                "incidentTrackingId" : 1
            },
            "name" : "incidentTrackingId_1",
            "ns" : "eidds.eidds",
            "background" : true
        },
        {
            "v" : 1,
            "key" : {
                "callId" : 1
            },
            "name" : "callId_1",
            "ns" : "eidds.eidds",
            "background" : true
        }
]

Notice how you don't see unique: true for the keys my program made in the above output. To fix this simply drop the indexes from MongoDB:

> db.eidds.dropIndex('callId_1')
{ "nIndexesWas" : 3, "ok" : 1 }
> db.eidds.dropIndex('incidentTrackingId_1')
{ "nIndexesWas" : 2, "ok" : 1 }
>

Now make sure you have the Schema Types setup correctly for indexing and uniqueness:

var eiddSchema = new mongoose.Schema({
    // Notice the "index: true" and "unique: true"
    incidentTrackingId : { type: String, required: true, index: true, unique: true }
});

Now re-run your application and re-check the indexes in MongoDB:

> db.eidds.getIndexes()
[
        {
            "v" : 1,
            "key" : {
                "_id" : 1
            },
            "name" : "_id_",
            "ns" : "eidds.eidds"
        },
        {
            "v" : 1,
            "unique" : true,
            "key" : {
                "incidentTrackingId" : 1
            },
            "name" : "incidentTrackingId_1",
            "ns" : "eidds.eidds",
            "background" : true
        },
        {
            "v" : 1,
            "unique" : true,
            "key" : {
                "callId" : 1
            },
            "name" : "callId_1",
            "ns" : "eidds.eidds",
            "background" : true
        }
]

Now you see unique: true in the above output.

Now when you input the same value for the indexed key it will generate a MongoError: E11000 duplicate key error collection.

[cwysong85]

Just an FYI, an error is outputed from the Model.save method rather than the Model.on('index') method. So you do not have to listen to the Model.on('index') method for duplicate keys error.

[vkarpov15]

@cwysong85 is correct, the index event only fires once for each model - when mongoose is done calling ensureIndex() for every index on that model or an error occurred

[techguysid]

Any updates on this ? I'm still facing the same error, even after adding unique and index as true in schema.

[vkarpov15]

@techguysid try enabling the emitIndex errors option: http://mongoosejs.com/docs/guide.html#emitIndexErrors and see if you get any errors.

[yattias]

This typically happens in test environments where you drop/recreate collections rapidly. You can create a file named __before.js (suggestion, not required) and add the following to it to ensure your indexes are created correctly before your tests run.

// __before.js
import mongoose from 'mongoose';

before(async function(){
  const ensureIndexOnCollection = (name) => {
    return new Promise(function(resolve, reject){
      mongoose.connection.collections[name].ensureIndex({ "key": 1 }, { "unique": true }, function(){
        resolve();
      });
    });
  }

  const collectionNames = Object.keys(mongoose.connection.collections);
  let currentCollection;

  for (let i = 0; i < collectionNames.length; i++) {
    currentCollection = collectionNames[i];
    await ensureIndexOnCollection(currentCollection);
  }
});

Alternatively, you can quickly test to see if this is your issue by typing:

mongoose.connection.collections[PUT_COLLECTION_NAME_HERE].ensureIndex({ "key": 1 }, { "unique": true }, function(){

      });

[jsMichot]

I'm trying to prevent duplicate events from going into the events array. This is NOT working:

const feedbackSchema = mongoose.Schema({
  pros: String,
  cons: String,
  journal: String,
});

const eventSchema = mongoose.Schema({
  title: String,
  category: String,
  tag: String,
  description: { type: String, index: true, unique: true },
  feedback: [feedbackSchema],
  timeStart: Date,
  timeEnd: Date,
  isComplete: Boolean,
});

const userSchema = mongoose.Schema({
  googleId: { type: String, required: true, unique: true },
  email: { type: String, required: true, unique: true },
  name: String,
  firstName: String,
  events: [eventSchema],
}, {
  usePushEach: true,
});

[vkarpov15]

@jsMichot have you read the FAQ on this? http://mongoosejs.com/docs/faq.html#unique-doesnt-work

[jsMichot]

thank you, @vkarpov15 i wrote a function that solves the issue:

const addEvent = async (id, event, callback) => {
  await db.User.findOne({ googleId: id }, async (err, user) => {
    const existingEvent = user.events.reduce((doesExist, e) => {
      if (e.title === event.title && e.description === event.description) {
        doesExist = true;
      }
      return doesExist;
    }, false);
    if (!existingEvent) {
      const newEvent = new db.IEvent({
        title: event.title || '',
        category: event.category || '',
        date: event.date || 'you will know when the time is right',
        description: event.description || '',
        isComplete: event.isComplete || false,
      });
      user.events.push(newEvent);
      await user.save();
    }
    callback(user);
  });
};

[sjcodebook]

Use dropDups to ensure dropping duplicate records in your schemas and also change index: { unique: true } to unique: true like=>

var UserSchema = mongoose.Schema({
  username: { type: String, required: true},
  email: {type: String, required: true, unique: true, dropDups: true},
  password: { type: String, required: true },
});

module.exports = mongoose.model('User', UserSchema)

and restart mongoDB