Mongoose: text index doesn't get created

[jaruszewskiG]

Hi, I'm trying to do full text search with mongoose. It doesn't seem to work though. Here's how I'm adding text indexes to one of my schemas:

AdvisorSchema.index({
  name: 'text',
  email: 'text',
  phone: 'text'
});

And then I'm using it like that:

    Advisor.find({
      $text: {
        $search: filterWord,
        $caseSensitive: false,
      }
    });

But I'mstill getting error: text index required for $text query.

Everything works fine though when I'm adding text indexes to the mongo database directly by db.advisors.createIndex({ name: 'text', email: 'text', phone: 'text' }), but I need it to be done by mongoose for my test database.

Am I doing something wrong?

[lineus]

@jaruszewskiG it sounds like your query is running before the index is completely created on the collection.

You can add await Advisor.init() before you start querying your data and that should resolve the issue ( you can uncomment the init call in the failing example below to fix the script ).

For more explicit control over your index, you can disable mongoose's autoIndexing and await the index creation manually.

By adding the schema option { autoIndex: false } and calling Model.createIndexes() we can ensure that mongoose isn't building the index automatically and that the index exists before running our query. ( this is demonstrated in the "working" example below )

Here is a failing example followed by a working one.

Failing Example

7058.js

#!/usr/bin/env node
'use strict';

const numberOfDocs = Number(process.argv[2]);
const assert = require('assert');
const mongoose = require('mongoose');
mongoose.set('useCreateIndex', true);
const { Schema, connection} = mongoose;
const GH = 'gh7058';
const URI = `mongodb://localhost:27017/${GH}`;
const OPTS = { family: 4, useNewUrlParser: true };

const schema = new Schema({
  name: String,
  email: String,
  phone: String
});

const index = { name: 'text', email: 'text', phone: 'text' };
schema.index(index);

const Test = mongoose.model('test', schema);

const tests = [];
for (let n = 0; n < numberOfDocs; n++) {
  let suffix = (n % 2 === 0) ? '0':'1';
  tests.push({
    name: `test${n}`,
    email: `${n}@xyz.com`,
    phone: `123-456-789${suffix}`
  });
}

async function run() {
  // connect and reset db
  await mongoose.connect(URI, OPTS);
  await connection.dropDatabase();

  // track time for fun
  let start = Date.now();
  let end;
  let delta;
  function updateTimes() {
    end = Date.now();
    delta = (end - start) / 1000;
    start = Date.now();
  }

  // simulate existing collection
  await Test.collection.insertMany(tests).then(updateTimes);
  const count = await Test.countDocuments();
  console.log(`created ${count} documents: ${delta}`);

  // uncomment the following line to make this script succeed
  // await Test.init();

  //find docs
  const query = { $text: { $search: '7890' } };
  const docs = await Test.find(query);

  assert.strictEqual(docs.length, numberOfDocs/2);
  console.log(`found ${docs.length} matching documents.`);
  await connection.close();
}

run().catch(console.error);

Output:

issues: ./7058.js 10000
created 10000 documents: 1.447
{ MongoError: text index required for $text query
    at queryCallback (/Users/lineus/dev/node/mongoose/node_modules/mongodb-core/lib/cursor.js:248:25)
    at /Users/lineus/dev/node/mongoose/node_modules/mongodb-core/lib/connection/pool.js:532:18
    at process._tickCallback (internal/process/next_tick.js:61:11)
  ok: 0,
  errmsg: 'text index required for $text query',
  code: 27,
  codeName: 'IndexNotFound',
  operationTime:
   Timestamp { _bsontype: 'Timestamp', low_: 10001, high_: 1538217168 },
  '$clusterTime':
   { clusterTime:
      Timestamp { _bsontype: 'Timestamp', low_: 10001, high_: 1538217168 },
     signature: { hash: [Binary], keyId: 0 } },
  name: 'MongoError',
  [Symbol(mongoErrorContextSymbol)]: {} }
^C
issues:

Working Example

7058_2.js

#!/usr/bin/env node
'use strict';

const numberOfDocs = Number(process.argv[2]);
const assert = require('assert');
const mongoose = require('mongoose');
mongoose.set('useCreateIndex', true);
const { Schema, connection} = mongoose;
const GH = 'gh7058';
const URI = `mongodb://localhost:27017/${GH}`;
const OPTS = { family: 4, useNewUrlParser: true };

const schema = new Schema({
  name: String,
  email: String,
  phone: String
}, { autoIndex: false });

const index = { name: 'text', email: 'text', phone: 'text' };
schema.index(index);

const Test = mongoose.model('test', schema);

const tests = [];
for (let n = 0; n < numberOfDocs; n++) {
  let suffix = (n % 2 === 0) ? '0':'1';
  tests.push({
    name: `test${n}`,
    email: `${n}@xyz.com`,
    phone: `123-456-789${suffix}`
  });
}

async function run() {
  // connect and reset db
  await mongoose.connect(URI, OPTS);
  await connection.dropDatabase();

  // track time for fun
  let start = Date.now();
  let end;
  let delta;
  function updateTimes() {
    end = Date.now();
    delta = (end - start) / 1000;
    start = Date.now();
  }

  // simulate existing collection
  await Test.collection.insertMany(tests).then(updateTimes);
  const count = await Test.countDocuments();
  console.log(`created ${count} documents: ${delta}`);

  // be sure that the index actually exists before querying against it.
  await Test.createIndexes().then(updateTimes);
  console.log(`created indexes: ${delta}`);

  //find docs
  const query = { $text: { $search: '7890' } };
  const docs = await Test.find(query);

  assert.strictEqual(docs.length, numberOfDocs/2);
  console.log(`found ${docs.length} matching documents.`);
  await connection.close();
}

run().catch(console.error);

Output:

issues: ./7058_2.js 10000
created 10000 documents: 1.621
created indexes: 2.413
found 5000 matching documents.
issues:

[jaruszewskiG]

Hi, thanks for the response, @lineus !

I did it based on your 7058_2.js example. It works fine, but I'm a bit concerned about running createIndexes funciton everytime I'm querying the database. I'm mainly concerned about performance.

How does it work? Does it check if the index is already created and if it's not it does nothing?

[lineus]

@jaruszewskiG once your collection is stable ( i.e. you are done changing/adding/deleting indexes ), you should remove the createIndexes() call altogether. During development, you don't need to run createIndexes() for every query, just when you want to add a new index and wait before running queries that use it.

After looking at the Native Driver docs and the source code for the Native Driver's implementations of createIndex(es), it appears to me that createIndex(es) does not look for existing indexes the way ensureIndexes() does ( though ensureIndexes() is deprecated ).

[jaruszewskiG]

Thanks for the response, everything's clear now! :)

[prettydev]

How can I see whether the collection is stable or not before running queries?
I want to write as follows.
If (!IsStable) await Test.createIndexes();
Here, how to evaluate 'IsStable'?

[shakesBeardZ]

@creditcoder he means when there is no further changes on you collection, it's done and you are mostly done changing it.

[vkarpov15]

@creditcoder await Test.init(), see Model.init() docs