Mongoose: Sub document refs undefined, when not populating in 4.0.2

[rodriguise]

I am having a strange issue

var UserSchema = new Schema({
    application: { type: Schema.Types.ObjectId, ref: 'Application', required: true, index: true },
    name: { type: String, required: true, index: true },
  });

var User = mongoose.model('user', UserSchema)

User.findOne{ {_id: <id> }).exec(function(err, user) {
    console.log(user);
   //  { _id: <id>, name: <name> }  <-- application is missing even when explicitly set in db console
});

Any properties that are refs never get added to the document, if I remove the 'ref' statement, the objects load just fine. I am NOT trying to populate the sub-document in this instance, I just want the sub-document id.

[rodriguise]

This seems to be fixed in the 4.0.3-pre, but knowing a root cause would be nice if possible.

[rodriguise]

This cropped up again. Seems only schemas that have a 'ref' defined won't return anything after a fineOneAndUpdate. If I remove the 'ref' I get the id, if add the ref I get undefined.

[rodriguise]

I have also found this issue to happen when streaming.

 var stream = db.User
            .find(query)
            .stream();

 stream.on('data', function(user) {
     // user.application is null or undefined
}

[rodriguise]

Issue exists in 4.0.2 as well. Any model property that has a 'ref' defined, will return undefined for that property unless populate is called.

[swordfish444]

I'm also dead in the water because of this. Is there a temporary solution for this? mongoose 4 is starting to give me a headache.

I'm experiencing this when streaming as @rodriguise pointed out above.

[rodriguise]

I removed the 'ref' from the definition:

var UserSchema = new Schema({
    application: { type: Schema.Types.ObjectId, required: true, index: true },
    name: { type: String, required: true, index: true },
  });

This will return the ObjectId always and when I replaced all my populate calls with:

user.find().populate({path: 'application', model: 'Application'}).exec()

It's annoying, but this works fine.

[swordfish444]

Oh boy, this will be fun. I have 100+ schemas and the majority are referencing 2 or more additional.

[swordfish444]

Do you have an solutions involving modifying mongoose files?

[vkarpov15]

Hmm @rodriguise I can't seem to reproduce this, the below script works as expected for me on both master and 4.0.2:

var mongoose = require('mongoose');
mongoose.set('debug', true);
var util = require('util');

mongoose.connect('mongodb://localhost:27017/gh2951');

var UserSchema = new mongoose.Schema({
    application: { type: mongoose.Schema.Types.ObjectId, ref: 'Application', required: true, index: true },
    name: { type: String, required: true, index: true },
  });

var User = mongoose.model('user', UserSchema);

User.create({ application: '000000000000000000000000', name: 'test' }, function(error, doc) {
  User.findOne({ _id: doc._id }, function(error, doc) {
    console.log(doc);
    console.log(doc.application);
    process.exit(0);
  });
});

Output:

Mongoose: users.ensureIndex({ application: 1 }) { background: true, safe: undefined }  
Mongoose: users.insert({ application: ObjectId("000000000000000000000000"), name: 'test', _id: ObjectId("554bcbdeb3a4810234c75deb"), __v: 0 })   
Mongoose: users.ensureIndex({ name: 1 }) { background: true, safe: undefined }  
Mongoose: users.findOne({ _id: ObjectId("554bcbdeb3a4810234c75deb") }) { fields: undefined }  
{ _id: 554bcbdeb3a4810234c75deb,
  application: 000000000000000000000000,
  name: 'test',
  __v: 0 }
000000000000000000000000

As expected. Can you confirm the above script works for you, and, if so, what I'm doing above that's different from your example?

[swordfish444]

It's occurring when streaming the documents. Redo your test but instead stream the documents as the example @rodriguise provided:

 var stream = db.User
        .find(query)
        .stream();

 stream.on('data', function(user) {
     // user.application is null or undefined
 }

[vkarpov15]

Same thing with streams:

var mongoose = require('mongoose');
mongoose.set('debug', true);
var util = require('util');

mongoose.connect('mongodb://localhost:27017/gh2951');

var UserSchema = new mongoose.Schema({
    application: { type: mongoose.Schema.Types.ObjectId, ref: 'Application', required: true, index: true },
    name: { type: String, required: true, index: true },
  });

var User = mongoose.model('user', UserSchema);

User.create({ application: '000000000000000000000000', name: 'test' }, function(error, doc) {
  User.find({ _id: doc._id }).stream().on('data', function(doc) {
    console.log(doc);
    console.log(doc.application);
    process.exit(0);
  });
});

Output:

Mongoose: users.ensureIndex({ application: 1 }) { background: true, safe: undefined }  
Mongoose: users.insert({ application: ObjectId("000000000000000000000000"), name: 'test', _id: ObjectId("554bd229c0db694137e8bfde"), __v: 0 }) {}  
Mongoose: users.ensureIndex({ name: 1 }) { background: true, safe: undefined }  
Mongoose: users.find({ _id: ObjectId("554bd229c0db694137e8bfde") }) { fields: undefined }  
{ _id: 554bd229c0db694137e8bfde,
  application: 000000000000000000000000,
  name: 'test',
  __v: 0 }
000000000000000000000000

[swordfish444]

Hmm it's something else then. All i know, is when I remove ref from my schema its working. The problem is, if I go this route than I'm recreating what should already be working in mongoose having to keep a dictionary of ref paths to model name.

[swordfish444]

Could this be something related to using a replica set db? I'm trying everything here and I cannot figure out why.

[vkarpov15]

It's very strange that removing ref fixes the issue. Can you show me a more complete example of your schema - are you using custom getters / setters, select field, etc.?

[swordfish444]

I'll try and put something together from what we have. It's difficult because there are so many pieces involved. I'm using an older version of baucis (pre express 4.0) which shouldn't make a difference from how it uses mongoose, but maybe I've overlooked something. Judging from some of the other issues cropping up with 4+ mongoose, I'm thinking there are deeper issues still needing to be worked out before this can be used in production.

Switching to 4 was hopefully going to allow chaining of middleware to function properly. Where middleware will execute for the model that was initiated by middleware to begin with. I'll update you with my progress on putting together a more complete example.

@rodriguise Would you be able to also provide a more complete of example of your setup thats causing the same null subdocuments?

[vkarpov15]

I've been using mongoose 4 in production since 3.9.7 in December, and the only issues that cropped up where bugs in my code rather than mongoose. Likewise, I know a couple companies here in the NY area have upgraded without issue. Bugs, there definitely are, but mongoose 4 is a huge improvement in terms of stability and functionality over its predecessor.

Which version of baucis are you using? Perhaps that may contribute to the issue. I doubt that replica set has anything to do with this, mongoose doesn't have any special behavior for replica sets, that's all in the underlying driver.

[swordfish444]

@vkarpov15 Okay so I'm a little bit shocked right now... I updated mongoose to the latest commit you submitted for #2926 and updated my project locally, and as bizarre as it is... I'm able to see the subdocuments now.

Leave this open over the weekend and if you don't see anything more from me, then close on monday. Thanks for the help once again.

[vkarpov15]

That's great to hear! Looks like some of the fixes that are scheduled for 4.0.3 are already paying dividends :)

[doomdagger]

I've trapped in the same issue here, using mongoose 4.0.2.

// part of my schema structure
var userSchema = new mongoose.Schema({
    ......
    roles: [{
                type: String,
                ref: 'Role'
    }],
    ......
});

var User = mongoose.model('User', userSchema);

if I use User.findOne to fetch the document, the 'roles' path will be an empty array. If I remove the ref: 'Role' showed above, the issue will disappear.

However, to my surprise, this 'ref' issue won't occur for the following code I write for testing:

var blogSchema = new mongoose.Schema({
    title: String,
    pest: [{
        type: String,
        ref: 'Pest'
    }]
});

var pestSchema = new Schema({
    gogo: String
});

var Blog = mongoose.model('Blog', blogSchema);
var Pest = mongoose.model('Pest', pestSchema);

var blog = new Blog({
    title: "heihei",
    pest: ["000000000000000000000000"]
});

// I use bluebird's promisifyAll to generate the async method
Blog.findOneAsync({_id: id}).then(function (res) {
        console.log(res);
        // { title: 'heihei', pest: [ '000000000000000000000000' ] }
});

I wonder why?

[doomdagger]

Hi, Guys! I kind of figuring out why I have the issue mentioned above.

// here is part of my User schema definition
var userSchema = new mongoose.Schema({
    ......
    roles: [{
                type: String,  // I use String type for the ref key
                ref: 'Role'
    }],
    ......
});

But when inserting the documents, I use the original api provided by mongodb
driver, and mistakenly use ObjectId as data type of path 'roles'.

var user = {roles: [Object('000000000000000000000000')], ......}

After inserting this document into mongodb, the path 'roles' of the document
'user' will have the ObjectId as its data type.
While if using mongoose to insert the document, the ObjectId will be automatically casted into
String, thus keeping the coherence of the data type.
Therefore, I try to change the code into this:

// casting the ObjectId to String
var user = {roles: [ Object('000000000000000000000000')+'' ], ......}

Even if I still using mongodb driver to insert the document, the data type will
not be ObjectId anymore.
Now if I use find method provided by mongoose to fetch the documents, all path can
be retrieved and problem disappears. Seems that it's the data type mismatch causing this issue.
Hope this can be helpful to someone. :smile:

[stableShip]

happen again when using [email protected]
I have no idea why , just remove the ref, then I can get the id.
or change the data type to String in database. like:
"community_id" : ObjectId("572aedf2c899c2e631ab2cf3") to "community_id" : "572aedf2c899c2e631ab2cf3"
schema:

new Schema({
 community_id: {type: ObjectId, ref: 'community'}, 
//....
});

all data did not create by code, I insert date use mongochef in the beginning. just like:

{
    "_id": ObjectId("5735b8af1fb0f51ee5d2e956"),
     "community_id" : ObjectId("572aedf2c899c2e631ab2cf3")
}

[vkarpov15]

@stableShip what does your query look like?

[stableShip]

yield User.findOne({name: "someOne"});

using npm3. node -v:4.4.1

[vkarpov15]

How is name defined in your schema?

[stableShip]

var userSchema = new Schema({
    name: String,
   community_id: {type: ObjectId, ref: 'community'}
});

[vkarpov15]

Below script works fine for me. Re-open with additional context if this is still an issue:

'use strict';

var assert = require('assert');
var mongoose = require('mongoose');
var Schema = mongoose.Schema;

mongoose.connect('mongodb://localhost/gh2951');
mongoose.set('debug', true);

var co = require('co');

var userSchema = new Schema({
  name: String,
  community_id: {type: mongoose.Schema.Types.ObjectId, ref: 'community'}
});

var User = mongoose.model('User', userSchema);

User.remove({}, function(error) {
  assert.ifError(error);
  User.collection.insertOne({ name: 'test', community_id: new mongoose.Types.ObjectId('000000000000000000000000') }, function(error) {
    assert.ifError(error);
    User.findOne({ name: 'test' }, function(error, doc) {
      assert.ifError(error);
      assert.ok(doc.community_id);
      console.log('done');
      process.exit(0);
    });
  });
});

[MichielDeMey]

Looks like we're experiencing the exact same issue on mongoose:4.8.1.
One difference is that our schema's are defined in a separate node module. Model initialisation however is still happening in our root project.

Is there a way to debug this issue further?

[torihuang]

+1

[whats0n0]

I got the same issue in 4.8.4. Is this known or not reproduced? @vkarpov15

[vkarpov15]

Not reproduceable. Please provide a standalone script that reproduces this issue.

[whats0n0]

Okay, so just as @MichielDeMey mentioned the issue is apparently caused by the schema being created in a separate node module (using a separate mongoose instance).

I don't know if this is not allowed or if it's a bug, but either way I have created a repo reproducing the issue: https://github.com/whats0n0/mongoose-2951

[mgcrea]

Any chance we could reopen this? I'm encountering this as well on latest release.

[vkarpov15]

@mgcrea opened up a separate issue to track :point_up: will try to repro later