Mongoose: Not able to persist array of objects in mongo using mongoose

[mukulgupta2507]

Hi,

I'm trying to persist an array of objects in a document using mongoose. I have tried multiple times but it's not persisting array in document. It places an empty array in document.

Following is my Schema:

var ProfileSchema = new Schema({

  name: String,
  PagesData: [{
    pageAccessToken: {type: String, get: decryptText, set: encryptText},
    category: String,
    name: String,
    id: String,
    perms: [String]
  }]

});

module.exports = mongoose.model('Profile', ProfileSchema);

I'm trying to save a document with an array of objects using following query:

var newProfile = new Profile();
newProfile.name = "someName";
newProfile.PagesData = [ { pageAccessToken: 'someToken',
    category: 'Bags/Luggage',
    name: 'someBrandName',
    id: '12345',
    perms: 
     [ 'ADMINISTER',
       'EDIT_PROFILE',
       'CREATE_CONTENT' ] } ];

newProfile.save(function(err, result, numAffected){
    if(err) {
        console.log(err);
        res.send(500, "Error");
    }
    console.log(result);
    res.send(200, "Success");
});

I tried debugging the mongo commands using

    require('mongoose').set('debug', true);

On Debug logs it shows, empty array during insert command execution.

Can anyone please tell me how can I store this array of object in my schema ?

Thanks,

[vkarpov15]

Hmmm does it work if you remove the id field?

[mukulgupta2507]

@vkarpov15 : I tried using save without id field but even then it is not working. Any idea what can be the possible reason ? Also, I have tried .create but that is also not working.

[mukulgupta2507]

@vkarpov15 : Just for information, are you able to reproduce the issue ? Also, if this is a bug, then it will definitely take some time to be solved. Meanwhile, is there any work around which I can use for solving my case ?

[vkarpov15]

Nope can't repro, below script works fine for me in 4.1.2. Which version are you using?

var assert = require('assert');
var mongoose = require('mongoose');
mongoose.set('debug', true);
mongoose.connect('mongodb://localhost/gh3249');

var Schema = mongoose.Schema;

var ProfileSchema = new Schema({

  name: String,
  PagesData: [{
    pageAccessToken: { type: String },
    category: String,
    name: String,
    id: String,
    perms: [String]
  }]

});

var Profile = mongoose.model('Profile', ProfileSchema);

var newProfile = new Profile();
newProfile.name = "someName";
newProfile.PagesData = [ { pageAccessToken: 'someToken',
    category: 'Bags/Luggage',
    name: 'someBrandName',
    id: '12345',
    perms: 
     [ 'ADMINISTER',
       'EDIT_PROFILE',
       'CREATE_CONTENT' ] } ];

newProfile.save(function(error, result, numAffected) {
  assert.ifError(error);
  assert.ok(result.PagesData.length);
  Profile.findById(result._id).exec(function(error, doc) {
    assert.ifError(error);
    assert.ok(doc.PagesData.length);
    console.log('done');
    process.exit(0);
  });
});

[mukulgupta2507]

@vkarpov15
This is my dependecy in package.json.
"mongoose": "~3.8.8"

[mukulgupta2507]

I have upgraded to mongoose: 4.1.2. It's still not working for me.
Also, I have changed id to pageId in the above schema so as to avoid any issue.

For debugging purpose, I tried putting console.log statements at various points in the code. There is one strange behavior which I'm observing.

pagesData is populated through some api call and is correct, I have verified it. 
        var pagesArray = [];
        console.log("Start Debugging");
        for(var i=0;i<pagesData.length;i++) {
            console.log(pagesData[i]);
            var fbPageObj = {'pageAccessToken': pagesData[i]['access_token'], 'category': pagesData[i]['category'], 'name': pagesData[i]['name'], 'pageId': pagesData[i]['id'],'perms': pagesData[i]['perms']};
            pagesArray.push(fbPageObj);
        }
        newSocialProfile['fbpagesData'] = pagesArray;
        console.log(JSON.stringify(pagesArray));  // correctly populated
        console.log(JSON.stringify(newSocialProfile.fbpagesData));  // correctly prints array
        console.log(JSON.stringify(newSocialProfile)); // Show empty array for fbpagesData field, this is strange
        console.log("after all");
        console.log(JSON.stringify(newSocialProfile.fbpagesData));  // this again correctly shows the array data
        console.log("end");
        var obj = newSocialProfile.toObject(); 

Final obj is printed as:

{ name: 'facebook',
  _id: 55cacb6d62fd78f42834464d,
  fbPagesData: [],  // This is the issue, why is this empty ? }

[vkarpov15]

Do you have any transforms or other options defined in your schema?

Below script works fine for me. Try modifying the below script with your actual schema and see if you can give me a standalone example that reproduces this behavior.

var assert = require('assert');
var mongoose = require('mongoose');
mongoose.set('debug', true);
mongoose.connect('mongodb://localhost/gh3249');

var Schema = mongoose.Schema;

var ProfileSchema = new Schema({

  name: String,
  PagesData: [{
    pageAccessToken: { type: String },
    category: String,
    name: String,
    id: String,
    perms: [String]
  }]

});

var Profile = mongoose.model('Profile', ProfileSchema);

var newProfile = new Profile();
newProfile.name = "someName";
newProfile.PagesData = [ { pageAccessToken: 'someToken',
    category: 'Bags/Luggage',
    name: 'someBrandName',
    id: '12345',
    perms: 
     [ 'ADMINISTER',
       'EDIT_PROFILE',
       'CREATE_CONTENT' ] } ];

newProfile.save(function(error, result, numAffected) {
  assert.ifError(error);
  assert.ok(result.PagesData.length);
  Profile.findById(result._id).exec(function(error, doc) {
    assert.ifError(error);
    assert.ok(doc.PagesData.length);
    console.log('done');
    process.exit(0);
  });
});

[mukulgupta2507]

I have updated your script but it is working fine in your case. I'm still stuck in my case and will be thankful to you if you can provide any help in this case.

var assert = require('assert');
var mongoose = require('mongoose');
var crypto = require('crypto');

mongoose.set('debug', true);
mongoose.connect('mongodb://localhost/test');

var cryptKey = crypto.createHash('sha256').update('Nixnogen').digest();
var iv = "someIv";

var Schema = mongoose.Schema;

var ProfileSchema = new Schema({

  name: String,
  PagesData: [{
    pageAccessToken: { type: String, get: decryptText, set: encryptText },
    category: String,
    name: String,
    id: String,
    perms: [String]
  }],
  type: String

});

function encryptText(text){
        var cipher = crypto.createCipher('aes-256-cbc', cryptKey, iv);
        var crypted = cipher.update(text,'utf8','binary');
        crypted += cipher.final('binary');
    crypted = new Buffer(crypted, 'binary').toString('base64');
        return crypted;
}

function decryptText(text){
        if (text === null || typeof text === 'undefined' || text === '') {return text;};
    text = new Buffer(text, 'base64').toString('binary');
        var decipher = crypto.createDecipher('aes-256-cbc', cryptKey, iv);
        var dec = decipher.update(text,'binary','utf8');
        dec += decipher.final('utf8');
        return dec;
}

var Profile = mongoose.model('Profile', ProfileSchema);

var newProfile = new Profile();
newProfile.name = "someName";
var data = [];
var dataObj = {pageAccessToken:'iqULIV8BssssAIFJbchZAla3MF90UFZA6Q', category: 'Bags/Luggage', name: 'someBrandName', id: '12345', perms:    [ 'ADMINISTER',
     'EDIT_PROFILE',
     'CREATE_CONTENT',
     'MODERATE_CONTENT',
     'CREATE_ADS',
     'BASIC_ADMIN' ]};
data.push(dataObj);
newProfile.type = "facebook";

newProfile.PagesData = data;
console.log(JSON.stringify(newProfile));
var obj = newProfile.toObject();
delete  obj._id;

Profile.update({type: "facebook"}, obj, {upsert: true},function(err, result){
    if(err) {
        console.log(err);
    }
    console.log(result);
});

In my case, why newProfile.PagesData is not storing the correct value, please help me.

Thanks

[mukulgupta2507]

@vkarpov15
Have you taken a look at my previous code where I'm setting the value of array in the new object ? Do you think there is anything which I'm doing wrong ? It's been too long and I'm still not able to figure out the root cause of the problem. It'll be great if you can give some pointers towards the possible problems in the code.

[chetverikov]

@vkarpov15 I have a similar problem too.
example:

   // in req.body
   // regions_price: [{title: "wfe", price: 0, min: 0, free: 0}, {title: "wfwff", price: 2, min: 1, free: 30}]

    var self = this
      , id = req.params.id
      , body = req.body;

    this.model.findById(id, function(err, doc) {
      if (err || !doc) return done(err || new Err.NotFound('Document #' + id + ' not found'));
      // in doc.regions_price = [{title: "wfe", price: 0, min: 0, free: 0}];
      doc.set(body); 
      doc.save(function(err, document) {
        if (err) return done(err);

        // document not contain doc.regions_price[1] =  {title: "wfwff", price: 2, min: 1, free: 30};

        req.params.id = document.id;

        methods['get /:id'].call(self, req, res, done);
      });
    });

[chetverikov]

in 4.1.0, 4.1.1, 4.1.2

[mukulgupta2507]

As a work around for my problem, I decided to update the array field separately, but somehow it is also not working.

        Profile.update({type: "facebook"}, {'PagesData':pagesArray}, {upsert:true}, function(err, result){
            if(err) {
                console.log(err);
                res.send(500, err);
            }
            console.log("inserting pagesData");
            console.log(result);
            res.send(200, "Success");
        });

Result: { ok: 0, n: 0, nModified: 0 }

Any idea why is this not working ? I think this is the minimum thing which one can get, right ?

Updates:

Above query works for simple data fields:

        Profile.update({type: "facebook"}, {'type':'google'}, {upsert:true}, function(err, result){
            if(err) {
                console.log(err);
                res.send(500, err);
            }
            console.log("inserting pagesData");
            console.log(result);
            res.send(200, "Success");
        });
Result: { ok: 1, nModified: 1, n: 1 }

There is something wrong in the array of objects field, that's for sure. I'm surprised why your standalone script is working fine.

[vkarpov15]

Hmmm @mukulgupta2507 the script you provided gives me the below output:

$ node gh-3249.js 
{"name":"someName","_id":"55ccbb6bc4588ba85f737659","PagesData":[{"pageAccessToken":"W7S2iJMM2CLEjXtOk53GSJT0hDCDkJdlEe6NjXBWPpxQsGbjwTvFRFSU16uPWWXl","category":"Bags/Luggage","name":"someBrandName","id":"12345","_id":"55ccbb6bc4588ba85f73765a","perms":["ADMINISTER","EDIT_PROFILE","CREATE_CONTENT","MODERATE_CONTENT","CREATE_ADS","BASIC_ADMIN"]}]}
Mongoose: profiles.update({ type: 'facebook' }) { '$set': { name: 'someName', PagesData: [ { _id: ObjectId("55ccbb6bc4588ba85f73765a"), id: '12345', name: 'someBrandName', category: 'Bags/Luggage', pageAccessToken: 'rlrzXBtAP6XeZmDQ7EQlPRzHjOIiHbWENNBZih7CHY4vqhY10X2T6gmFfRJHZkjQxfsYcyjVyJYevsRabWADgYzAtwUimES+KznkGRfCFY0=', perms: [ 'ADMINISTER', 'EDIT_PROFILE', 'CREATE_CONTENT', 'MODERATE_CONTENT', 'CREATE_ADS', 'BASIC_ADMIN' ] } ] } } { upsert: true } 
{ ok: 1,
  nModified: 0,
  n: 1,
  upserted: [ { index: 0, _id: 55ccbb6b215dc792999d4536 } ] }

The output looks like what I expect and PagesData is successfully persisted to mongodb. What does the output look like on your end? Also, can you double check node_modules/mongoose/package.json and make sure you have the correct version?

One more thing you can try; try removing the type field in your schema.

[mukulgupta2507]

@vkarpov15
Few quick doubts:

1. I can see a difference in the syntax of update query used by me and the one used by you. Do you think that can make any difference in persisting data?
2. You are using $set explicitly whereas I have used a object directly.
3. You are directly giving the pageData array. Have you tried assigning it to a variable and then update using that variable, the way I have done that. Remember that the other guy is also assigning the array of objects through some variable. I know that shouldn't make a difference but just want to double check.
4. Here is the output from my package.json:

  "dependencies": {
    "async": "^1.2.1",
    "bcrypt-nodejs": "0.0.3",
    "body-parser": "~1.5.0",
    "composable-middleware": "^0.3.0",
    "compression": "~1.0.1",
    "connect-mongo": "^0.4.1",
    "cookie-parser": "~1.0.1",
    "email-templates": "^2.0.1",
    "errorhandler": "~1.0.0",
    "express": "~4.0.0",
    "express-jwt": "^0.1.3",
    "express-session": "^1.11.3",
    "fb": "^0.7.0",
    "hiredis": "^0.4.0",
    "jade": "~1.2.0",
    "jsonwebtoken": "^0.3.0",
    "lodash": "~2.4.1",
    "method-override": "~1.0.0",
    "mongoose": "^4.1.2",
    "morgan": "~1.0.0",
    "nodemailer": "^1.4.0",
    "parseuri": "0.0.4",
    "passport": "~0.2.0",
    "passport-local": "~0.1.6",
    "paypal-rest-sdk": "^1.6.1",
    "redis": "^0.12.1",
    "serve-favicon": "~2.0.1",
    "shortid": "^2.2.2",
    "socket.io": "^1.3.6",
    "socket.io-client": "^1.3.6",
    "socketio-jwt": "^2.3.5"
  }

1. You mentioned try ""removing the type field in your schema."" Should I use it like:

{String, get: decryptText, set: encryptText}

1. Have you used get and set methods like I mentioned in my script.
2. Also, if I'm using your script directly it is working fine but somehow from code it's not working.

P.S. I'm still trying out different techniques in order to resolve the issue.

Thanks

Update: Tried by giving directly the object without using any variable, still not working.
{ ok: 0, n: 0, nModified: 0 } This is the result which I'm getting again and again.

[vkarpov15]

No I mean:

var ProfileSchema = new Schema({

  name: String,
  PagesData: [{
    pageAccessToken: { type: String, get: decryptText, set: encryptText },
    category: String,
    name: String,
    id: String,
    perms: [String]
  }],
  type: String // <----- try getting rid of this

});

Also, here's the exact script I'm running that gives me the correct output:

var assert = require('assert');
var mongoose = require('mongoose');
var crypto = require('crypto');

mongoose.set('debug', true);
mongoose.connect('mongodb://localhost/gh3249');

var cryptKey = crypto.createHash('sha256').update('Nixnogen').digest();
var iv = "someIv";

var Schema = mongoose.Schema;

var ProfileSchema = new Schema({

  name: String,
  PagesData: [{
    pageAccessToken: { type: String, get: decryptText, set: encryptText },
    category: String,
    name: String,
    id: String,
    perms: [String]
  }],
  //type: String

});

function encryptText(text){
        var cipher = crypto.createCipher('aes-256-cbc', cryptKey, iv);
        var crypted = cipher.update(text,'utf8','binary');
        crypted += cipher.final('binary');
    crypted = new Buffer(crypted, 'binary').toString('base64');
        return crypted;
}

function decryptText(text){
        if (text === null || typeof text === 'undefined' || text === '') {return text;};
    text = new Buffer(text, 'base64').toString('binary');
        var decipher = crypto.createDecipher('aes-256-cbc', cryptKey, iv);
        var dec = decipher.update(text,'binary','utf8');
        dec += decipher.final('utf8');
        return dec;
}

var Profile = mongoose.model('Profile', ProfileSchema);

var newProfile = new Profile();
newProfile.name = "someName";
var data = [];
var dataObj = {pageAccessToken:'iqULIV8BssssAIFJbchZAla3MF90UFZA6Q', category: 'Bags/Luggage', name: 'someBrandName', id: '12345', perms:    [ 'ADMINISTER',
     'EDIT_PROFILE',
     'CREATE_CONTENT',
     'MODERATE_CONTENT',
     'CREATE_ADS',
     'BASIC_ADMIN' ]};
data.push(dataObj);
newProfile.type = "facebook";

newProfile.PagesData = data;
console.log(JSON.stringify(newProfile));
var obj = newProfile.toObject();
delete  obj._id;

Profile.update({type: "facebook"}, obj, {upsert: true},function(err, result){
    if(err) {
        console.log(err);
    }
    console.log(result);
});

It's the same as yours, but without the 'type' field. It works either way though, as long as I have a clean database.

Also, @chetverikov, can you provide me a standalone script please? Would be very helpful :)

[mukulgupta2507]

@vkarpov15
I'm still not able to come up with a script which can reproduce the issue but on debugging my own code I have found out something.

In my code, PagesData is not there before even executing Mongoose.update() command. These few lines are the root cause of the problem from my analysis:

    newProfile.fbpagesData = [{"pageAccessToken":"kbckjbdajbd","category":"Bags/Luggage","name":"My brand","pageId":"684738011661772","perms":["ADMINISTER","EDIT_PROFILE","CREATE_CONTENT","MODERATE_CONTENT","CREATE_ADS","BASIC_ADMIN"]}]; 
    console.log(JSON.stringify(newProfile.fbpagesData));  // So far good, I can see my object with the pagesData in newProfile.
    console.log(newProfile.toString());  // Here it is weird pagesData is printed as [ ], empty array
    console.log(newProfile.fbpagesData);  // this correctly prints the value of array, again strange
            var obj = newProfile.toObject();  // Here I convert it to Object 
    delete  obj._id;                                      // deletes the _id field so as to ensure upsert happens
    console.log(JSON.stringify(obj));         // Here also pagesData is printed as [ ]
    console.log(JSON.stringify(obj.fbpagesData));   // Here it prints undefined

So, what appears to me is that there is somehow no data for pagesData and that's why it is not getting persisted in mongo. So, there is something weird happening in these few lines of code. I tried to reproduce the issue but no luck :(

Do you think there is something wrong that is happening over here ? I'm still not sure if this is the root cause of the problem but just wanted to share my findings

Thanks

[chetverikov]

@vkarpov15 hohoohohoh
While I was writing the test I realized that the problem in my plugin, which keeps a history of document changes, but not yet figured out what the problem is...

[vkarpov15]

@mukulgupta2507 can you try printing out the value of newProfile.$isValid('fbpagesData');? If it's false, try printing newProfile.$__.validationError.errors['fbpagesData']

[mukulgupta2507]

@vkarpov15
It is printing true for the newProfile.$isValid('fbpagesData'); Don't know what is the issue here. I think I should change the schema itself. That will only solve the issue :(

[vkarpov15]

Another suggestion: try putting a try/catch around your encryptText and decryptText functions. There may be an error in those that I'm not seeing and for some reason isn't happening on my machine.

[mukulgupta2507]

@vkarpov15
I have changed my schema itself to unblock me. But stuck in some another problem:

I want to create a map with a objectId as key and an array of string values as its value. The closest that I can get is:

var schema = new Schema({
   map: [{myId: {type:mongoose.Schema.Types.ObjectId, ref: 'MyOtherCollection'}, values: [String]}]
});

But somehow this is not working for me. When I perform an update with {upsert: true}, it is not correctly populating the key: value in the map. In fact, I'm not even sure if I have declared the schema correctly.

Can you tell me if the schema is correct ? Also, How can I perform an update with {upsert: true} for this schema?
My use case is I want to keep a list of values for a given objectId. I don't want any duplicates entries with same key, that's why picked map.

Please suggest if the approach is correct or should this be modelled some other way?

[vkarpov15]

The schema looks correct. How are you using it?

Also, why not just store the array 'values' in 'MyOtherCollection'?

[vkarpov15]

Issue's been stale for a while, re-open if this is still an issue.

[123xylem]

Im trying to populate groups product att with numerous product models
But everytime I put a new one in it knocks the old one out
`//POST new Product
router.post("/home",function(req,res){
product.create(req.body.product,function(err,newProduct){ //WHY ISNT IT SAVING NEW PRODUCT TO DB
if(err){console.log(err)}
else{
var groupName=newProduct.category;

        group.findOneAndUpdate({name:groupName}, { product: newProduct.id },function(err,found){
              console.log(found,"FoundCategory Before product Ids in it<<<<<<<<")
            if (err){console.log(err)}
            else{
                found.product.push(newProduct);        

                found.save();               
                console.log(found,"FoundCategory AFTER product Ids in it<<<<<<<<")

                res.redirect("/home")}
        })

    }
})

})
Any idea why the new product overtakes the old product field in the array and the old one doesnt save? Below is my group schema var groupSchema= new mongoose.Schema({

name:String,
image:String,
product : [{type:mongoose.Schema.Types.ObjectId,
            ref: 'product' }]

})
`