Mongoose: aggregate $match to match by _id, values must be cast to ObjectId

[whitecolor]

If you want use aggregation framework and use $match to match elements by _id you can not pass them as strings:

This options do not work:

{$unwind : "$items" },
            {$match: {'items._id': {$in:['5140a09be5c703ac2c000002', '5140a09be5c703ac2c000003']}} }

This works:

{$unwind : "$items" },
            {$match: {'items._id': {$in:[mongoose.Types.ObjectId('5140a09be5c703ac2c000002'), mongoose.Types.ObjectId('5140a09be5c703ac2c000003')]}} }

I think it is a bug as in other queries strings are implicitly converted to objectId type.

[aheckmann]

this _may_ be possible at some point if someone wants to analyze the entire aggregation pipeline for $project usage but until that happens casting is not supported.

I just realized that the docs are not clear. Will update that.

[mikeclymer]

+1 for getting casting built into the aggregation pipeline.

In the meantime, here is my ObjectId casting workaround, along with dynamic pipeline creation. Note: I use the aggregation framework for basic document filtering.

function buildFilterPipeline(modelType, data) {
  var aggregate = modelType.aggregate();

  // Append matchers as filters to the aggregate pipeline.
  for (var propertyName in data) {
    var filter       = {};
    var propertyPath = modelType.schema.path(propertyName);
    if (!propertyPath) throw ModelError.INVALID_FILTER_PROPERTY;

    if (propertyPath.instance === 'ObjectID') {
      filter[propertyName] = mongoose.Types.ObjectId(data[propertyName]);
    } else {
      filter[propertyName] = data[propertyName];
    }

    aggregate.match(filter);
  }

  return aggregate;
}

[codeHatcher]

Is casting still built into the aggregation pipeline? I am tracking down an issue that might be caused by my assumption that it will autocast a string to objectid.

[codeHatcher]

This still seems to be the case that it will not auto cast. Mikeclymer's post confused me into thinking that was not the issue I was experiencing.

[cuongsimplypost]

+1 for built-in casting for aggregation pipeline

[vkarpov15]

If you want to build it you're most welcome, but it's way more tricky than it's worth because $project and $group can change the schema in surprising ways so it's hard to infer what should be an objectid.

[miqmago]

I've just implemented this helper function that converts any suspicius ObjectId string to its $or representation. Maybe it's kind of inneficient but for me has been the workaround. Don't know if this could be the way that mongoose could do it (sincerely haven't got the chance to look inside the library), but if needed, I've also implemented all test cases.

Also have to say that this only works for strings and for $in operator, but hope that could be easily adapted to other operators:

function isCouldbeObjectId(str) {
    if (typeof str === 'string') {
        return /^[a-f\d]{24}$/i.test(str);
    } else if (Array.isArray(str)) {
        return str.every(arrStr => /^[a-f\d]{24}$/i.test(arrStr));
    }
    return false;
}

/**
 * Converts a query of ObjectId's to $or with the string value
 * and the casted ObjectId values
 *
 *    {
 *        user: objIdStr,
 *        user1: {
 *            $in: [objIdStr, objIdStr],
 *        },
 *        subobject: {
 *            user: objIdStr,
 *            user1: {
 *                $in: [objIdStr, objIdStr],
 *            },
 *        },
 *    }
 *
 * Will be converted to:
 *
 *    {
 *        $and: [
 *            { $or: [{ user: objIdStr }, { user: objIdObj }] },
 *            {
 *                $or: [
 *                    { user1: { $in: [objIdStr, objIdStr] } },
 *                    { user1: { $in: [objIdObj, objIdObj] } },
 *                ],
 *            },
 *        ],
 *        subobject: {
 *             $and: [
 *                { $or: [{ user: objIdStr }, { user: objIdObj }] },
 *                {
 *                    $or: [
 *                        { user1: { $in: [objIdStr, objIdStr] } },
 *                        { user1: { $in: [objIdObj, objIdObj] } }
 *                    ],
 *                },
 *            ],
 *        },
 *    },
 *
 * @param {Object} query the query that will be converted
 * @return converted query
 */
export function convertToObjectId$or(query) {
    /* eslint-disable no-param-reassign */
    if (typeof query !== 'object' || Array.isArray(query)) {
        return query;
    }

    return Object.keys(query).reduce((curr, subKey) => {
        if (isCouldbeObjectId(query[subKey])) {
            // Is an array of strings similar to ObjectId
            // or an string similar to ObjectId
            let multiMatch;
            const $or = [];

            multiMatch = {};
            multiMatch[subKey] = query[subKey];
            $or.push(multiMatch);

            multiMatch = {};
            multiMatch[subKey] = Array.isArray(query[subKey]) ?
                query[subKey].map(v => new Types.ObjectId(v)) :
                new Types.ObjectId(query[subKey]);
            $or.push(multiMatch);

            if (curr.$and) {
                curr.$and.push({ $or });
            } else if (curr.$or) {
                curr.$and = [{ $or: curr.$or }, { $or }];
                delete curr.$or;
            } else {
                curr.$or = $or;
            }
        } else if (typeof query[subKey] === 'object' && query[subKey].$in && isCouldbeObjectId(query[subKey].$in)) {
            // Is an array of strings similar to ObjectId
            // or an string similar to ObjectId
            let multiMatch;
            const $or = [];

            multiMatch = {};
            multiMatch[subKey] = query[subKey];
            $or.push(multiMatch);

            multiMatch = {};
            multiMatch[subKey] = {
                $in: query[subKey].$in.map(v => new Types.ObjectId(v)),
            };
            $or.push(multiMatch);

            if (curr.$and) {
                curr.$and.push({ $or });
            } else if (curr.$or) {
                curr.$and = [{ $or: curr.$or }, { $or }];
                delete curr.$or;
            } else {
                curr.$or = $or;
            }
        } else if (typeof query[subKey] === 'object' && !Array.isArray(query[subKey])) {
            curr[subKey] = convertToObjectId$or(query[subKey]);
        } else {
            curr[subKey] = query[subKey];
        }
        return curr;
    }, {});
    /* eslint-enable no-param-reassign */
}

[lazamar]

Oh man, this is really needed.

Here is my version of it.

const isObjectId = v => mongoose.Types.ObjectId.isValid(v);
const toObjectId = v => mongoose.Types.ObjectId(v);
const isPrimitiveType = v =>
    typeof v === "boolean"
    || typeof v === "number"
    || typeof v === "undefined"
    || v === null
    || v instanceof RegExp
    || typeof v === "string"
    || v instanceof Date;


const parseValue = v => (isObjectId(v) ? toObjectId(v) : isPrimitiveType(v) ? v : parseQuery(v));

const parseQuery = q => (Array.isArray(q) ? q.map(parseValue) : mapObjIndexed(parseValue, q));

I'm using mapObjIndexed from ramda ((http://ramdajs.com/docs/#mapObjIndexed)[http://ramdajs.com/docs/#mapObjIndexed])

[soichih]

I can't understand why mongo won't just auto-cast string to ObjectId, but here is what I've tried.

        function convert(node) {
            for(var k in node) {
                var v = node[k];
                if(v === null) continue;
                if(typeof v == 'string' && mongoose.Types.ObjectId.isValid(v)) node[k] = mongoose.Types.ObjectId(v);
                if(typeof v == 'object') convert(v); //recurse
            }
        }
        convert(query);

Please let me know if anyone can spot a bug..

[vkarpov15]

There's nothing wrong with that code that I can see, but the fundamental assumption is that anything that's a 24 char hex string in an aggregation pipeline should be cast to an ObjectId. That's not an assumption that I think mongoose should make, but if that assumption works for your application then go for it :+1:

[yilikunNEO]

i will very happy if this feature comes