Mongoengine: regression from 0.9.0: assert on reverse_delete_rule=CASCADE with source.id.field != dest.id.field

[alon]

The regression seems to be in queryset/base.py BaseQuerySet.delete, in creating the queryset in 0.9.0 there is only one clause, but in 0.10.0 there is an additional 'id__nin' clause that implicitly assumes the id of both objects is of the same field, and breaks when they are not, as in this example.

Additionally it doesn't make sense to check for id of a different collection not in this collection.

The following example breaks in 0.10.0 but works in 0.9.0:

from mongoengine import connect, Document, StringField, ReferenceField, CASCADE

client = connect("alon-test")
#db = client.get_database('alon-test')
#idtest = db.create_collection('idtest') # only in 0.10.0?

class Client(Document):
    id = StringField(primary_key=True)
    foo = StringField()


class Token(Document):
    client = ReferenceField('Client', dbref=False, reverse_delete_rule=CASCADE)
    somethingelse = StringField()


client = Client(id="1234", foo="bar")
client.save()

token = Token(client=client, somethingelse='something else entirely')
token.save()

client.delete()

The error in 0.10.0 results from the 'id__nin' clause on the rule's queryset that is checking the id of Token, which is ObjectId, but should be checking the id of Client, which is a StringField:

$ python mongo_test.py 
Traceback (most recent call last):
  File "mongo_test.py", line 24, in <module>
    client.delete()
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/document.py", line 491, in delete
    **self._object_key).delete(write_concern=write_concern, _from_doc_delete=True)
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/queryset/base.py", line 409, in delete
    ref_q_count = ref_q.count()
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/queryset/queryset.py", line 104, in count
    return super(QuerySet, self).count(with_limit_and_skip)
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/queryset/base.py", line 347, in count
    return self._cursor.count(with_limit_and_skip=with_limit_and_skip)
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/queryset/base.py", line 1445, in _cursor
    self._cursor_obj = self._collection.find(self._query,
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/queryset/base.py", line 1479, in _query
    self._mongo_query = self._query_obj.to_query(self._document)
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/queryset/visitor.py", line 90, in to_query
    query = query.accept(QueryCompilerVisitor(document))
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/queryset/visitor.py", line 155, in accept
    return visitor.visit_query(self)
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/queryset/visitor.py", line 78, in visit_query
    return transform.query(self.document, **query.query)
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/queryset/transform.py", line 102, in query
    value = [field.prepare_query_value(op, v) for v in value]
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/base/fields.py", line 446, in prepare_query_value
    return self.to_mongo(value)
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/base/fields.py", line 442, in to_mongo
    self.error(unicode(e))
  File "/home2/alon/consumer/env_main/lib/python2.7/site-packages/mongoengine/base/fields.py", line 144, in error
    raise ValidationError(message, errors=errors, field_name=field_name)
mongoengine.errors.ValidationError: u'1234' is not a valid ObjectId, it must be a 12-byte input or a 24-character hex string

The query in 0.9.0:

ref_q = document_cls.objects(**{field_name + '__in': self})

in 0.10.0:

ref_q = document_cls.objects(**{field_name + '__in': self, 'id__nin': cascade_refs})

[AndrewBurdyug]

Seems like related to this bug one more case.
For example we have models with custom name of field which used as primary key:

class User(Document):
    """Collection of users profiles."""

    email = EmailField(required=True, unique=True)
    username = StringField(regex=r'[a-zA-Z0-9_-]+$', max_length=120,
                           required=True, unique=True)
    password = StringField(max_length=64, required=True)
    is_superuser = BooleanField(default=False)
    is_disabled = BooleanField(default=False)

    meta = {
        'collection': 'users',
        'db_alias': 'makechat_test' if TEST_MODE else 'makechat',
        'indexes': ['email', 'username', 'password']
    }

    def __str__(self):
        """Standart python magic __str__ method."""
        return self.username

class Session(Document):
    """Collection of users sessions."""

    user = ReferenceField(User, reverse_delete_rule=CASCADE)
    created = DateTimeField(default=datetime.now)
    value = StringField(max_length=64, primary_key=True)

    meta = {
        'collection': 'sessions',
        'db_alias': 'makechat_test' if TEST_MODE else 'makechat',
        'indexes': [
            {'fields': ['created'], 'expireAfterSeconds': SESSION_TTL}
        ]
    }

then delete() will cause error:

>>> from makechat.models import User

>>> User.objects.all().delete()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/buran/envs/py3/lib/python3.4/site-packages/mongoengine/queryset/base.py", line 409, in delete
    ref_q_count = ref_q.count()
  File "/home/buran/envs/py3/lib/python3.4/site-packages/mongoengine/queryset/queryset.py", line 104, in count
    return super(QuerySet, self).count(with_limit_and_skip)
  File "/home/buran/envs/py3/lib/python3.4/site-packages/mongoengine/queryset/base.py", line 347, in count
    return self._cursor.count(with_limit_and_skip=with_limit_and_skip)
  File "/home/buran/envs/py3/lib/python3.4/site-packages/mongoengine/queryset/base.py", line 1481, in _cursor
    self._cursor_obj = self._collection.find(self._query,
  File "/home/buran/envs/py3/lib/python3.4/site-packages/mongoengine/queryset/base.py", line 1515, in _query
    self._mongo_query = self._query_obj.to_query(self._document)
  File "/home/buran/envs/py3/lib/python3.4/site-packages/mongoengine/queryset/visitor.py", line 90, in to_query
    query = query.accept(QueryCompilerVisitor(document))
  File "/home/buran/envs/py3/lib/python3.4/site-packages/mongoengine/queryset/visitor.py", line 155, in accept
    return visitor.visit_query(self)
  File "/home/buran/envs/py3/lib/python3.4/site-packages/mongoengine/queryset/visitor.py", line 78, in visit_query
    return transform.query(self.document, **query.query)
  File "/home/buran/envs/py3/lib/python3.4/site-packages/mongoengine/queryset/transform.py", line 61, in query
    raise InvalidQueryError(e)
mongoengine.errors.InvalidQueryError: Cannot resolve field "id"
Cannot resolve field "id"

but changing name to id will solve this problem:

class Session(Document):
    """Collection of users sessions."""

    user = ReferenceField(User, reverse_delete_rule=CASCADE)
    created = DateTimeField(default=datetime.now)
    id = StringField(max_length=64, primary_key=True)

    meta = {
        'collection': 'sessions',
        'db_alias': 'makechat_test' if TEST_MODE else 'makechat',
        'indexes': [
            {'fields': ['created'], 'expireAfterSeconds': SESSION_TTL}
        ]
    }

>>> from makechat.models import User

>>> User.objects.all().delete()
0

>>> 

Python 3.4.3
mongoengine 0.10.6

[leerobert]

Can also confirm this issue....

>>> ProblemSet.objects[0].problems
[<Problem: (_id=5679a27069684a06db90ef7a, type=56f1673c1e08170c7b6189e5)>, <Problem: (_id=5679a87369684a06fed59567, type=56f1673c1e08170c7b6189e5)>]
>>> p = ProblemSet.objects[0].problems[0]
>>> ProblemSet.objects(problems__in=p)
Traceback (most recent call last):
  File "<console>", line 1, in <module>
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/queryset/queryset.py", line 58, in __repr__
    self._populate_cache()
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/queryset/queryset.py", line 92, in _populate_cache
    self._result_cache.append(next(self))
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/queryset/base.py", line 1407, in __next__
    raw_doc = next(self._cursor)
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/queryset/base.py", line 1481, in _cursor
    self._cursor_obj = self._collection.find(self._query,
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/queryset/base.py", line 1515, in _query
    self._mongo_query = self._query_obj.to_query(self._document)
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/queryset/visitor.py", line 90, in to_query
    query = query.accept(QueryCompilerVisitor(document))
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/queryset/visitor.py", line 155, in accept
    return visitor.visit_query(self)
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/queryset/visitor.py", line 78, in visit_query
    return transform.query(self.document, **query.query)
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/queryset/transform.py", line 102, in query
    value = [field.prepare_query_value(op, v) for v in value]
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/queryset/transform.py", line 102, in <listcomp>
    value = [field.prepare_query_value(op, v) for v in value]
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/fields.py", line 706, in prepare_query_value
    return self.field.prepare_query_value(op, value)
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/fields.py", line 992, in prepare_query_value
    return self.to_mongo(value)
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/fields.py", line 969, in to_mongo
    id_ = id_field.to_mongo(id_)
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/base/fields.py", line 453, in to_mongo
    self.error(str(e))
  File "/Users/robert/.envs/api/lib/python3.4/site-packages/mongoengine/base/fields.py", line 155, in error
    raise ValidationError(message, errors=errors, field_name=field_name)
mongoengine.errors.ValidationError: 'id' is not a valid ObjectId, it must be a 12-byte input or a 24-character hex string

[wojcikstefan]

This is fixed now, sorry it took that long.

[wojcikstefan]

(this is also a dupe of #1224)