[5.1.x] Backports for 5.1.0b3 release

django_mongodb_backend/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "5.1.0b3.dev0"
+__version__ = "5.1.0b3"
 
 # Check Django compatibility before other imports which may fail if the
 # wrong version of Django is installed.

django_mongodb_backend/compiler.py

@@ -332,15 +332,6 @@ def cursor_iter(self, cursor, chunk_size, columns):
 
     def check_query(self):
         """Check if the current query is supported by the database."""
-        if self.query.distinct:
-            # This is a heuristic to detect QuerySet.datetimes() and dates().
-            # "datetimefield" and "datefield" are the names of the annotations
-            # the methods use. A user could annotate with the same names which
-            # would give an incorrect error message.
-            if "datetimefield" in self.query.annotations:
-                raise NotSupportedError("QuerySet.datetimes() is not supported on MongoDB.")
-            if "datefield" in self.query.annotations:
-                raise NotSupportedError("QuerySet.dates() is not supported on MongoDB.")
         if self.query.extra:
             if any(key.startswith("_prefetch_related_") for key in self.query.extra):
                 raise NotSupportedError("QuerySet.prefetch_related() is not supported on MongoDB.")

django_mongodb_backend/features.py

@@ -58,10 +58,6 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         "model_fields.test_jsonfield.TestQuerying.test_icontains",
         # MongoDB gives ROUND(365, -1)=360 instead of 370 like other databases.
         "db_functions.math.test_round.RoundTests.test_integer_with_negative_precision",
-        # Truncating in another timezone doesn't work becauase MongoDB converts
-        # the result back to UTC.
-        "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_func_with_timezone",
-        "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_timezone_applied_before_truncation",
         # Unexpected alias_refcount in alias_map.
         "queries.tests.Queries1Tests.test_order_by_tables",
         # The $sum aggregation returns 0 instead of None for null.
@@ -272,81 +268,6 @@ def django_test_expected_failures(self):
             "update.tests.AdvancedTests.test_update_ordered_by_m2m_annotation",
             "update.tests.AdvancedTests.test_update_ordered_by_m2m_annotation_desc",
         },
-        "QuerySet.dates() is not supported on MongoDB.": {
-            "admin_changelist.tests.ChangeListTests.test_computed_list_display_localization",
-            "admin_changelist.tests.ChangeListTests.test_object_tools_displayed_no_add_permission",
-            "admin_views.tests.AdminViewBasicTest.test_change_list_sorting_override_model_admin",
-            "admin_views.tests.AdminViewBasicTest.test_multiple_sort_same_field",
-            "admin_views.tests.AdminViewListEditable.test_inheritance",
-            "admin_views.tests.CSSTest.test_changelist_field_classes",
-            "admin_views.tests.DateHierarchyTests",
-            "aggregation.tests.AggregateTestCase.test_dates_with_aggregation",
-            "annotations.tests.AliasTests.test_dates_alias",
-            "aggregation_regress.tests.AggregationTests.test_more_more_more2",
-            "backends.tests.DateQuotingTest.test_django_date_trunc",
-            "dates.tests.DatesTests.test_dates_trunc_datetime_fields",
-            "dates.tests.DatesTests.test_related_model_traverse",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_allow_empty_archive_view",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_by_month",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_context_object_name",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_custom_sorting",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_custom_sorting_dec",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_template",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_template_suffix",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_date_list_order",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_no_duplicate_query",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_paginated_archive_view",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_paginated_archive_view_does_not_load_entire_table",
-            "generic_views.test_dates.MonthArchiveViewTests.test_custom_month_format",
-            "generic_views.test_dates.MonthArchiveViewTests.test_date_list_order",
-            "generic_views.test_dates.MonthArchiveViewTests.test_month_view",
-            "generic_views.test_dates.MonthArchiveViewTests.test_month_view_allow_empty",
-            "generic_views.test_dates.MonthArchiveViewTests.test_month_view_allow_future",
-            "generic_views.test_dates.MonthArchiveViewTests.test_month_view_get_month_from_request",
-            "generic_views.test_dates.MonthArchiveViewTests.test_month_view_paginated",
-            "generic_views.test_dates.MonthArchiveViewTests.test_previous_month_without_content",
-            "generic_views.test_dates.YearArchiveViewTests.test_date_list_order",
-            "generic_views.test_dates.YearArchiveViewTests.test_get_context_data_receives_extra_context",
-            "generic_views.test_dates.YearArchiveViewTests.test_no_duplicate_query",
-            "generic_views.test_dates.YearArchiveViewTests.test_year_view",
-            "generic_views.test_dates.YearArchiveViewTests.test_year_view_allow_future",
-            "generic_views.test_dates.YearArchiveViewTests.test_year_view_custom_sort_order",
-            "generic_views.test_dates.YearArchiveViewTests.test_year_view_empty",
-            "generic_views.test_dates.YearArchiveViewTests.test_year_view_make_object_list",
-            "generic_views.test_dates.YearArchiveViewTests.test_year_view_paginated",
-            "generic_views.test_dates.YearArchiveViewTests.test_year_view_two_custom_sort_orders",
-            "many_to_one.tests.ManyToOneTests.test_select_related",
-            "model_regress.tests.ModelTests.test_date_filter_null",
-            "reserved_names.tests.ReservedNameTests.test_dates",
-            "queryset_pickle.tests.PickleabilityTestCase.test_specialized_queryset",
-        },
-        "QuerySet.datetimes() is not supported on MongoDB.": {
-            "admin_views.test_templatetags.DateHierarchyTests",
-            "admin_views.test_templatetags.AdminTemplateTagsTest.test_override_change_list_template_tags",
-            "admin_views.tests.AdminViewBasicTest.test_date_hierarchy_empty_queryset",
-            "admin_views.tests.AdminViewBasicTest.test_date_hierarchy_local_date_differ_from_utc",
-            "admin_views.tests.AdminViewBasicTest.test_date_hierarchy_timezone_dst",
-            "annotations.tests.AliasTests.test_datetimes_alias",
-            "datetimes.tests.DateTimesTests.test_21432",
-            "datetimes.tests.DateTimesTests.test_datetimes_has_lazy_iterator",
-            "datetimes.tests.DateTimesTests.test_datetimes_returns_available_dates_for_given_scope_and_given_field",
-            "datetimes.tests.DateTimesTests.test_related_model_traverse",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_aware_datetime_archive_view",
-            "generic_views.test_dates.ArchiveIndexViewTests.test_datetime_archive_view",
-            "generic_views.test_dates.MonthArchiveViewTests.test_aware_datetime_month_view",
-            "generic_views.test_dates.MonthArchiveViewTests.test_datetime_month_view",
-            "generic_views.test_dates.YearArchiveViewTests.test_aware_datetime_year_view",
-            "generic_views.test_dates.YearArchiveViewTests.test_datetime_year_view",
-            "model_inheritance_regress.tests.ModelInheritanceTest.test_issue_7105",
-            "queries.tests.Queries1Tests.test_ticket7155",
-            "queries.tests.Queries1Tests.test_ticket7791",
-            "queries.tests.Queries1Tests.test_tickets_6180_6203",
-            "queries.tests.Queries1Tests.test_tickets_7087_12242",
-            "timezones.tests.LegacyDatabaseTests.test_query_datetimes",
-            "timezones.tests.NewDatabaseTests.test_query_datetimes",
-            "timezones.tests.NewDatabaseTests.test_query_datetimes_in_other_timezone",
-        },
         "QuerySet.extra() is not supported.": {
             "aggregation.tests.AggregateTestCase.test_exists_extra_where_with_aggregate",
             "annotations.tests.NonAggregateAnnotationTestCase.test_column_field_ordering",
@@ -366,6 +287,7 @@ def django_test_expected_failures(self):
             "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_with_extra_and_values_list",
             "queries.tests.EscapingTests.test_ticket_7302",
             "queries.tests.Queries1Tests.test_tickets_1878_2939",
+            "queries.tests.Queries1Tests.test_tickets_7087_12242",
             "queries.tests.Queries5Tests.test_extra_select_literal_percent_s",
             "queries.tests.Queries5Tests.test_ticket7256",
             "queries.tests.ValuesQuerysetTests.test_extra_multiple_select_params_values_order_by",

django_mongodb_backend/fields/embedded_model.py

@@ -7,7 +7,6 @@
 from django.db.models.lookups import Transform
 
 from .. import forms
-from .json import build_json_mql_path
 
 
 class EmbeddedModelField(models.Field):
@@ -155,54 +154,41 @@ def __init__(self, key_name, ref_field, *args, **kwargs):
         self.key_name = str(key_name)
         self.ref_field = ref_field
 
+    def get_lookup(self, name):
+        return self.ref_field.get_lookup(name)
+
     def get_transform(self, name):
         """
         Validate that `name` is either a field of an embedded model or a
         lookup on an embedded model's field.
         """
-        result = None
-        if isinstance(self.ref_field, EmbeddedModelField):
-            opts = self.ref_field.embedded_model._meta
-            new_field = opts.get_field(name)
-            result = KeyTransformFactory(name, new_field)
+        if transform := self.ref_field.get_transform(name):
+            return transform
+        suggested_lookups = difflib.get_close_matches(name, self.ref_field.get_lookups())
+        if suggested_lookups:
+            suggested_lookups = " or ".join(suggested_lookups)
+            suggestion = f", perhaps you meant {suggested_lookups}?"
         else:
-            if self.ref_field.get_transform(name) is None:
-                suggested_lookups = difflib.get_close_matches(name, self.ref_field.get_lookups())
-                if suggested_lookups:
-                    suggested_lookups = " or ".join(suggested_lookups)
-                    suggestion = f", perhaps you meant {suggested_lookups}?"
-                else:
-                    suggestion = "."
-                raise FieldDoesNotExist(
-                    f"Unsupported lookup '{name}' for "
-                    f"{self.ref_field.__class__.__name__} '{self.ref_field.name}'"
-                    f"{suggestion}"
-                )
-            result = KeyTransformFactory(name, self.ref_field)
-        return result
+            suggestion = "."
+        raise FieldDoesNotExist(
+            f"Unsupported lookup '{name}' for "
+            f"{self.ref_field.__class__.__name__} '{self.ref_field.name}'"
+            f"{suggestion}"
+        )
 
-    def preprocess_lhs(self, compiler, connection):
+    def as_mql(self, compiler, connection):
         previous = self
-        embedded_key_transforms = []
-        json_key_transforms = []
+        key_transforms = []
         while isinstance(previous, KeyTransform):
-            if isinstance(previous.ref_field, EmbeddedModelField):
-                embedded_key_transforms.insert(0, previous.key_name)
-            else:
-                json_key_transforms.insert(0, previous.key_name)
+            key_transforms.insert(0, previous.key_name)
             previous = previous.lhs
         mql = previous.as_mql(compiler, connection)
-        # The first json_key_transform is the field name.
-        embedded_key_transforms.append(json_key_transforms.pop(0))
-        return mql, embedded_key_transforms, json_key_transforms
-
-    def as_mql(self, compiler, connection):
-        mql, key_transforms, json_key_transforms = self.preprocess_lhs(compiler, connection)
         transforms = ".".join(key_transforms)
-        result = f"{mql}.{transforms}"
-        if json_key_transforms:
-            result = build_json_mql_path(result, json_key_transforms)
-        return result
+        return f"{mql}.{transforms}"
+
+    @property
+    def output_field(self):
+        return self.ref_field
 
 
 class KeyTransformFactory:

django_mongodb_backend/functions.py

@@ -1,4 +1,8 @@
+from datetime import datetime
+
+from django.conf import settings
 from django.db import NotSupportedError
+from django.db.models import DateField, DateTimeField, TimeField
 from django.db.models.expressions import Func
 from django.db.models.functions.comparison import Cast, Coalesce, Greatest, Least, NullIf
 from django.db.models.functions.datetime import (
@@ -195,6 +199,33 @@ def trunc(self, compiler, connection):
     return {"$dateTrunc": lhs_mql}
 
 
+def trunc_convert_value(self, value, expression, connection):
+    if connection.vendor == "mongodb":
+        # A custom TruncBase.convert_value() for MongoDB.
+        if value is None:
+            return None
+        convert_to_tz = settings.USE_TZ and self.get_tzname() != "UTC"
+        if isinstance(self.output_field, DateTimeField):
+            if convert_to_tz:
+                # Unlike other databases, MongoDB returns the value in UTC,
+                # so rather than setting the time zone equal to self.tzinfo,
+                # the value must be converted to tzinfo.
+                value = value.astimezone(self.tzinfo)
+        elif isinstance(value, datetime):
+            if isinstance(self.output_field, DateField):
+                if convert_to_tz:
+                    value = value.astimezone(self.tzinfo)
+                # Truncate for Trunc(..., output_field=DateField)
+                value = value.date()
+            elif isinstance(self.output_field, TimeField):
+                if convert_to_tz:
+                    value = value.astimezone(self.tzinfo)
+                # Truncate for Trunc(..., output_field=TimeField)
+                value = value.time()
+        return value
+    return self.convert_value(value, expression, connection)
+
+
 def trunc_date(self, compiler, connection):
     # Cast to date rather than truncate to date.
     lhs_mql = process_lhs(self, compiler, connection)
@@ -217,6 +248,9 @@ def trunc_date(self, compiler, connection):
 
 
 def trunc_time(self, compiler, connection):
+    tzname = self.get_tzname()
+    if tzname and tzname != "UTC":
+        raise NotSupportedError(f"TruncTime with tzinfo ({tzname}) isn't supported on MongoDB.")
     lhs_mql = process_lhs(self, compiler, connection)
     return {
         "$dateFromString": {
@@ -254,6 +288,7 @@ def register_functions():
     Substr.as_mql = substr
     Trim.as_mql = trim("trim")
     TruncBase.as_mql = trunc
+    TruncBase.convert_value = trunc_convert_value
     TruncDate.as_mql = trunc_date
     TruncTime.as_mql = trunc_time
     Upper.as_mql = preserve_null("toUpper")

django_mongodb_backend/operations.py

@@ -10,7 +10,7 @@
 from django.db.backends.base.operations import BaseDatabaseOperations
 from django.db.models import TextField
 from django.db.models.expressions import Combinable, Expression
-from django.db.models.functions import Cast
+from django.db.models.functions import Cast, Trunc
 from django.utils import timezone
 from django.utils.regex_helper import _lazy_re_compile
 
@@ -97,16 +97,26 @@ def get_db_converters(self, expression):
                 ]
             )
         elif internal_type == "DateField":
-            converters.append(self.convert_datefield_value)
+            # Trunc(... output_field="DateField") values must remain datetime
+            # until Trunc.convert_value() so they can be converted from UTC
+            # before truncation.
+            if not isinstance(expression, Trunc):
+                converters.append(self.convert_datefield_value)
         elif internal_type == "DateTimeField":
             if settings.USE_TZ:
                 converters.append(self.convert_datetimefield_value)
         elif internal_type == "DecimalField":
             converters.append(self.convert_decimalfield_value)
+        elif internal_type == "EmbeddedModelField":
+            converters.append(self.convert_embeddedmodelfield_value)
         elif internal_type == "JSONField":
             converters.append(self.convert_jsonfield_value)
         elif internal_type == "TimeField":
-            converters.append(self.convert_timefield_value)
+            # Trunc(... output_field="TimeField") values must remain datetime
+            # until Trunc.convert_value() so they can be converted from UTC
+            # before truncation.
+            if not isinstance(expression, Trunc):
+                converters.append(self.convert_timefield_value)
         elif internal_type == "UUIDField":
             converters.append(self.convert_uuidfield_value)
         return converters
@@ -142,6 +152,18 @@ def convert_durationfield_value(self, value, expression, connection):
                 value = datetime.timedelta(milliseconds=int(str(value)))
         return value
 
+    def convert_embeddedmodelfield_value(self, value, expression, connection):
+        if value is not None:
+            # Apply database converters to each field of the embedded model.
+            for field in expression.output_field.embedded_model._meta.fields:
+                field_expr = Expression(output_field=field)
+                converters = connection.ops.get_db_converters(
+                    field_expr
+                ) + field_expr.get_db_converters(connection)
+                for converter in converters:
+                    value[field.attname] = converter(value[field.attname], field_expr, connection)
+        return value
+
     def convert_jsonfield_value(self, value, expression, connection):
         """
         Convert dict data to a string so that JSONField.from_db_value() can

docs/source/ref/models/querysets.rst

@@ -8,9 +8,6 @@ Supported ``QuerySet`` methods
 All of Django's :doc:`QuerySet methods <django:ref/models/querysets>` are
 supported, except:
 
-  - :meth:`bulk_update() <django.db.models.query.QuerySet.bulk_update>`
-  - :meth:`dates() <django.db.models.query.QuerySet.dates>`
-  - :meth:`datetimes() <django.db.models.query.QuerySet.datetimes>`
   - :meth:`extra() <django.db.models.query.QuerySet.extra>`
   - :meth:`prefetch_related() <django.db.models.query.QuerySet.prefetch_related>`
 

docs/source/releases/5.1.x.rst

@@ -2,6 +2,23 @@
 Django MongoDB Backend 5.1.x
 ============================
 
+5.1.0 beta 3
+============
+
+*May 13, 2025*
+
+- Added support for a field's custom lookups and transforms in
+  ``EmbeddedModelField``, e.g. ``ArrayField``’s ``contains``,
+  ``contained__by``, ``len``, etc.
+- Fixed the results of queries that use the ``tzinfo`` parameter of the
+  ``Trunc`` database functions.
+- Added support for ``QuerySet.dates()`` and ``datetimes()``.
+- Fixed loading of ``QuerySet`` results for embedded models that have fields
+  that use database converters. For example, a crash for ``DecimalField``:
+  ``ValidationError: ['“1” value must be a decimal number.']``).
+
+.. _django-mongodb-backend-5.1.0-beta-2:
+
 5.1.0 beta 2
 ============