Backport dbdiff.assertNoDiff

Author: pfouque

CHANGELOG

@@ -1,3 +1,6 @@
+Unreleased
+    Remove db-diff dependency
+
 2023-10-30
     Add support for Python 3.12
     Add support for Django 5.0

src/cities_light/apps.py

@@ -1,6 +1,10 @@
 from django.apps import AppConfig
+from django.core.serializers import register_serializer
 
 
 class CitiesLightConfig(AppConfig):
     default_auto_field = 'django.db.models.AutoField'
     name = 'cities_light'
+
+    def ready(self):
+        register_serializer('sorted_json', 'cities_light.serializers.json')

src/cities_light/serializers/__init__.py

@@ -0,0 +1 @@
+"""Serializers with predictible (ordered) output."""

src/cities_light/serializers/base.py

@@ -0,0 +1,80 @@
+"""Shared code for serializers."""
+
+import collections
+import datetime
+import decimal
+
+
+class BaseSerializerMixin(object):
+    """Serializer mixin for predictible and cross-db dumps."""
+
+    @classmethod
+    def recursive_dict_sort(cls, data):
+        """
+        Return a recursive OrderedDict for a dict.
+
+        Django's default model-to-dict logic - implemented in
+        django.core.serializers.python.Serializer.get_dump_object() - returns a
+        dict, this app registers a slightly modified version of the default
+        json serializer which returns OrderedDicts instead.
+        """
+        ordered_data = collections.OrderedDict(sorted(data.items()))
+
+        for key, value in ordered_data.items():
+            if isinstance(value, dict):
+                ordered_data[key] = cls.recursive_dict_sort(value)
+
+        return ordered_data
+
+    @classmethod
+    def remove_microseconds(cls, data):
+        """
+        Strip microseconds from datetimes for mysql.
+
+        MySQL doesn't have microseconds in datetimes, so dbdiff's serializer
+        removes microseconds from datetimes so that fixtures are cross-database
+        compatible which make them usable for cross-database testing.
+        """
+        for key, value in data['fields'].items():
+            if not isinstance(value, datetime.datetime):
+                continue
+
+            data['fields'][key] = datetime.datetime(
+                year=value.year,
+                month=value.month,
+                day=value.day,
+                hour=value.hour,
+                minute=value.minute,
+                second=value.second,
+                tzinfo=value.tzinfo
+            )
+
+    @classmethod
+    def normalize_decimals(cls, data):
+        """
+        Strip trailing zeros for constitency.
+
+        In addition, dbdiff serialization forces Decimal normalization, because
+        trailing zeros could happen in inconsistent ways.
+        """
+        for key, value in data['fields'].items():
+            if not isinstance(value, decimal.Decimal):
+                continue
+
+            if value % 1 == 0:
+                data['fields'][key] = int(value)
+            else:
+                data['fields'][key] = value.normalize()
+
+    def get_dump_object(self, obj):
+        """
+        Actual method used by Django serializers to dump dicts.
+
+        By overridding this method, we're able to run our various
+        data dump predictability methods.
+        """
+        data = super(BaseSerializerMixin, self).get_dump_object(obj)
+        self.remove_microseconds(data)
+        self.normalize_decimals(data)
+        data = self.recursive_dict_sort(data)
+        return data

src/cities_light/serializers/json.py

@@ -0,0 +1,15 @@
+"""Django JSON Serializer override."""
+
+from django.core.serializers import json as upstream
+
+from .base import BaseSerializerMixin
+
+
+__all__ = ('Serializer', 'Deserializer')
+
+
+class Serializer(BaseSerializerMixin, upstream.Serializer):
+    """Sorted dict JSON serializer."""
+
+
+Deserializer = upstream.Deserializer

src/cities_light/tests/base.py

@@ -1,11 +1,13 @@
 """."""
+import json
 import os
 from unittest import mock
 
 from django import test
 from django.core import management
 from django.conf import settings
 
+from io import StringIO
 
 class FixtureDir:
     """Helper class to construct fixture paths."""
@@ -80,3 +82,23 @@ def _patch(setting, *values):
             management.call_command('cities_light', progress=True,
                                     force_import_all=True,
                                     **options)
+
+    def export_data(self) -> bytes:
+        out = StringIO()
+        management.call_command(
+            "dumpdata",
+            "cities_light",
+            format="sorted_json",
+            natural_foreign=True,
+            indent=4,
+            stdout=out
+        )
+        return out.getvalue()
+
+    def assertNoDiff(self, fixture_path):
+        """Assert that dumped data matches fixture."""
+        
+        with open(fixture_path) as f:
+            self.assertListEqual(
+                json.loads(f.read()), json.loads(self.export_data())
+            )

src/cities_light/tests/fixtures/update/noinsert.json

@@ -29,6 +29,21 @@
     "model": "cities_light.region",
     "pk": 1
 },
+{
+    "fields": {
+        "alternate_names": "Юргинский район",
+        "country": [2017370],
+        "display_name": "Yurginskiy Rayon, Russia",
+        "geoname_code": "1485714",
+        "geoname_id": 1485714,
+        "name": "Yurginskiy Rayon",
+        "name_ascii": "Yurginskiy Rayon",
+        "region": [1503900],
+        "slug": "yurginskiy-rayon"
+    },
+    "model": "cities_light.subregion",
+    "pk": 1
+},
 {
     "fields": {
         "alternate_names": "\u041a\u0435\u043c\u0435\u0440\u043e\u0432\u043e",

src/cities_light/tests/test_fixtures.py

@@ -7,7 +7,7 @@
 from django.core.management import call_command
 from django.core.management.base import CommandError
 
-from dbdiff.fixture import Fixture
+# from dbdiff.fixture import Fixture
 from cities_light.settings import DATA_DIR, FIXTURES_BASE_URL
 from cities_light.management.commands.cities_light_fixtures import Command
 from cities_light.downloader import Downloader

src/cities_light/tests/test_import.py

@@ -1,7 +1,9 @@
 import glob
 import os
 
-from dbdiff.fixture import Fixture
+from django.core import management
+from django.core.management.commands import dumpdata
+
 from .base import TestImportBase, FixtureDir
 from ..settings import DATA_DIR
 
@@ -20,7 +22,8 @@ def test_single_city(self):
             'angouleme_city',
             'angouleme_translations'
         )
-        Fixture(fixture_dir.get_file_path('angouleme.json')).assertNoDiff()
+
+        self.assertNoDiff(fixture_dir.get_file_path("angouleme.json"))
 
     def test_single_city_zip(self):
         """Load single city."""
@@ -38,7 +41,7 @@ def test_single_city_zip(self):
             'angouleme_translations',
             file_type="zip"
         )
-        Fixture(FixtureDir('import').get_file_path('angouleme.json')).assertNoDiff()
+        self.assertNoDiff(FixtureDir('import').get_file_path("angouleme.json"))
 
     def test_city_wrong_timezone(self):
         """Load single city with wrong timezone."""
@@ -51,7 +54,8 @@ def test_city_wrong_timezone(self):
             'angouleme_city_wtz',
             'angouleme_translations'
         )
-        Fixture(fixture_dir.get_file_path('angouleme_wtz.json')).assertNoDiff()
+
+        self.assertNoDiff(FixtureDir('import').get_file_path("angouleme_wtz.json"))
 
         from ..loading import get_cities_model
         city_model = get_cities_model('City')

src/cities_light/tests/test_update.py

@@ -1,7 +1,6 @@
 """Tests for update records."""
 import unittest
 
-from dbdiff.fixture import Fixture
 from .base import TestImportBase, FixtureDir
 
 
@@ -30,9 +29,9 @@ def test_update_fields(self):
             'update_translations',
         )
 
-        Fixture(
+        self.assertNoDiff(
             fixture_dir.get_file_path('update_fields.json')
-        ).assertNoDiff()
+        )
 
     def test_update_fields_wrong_timezone(self):
         """Test all fields are updated, but timezone field is wrong."""
@@ -56,9 +55,9 @@ def test_update_fields_wrong_timezone(self):
             'update_translations',
         )
 
-        Fixture(
+        self.assertNoDiff(
             fixture_dir.get_file_path('update_fields_wtz.json')
-        ).assertNoDiff()
+        )
 
     def test_change_country(self):
         """Test change country for region/city."""
@@ -82,9 +81,9 @@ def test_change_country(self):
             'update_translations',
         )
 
-        Fixture(
+        self.assertNoDiff(
             fixture_dir.get_file_path('change_country.json')
-        ).assertNoDiff()
+        )
 
     def test_change_region_and_country(self):
         """Test change region and country."""
@@ -108,9 +107,9 @@ def test_change_region_and_country(self):
             'update_translations',
         )
 
-        Fixture(
+        self.assertNoDiff(
             fixture_dir.get_file_path('change_region_and_country.json')
-        ).assertNoDiff()
+        )
 
     def test_keep_slugs(self):
         """Test --keep-slugs option."""
@@ -135,9 +134,9 @@ def test_keep_slugs(self):
             keep_slugs=True
         )
 
-        Fixture(
+        self.assertNoDiff(
             fixture_dir.get_file_path('keep_slugs.json'),
-        ).assertNoDiff()
+        )
 
     def test_add_records(self):
         """Test that new records are added."""
@@ -161,9 +160,9 @@ def test_add_records(self):
             'add_translations'
         )
 
-        Fixture(
+        self.assertNoDiff(
             fixture_dir.get_file_path('add_records.json')
-        ).assertNoDiff()
+        )
 
     def test_noinsert(self):
         """Test --noinsert option."""
@@ -188,9 +187,9 @@ def test_noinsert(self):
             noinsert=True
         )
 
-        Fixture(
+        self.assertNoDiff(
             fixture_dir.get_file_path('noinsert.json'),
-        ).assertNoDiff()
+        )
 
     # TODO: make the test pass
     @unittest.skip("Obsolete records are not removed yet.")