Import module metadata in reposync

python/spacewalk/satellite_tools/appstreams.py

@@ -64,6 +64,12 @@ def __repr__(self):
         return f"{self.name}:{self.stream}:{self.version}:{self.context}:{self.arch}"
 
 
+class ModuleMdIndexingError(Exception):
+    """Exception raised when indexing module metadata fails."""
+
+    pass
+
+
 class ModuleMdImporter:
     """
     Imports a channel's AppStream modules from its Modulemd file to the database.
@@ -89,7 +95,7 @@ def validate(self):
         log(2, "  Validating module metadata file.")
         self._index_modulemd()
         if not self.modulemd_index.get_module_names():
-            raise IOError("Error reading module metadata.")
+            raise ModuleMdIndexingError("No module data exists in the metadata file.")
 
     def import_module_metadata(self):
         """Imports modules from the Modulemd file to the database."""
@@ -141,7 +147,12 @@ def import_module_metadata(self):
     def _index_modulemd(self):
         """Indexes the Modulemd file."""
         idx = Modulemd.ModuleIndex.new()
-        idx.update_from_file(self.modulemd_file, False)
+        try:
+            idx.update_from_file(self.modulemd_file, False)
+        except gi.repository.GLib.GError as e:
+            raise ModuleMdIndexingError(
+                f"An error occurred while indexing the module metadata from file '{self.modulemd_file}'."
+            ) from e
         self.modulemd_index = idx
 
     def _get_modules(self):

python/spacewalk/satellite_tools/reposync.py

@@ -77,6 +77,7 @@
     dumpEMAIL_LOG,
     log2background,
 )
+from spacewalk.satellite_tools.appstreams import ModuleMdImporter, ModuleMdIndexingError
 
 translation = gettext.translation("spacewalk-backend-server", fallback=True)
 _ = translation.gettext
@@ -751,31 +752,56 @@ def sync(self, update_repodata=True):
                         # this fetch also the normal xml primary file
                         repo_checksum_type = plugin.get_md_checksum_type()
 
+                        modulemd_importer = None
                         if not self.no_packages:
-                            self.import_groups(plugin)
                             if repo_type == "yum":
-                                self.import_modules(plugin)
-                            ret = self.import_packages(
-                                plugin, data["id"], url, is_non_local_repo
-                            )
-                            failed_packages += ret
+                                modulemd_path = self.import_modules(plugin)
+                                if modulemd_path:
+                                    modulemd_importer = ModuleMdImporter(
+                                        self.channel["id"], modulemd_path
+                                    )
+                                    try:
+                                        modulemd_importer.validate()
+                                    except ModuleMdIndexingError as e:
+                                        log(
+                                            0,
+                                            f"An error occurred while reading module metadata: {e}",
+                                        )
+                                        self.sendErrorMail(str(e))
+                                        sync_error = -1
+
+                        if sync_error == 0:
+                            self.import_groups(plugin)
 
-                        if not self.no_errata:
-                            self.import_updates(plugin)
+                            if not self.no_packages:
+                                ret = self.import_packages(
+                                    plugin, data["id"], url, is_non_local_repo
+                                )
+                                failed_packages += ret
 
-                        self.import_mediaproducts(plugin)
+                            if not self.no_errata:
+                                self.import_updates(plugin)
 
-                        # only for repos obtained from the DB
-                        if self.sync_kickstart and data["repo_label"]:
-                            try:
-                                self.import_kickstart(
-                                    plugin, data["repo_label"], is_non_local_repo
-                                )
-                            except:
-                                rhnSQL.rollback()
-                                raise
-                        self.import_products(plugin)
-                        self.import_susedata(plugin)
+                            if modulemd_importer:
+                                try:
+                                    modulemd_importer.import_module_metadata()
+                                except:
+                                    rhnSQL.rollback()
+                                    raise
+
+                            self.import_mediaproducts(plugin)
+
+                            # only for repos obtained from the DB
+                            if self.sync_kickstart and data["repo_label"]:
+                                try:
+                                    self.import_kickstart(
+                                        plugin, data["repo_label"], is_non_local_repo
+                                    )
+                                except:
+                                    rhnSQL.rollback()
+                                    raise
+                            self.import_products(plugin)
+                            self.import_susedata(plugin)
 
                 except rhnSQL.SQLError as e:
                     # pylint: disable-next=consider-using-f-string
@@ -1128,7 +1154,9 @@ def import_groups(self, plug):
     def import_modules(self, plug):
         modulesfile = plug.get_modules()
         if modulesfile:
-            self.copy_metadata_file(plug, modulesfile, "modules", relative_modules_dir)
+            return self.copy_metadata_file(
+                plug, modulesfile, "modules", relative_modules_dir
+            )
 
     def import_mediaproducts(self, plug):
         mediaproducts = plug.get_mediaproducts()

python/spacewalk/spacewalk-backend.changes.cbbayburt.appstreams-reposync

@@ -0,0 +1 @@
+- Import module metadata during reposync

python/test/unit/spacewalk/satellite_tools/test_appstreams.py

@@ -1,10 +1,16 @@
 """Unit tests for AppStreams reposync module."""
+
 import os
 import pytest
 from mock import Mock
 
 import spacewalk.satellite_tools.appstreams
-from spacewalk.satellite_tools.appstreams import ModuleMdImporter, Nsvca, Nevra
+from spacewalk.satellite_tools.appstreams import (
+    ModuleMdImporter,
+    ModuleMdIndexingError,
+    Nsvca,
+    Nevra,
+)
 
 import gi
 
@@ -67,10 +73,10 @@ def test_validate(importer):
     importer.validate()
 
     importer.modulemd_file = "INVALID_FILE"
-    with pytest.raises(gi.repository.GLib.GError) as exc:
+    with pytest.raises(ModuleMdIndexingError) as exc:
         importer.validate()
 
-    assert exc.value.domain == "modulemd-yaml-error-quark"
+    assert exc.value.__cause__.domain == "modulemd-yaml-error-quark"
 
 
 # pylint: disable=protected-access

python/test/unit/spacewalk/satellite_tools/test_reposync.py

@@ -89,6 +89,11 @@ def setUp(self):
             ],
         )
 
+        spacewalk.satellite_tools.appstreams.ModuleMdImporter.validate = Mock()
+        spacewalk.satellite_tools.appstreams.ModuleMdImporter.import_module_metadata = (
+            Mock()
+        )
+
     def tearDown(self):
         self.stdout.close()
         sys.stdout = self.saved_stdout
@@ -97,6 +102,7 @@ def tearDown(self):
         sys.stderr = self.saved_stderr
 
         imp.reload(spacewalk.satellite_tools.reposync)
+        imp.reload(spacewalk.satellite_tools.appstreams)
 
     def test_init_succeeds_with_correct_attributes(self):
         rs = _init_reposync(self.reposync, "Label", RTYPE)
@@ -216,6 +222,7 @@ def test_sync_success_no_regen(self):
 
         _mock_rhnsql(self.reposync, [None, []])
         rs = _mock_sync(self.reposync, rs)
+
         with patch("uyuni.common.context_managers.CFG", CFG):
             rs.sync()
 
@@ -946,10 +953,17 @@ def setUp(self):
             import_products=Mock(),
             import_susedata=Mock(),
         )
+        appstreams_patcher = patch.multiple(
+            "spacewalk.satellite_tools.appstreams.ModuleMdImporter",
+            validate=Mock(),
+            import_module_metadata=Mock(),
+        )
         module_patcher.start()
         class_patcher.start()
+        appstreams_patcher.start()
         self.addCleanup(module_patcher.stop)
         self.addCleanup(class_patcher.stop)
+        self.addCleanup(appstreams_patcher.stop)
 
     @patch("uyuni.common.context_managers.initCFG", Mock())
     def test_pass_multiple_urls_params(self):