Merge branch 'ansible:master' into increase-ansible-lint

.github/workflows/main.yml

@@ -70,6 +70,6 @@ jobs:
         python -m build --sdist --wheel --outdir dist/ .
 
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.pypi_token }}

CHANGES.rst

@@ -1,3 +1,19 @@
+galaxy-importer 0.4.18 (2024-01-18)
+===================================
+
+Features
+--------
+
+- enable docs rendering for filter and test plugins (`AAH-2854 <https://issues.redhat.com/browse/AAH-2854>`_)
+- Add command to convert markdown to html (`AAH-3009 <https://issues.redhat.com/browse/AAH-3009>`_)
+
+
+Bugfixes
+--------
+
+- Ignore certain linter rules for ruff and pylint inside the EDA tox tests. (`AAH-2997 <https://issues.redhat.com/browse/AAH-2997>`_)
+
+
 galaxy-importer 0.4.18 (2023-12-06)
 ===================================
 

galaxy_importer/__init__.py

@@ -1 +1 @@
-__version__ = "0.4.18"
+__version__ = "0.4.19"

galaxy_importer/constants.py

@@ -24,12 +24,14 @@
     "callback",
     "cliconf",
     "connection",
+    "filter",
     "httpapi",
     "inventory",
     "lookup",
     "shell",
     "module",
     "strategy",
+    "test",
     "vars",
 ]
 ANSIBLE_DOC_PLUGIN_MAP = {"module": "modules"}

galaxy_importer/loaders/doc_string.py

@@ -47,9 +47,9 @@ def load(self):
             return docs
 
         for plugin_type in constants.ANSIBLE_DOC_SUPPORTED_TYPES:
-            plugin_dir_name = constants.ANSIBLE_DOC_PLUGIN_MAP.get(plugin_type, plugin_type)
-
-            plugins = self._get_plugins(os.path.join(self.path, "plugins", plugin_dir_name))
+            # use ansible-doc to list all the plugins of this type
+            found_plugins = self._run_ansible_doc_list(plugin_type)
+            plugins = sorted(list(found_plugins.keys()))
 
             if not plugins:
                 continue
@@ -60,6 +60,20 @@ def load(self):
 
         return docs
 
+    @property
+    def _collections_path(self):
+        return "/".join(self.path.split("/")[:-3])
+
+    @property
+    def _base_ansible_doc_cmd(self):
+        return [
+            "/usr/bin/env",
+            f"ANSIBLE_COLLECTIONS_PATHS={self._collections_path}",
+            f"ANSIBLE_COLLECTIONS_PATH={self._collections_path}",
+            f"ANSIBLE_LOCAL_TEMP={self.cfg.ansible_local_tmp}",
+            "ansible-doc",
+        ]
+
     def _get_plugins(self, plugin_dir):
         """Get list of fully qualified plugin names inside directory.
 
@@ -79,21 +93,42 @@ def _get_plugins(self, plugin_dir):
                 fq_name_parts.append(os.path.basename(file_path)[:-3])
 
                 plugins.append(".".join(fq_name_parts))
+
         return plugins
 
-    def _run_ansible_doc(self, plugin_type, plugins):
-        collections_path = "/".join(self.path.split("/")[:-3])
-        cmd = [
-            "/usr/bin/env",
-            f"ANSIBLE_COLLECTIONS_PATHS={collections_path}",
-            f"ANSIBLE_LOCAL_TEMP={self.cfg.ansible_local_tmp}",
-            "ansible-doc",
+    def _run_ansible_doc_list(self, plugin_type):
+        """Use ansible-doc to get a list of plugins for the collection by type."""
+        cmd = self._base_ansible_doc_cmd + [
+            "--list",
             "--type",
             plugin_type,
             "--json",
-        ] + plugins
+            self.fq_collection_name,
+        ]
+        self.log.debug("CMD: {}".format(" ".join(cmd)))
+        proc = Popen(cmd, cwd=self._collections_path, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = proc.communicate()
+        if proc.returncode != 0:
+            self.log.error(
+                'Error running ansible-doc: cmd="{cmd}" returncode="{rc}" {err}'.format(
+                    cmd=" ".join(cmd), rc=proc.returncode, err=stderr
+                )
+            )
+            return {}
+        return json.loads(stdout)
+
+    def _run_ansible_doc(self, plugin_type, plugins):
+        cmd = (
+            self._base_ansible_doc_cmd
+            + [
+                "--type",
+                plugin_type,
+                "--json",
+            ]
+            + plugins
+        )
         self.log.debug("CMD: {}".format(" ".join(cmd)))
-        proc = Popen(cmd, cwd=collections_path, stdout=PIPE, stderr=PIPE)
+        proc = Popen(cmd, cwd=self._collections_path, stdout=PIPE, stderr=PIPE)
         stdout, stderr = proc.communicate()
         if proc.returncode != 0:
             self.log.error(

tests/integration/conftest.py

@@ -38,6 +38,28 @@ def local_config():
     return {"GALAXY_IMPORTER_CONFIG": config_path}
 
 
+@pytest.fixture
+def local_fast_config():
+    """Disable the slow stuff."""
+    config = [
+        "[galaxy-importer]",
+        "RUN_ANSIBLE_TEST=False",
+        "RUN_ANSIBLE_LINT=False",
+        "ANSIBLE_TEST_LOCAL_IMAGE=False",
+        "LOCAL_IMAGE_DOCKER=False",
+    ]
+    config = "\n".join(config)
+
+    tdir = tempfile.mkdtemp()
+    atexit.register(clean_files, tdir)
+
+    config_path = os.path.join(tdir, "galaxy-importer.cfg")
+    with open(config_path, "w") as f:
+        f.write(config)
+
+    return {"GALAXY_IMPORTER_CONFIG": config_path}
+
+
 @pytest.fixture
 def local_image_config():
     config = [

tests/integration/test_collections.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+
+import copy
+import json
+import os
+import subprocess
+
+
+def test_collection_community_general_import(workdir, local_fast_config):
+    assert os.path.exists(workdir)
+    url = (
+        "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/"
+        + "collections/artifacts/community-general-8.2.0.tar.gz"
+    )
+    dst = os.path.join(workdir, os.path.basename(url))
+    pid = subprocess.run(f"curl -L -o {dst} {url}", shell=True)
+    assert pid.returncode == 0
+    assert os.path.exists(dst)
+
+    env = copy.deepcopy(dict(os.environ))
+    env.update(local_fast_config)
+
+    cmd = f"python3 -m galaxy_importer.main {dst}"
+    pid = subprocess.run(
+        cmd, shell=True, cwd=workdir, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    assert pid.returncode == 0, pid.stdout
+
+    # the log should contain all the relevant messages
+    log = pid.stdout.decode("utf-8")
+
+    # should have no errors
+    assert "error" not in log.lower()
+
+    # should have skipped ansible-test
+    assert "skip ansible-test sanity test" in log
+
+    # check for success message
+    assert "Importer processing completed successfully" in log
+
+    # it should have stored structured data in the pwd
+    results_file = os.path.join(workdir, "importer_result.json")
+    assert os.path.exists(results_file)
+    with open(results_file, "r") as f:
+        results = json.loads(f.read())
+
+    # the data should have all the relevant bits
+    assert results["metadata"]["namespace"] == "community"
+    assert results["metadata"]["name"] == "general"
+    assert results["metadata"]["version"] == "8.2.0"
+    assert results["requires_ansible"] == ">=2.13.0"
+
+    # make sure it found all the files
+    contents = dict(((x["content_type"], x["name"]), x) for x in results["contents"])
+    assert len(contents.keys()) == 831
+
+    # check a small sample
+    assert ("test", "a_module") in contents
+    assert ("module_utils", "version") in contents
+    assert ("module", "xfconf") in contents
+    assert ("module", "jabber") in contents
+    assert ("filter", "time") in contents
+    assert ("doc_fragments", "nomad") in contents
+    assert ("connection", "lxc") in contents
+    assert ("callback", "yaml") in contents
+    assert ("cache", "yaml") in contents
+    assert ("become", "pbrun") in contents
+    assert ("action", "shutdown") in contents
+
+    # make sure it found all the docs
+    docs_contents = dict(
+        ((x["content_type"], x["content_name"]), x) for x in results["docs_blob"]["contents"]
+    )
+    assert len(docs_contents.keys()) == 831
+
+    # check a small sample
+    assert ("test", "a_module") in docs_contents
+    assert ("module_utils", "version") in docs_contents
+    assert ("module", "xfconf") in docs_contents
+    assert ("module", "jabber") in docs_contents
+    assert ("filter", "time") in docs_contents
+    assert ("doc_fragments", "nomad") in docs_contents
+    assert ("connection", "lxc") in docs_contents
+    assert ("callback", "yaml") in docs_contents
+    assert ("cache", "yaml") in docs_contents
+    assert ("become", "pbrun") in docs_contents
+    assert ("action", "shutdown") in docs_contents

tests/unit/test_loader_collection.py

@@ -204,7 +204,11 @@ def test_manifest_success(_build_docs_blob, populated_collection_root):
     data = CollectionLoader(
         populated_collection_root,
         filename,
-        cfg=SimpleNamespace(run_ansible_doc=True, run_ansible_lint=False),
+        cfg=SimpleNamespace(
+            run_ansible_doc=True,
+            run_ansible_lint=False,
+            ansible_local_tmp=populated_collection_root,
+        ),
     ).load()
     assert data.metadata.namespace == "my_namespace"
     assert data.metadata.name == "my_collection"
@@ -375,7 +379,11 @@ def test_filename_empty_value(_build_docs_blob, populated_collection_root):
     data = CollectionLoader(
         populated_collection_root,
         filename,
-        cfg=SimpleNamespace(run_ansible_doc=True, run_ansible_lint=False),
+        cfg=SimpleNamespace(
+            run_ansible_doc=True,
+            run_ansible_lint=False,
+            ansible_local_tmp=populated_collection_root,
+        ),
     ).load()
     assert data.metadata.namespace == "my_namespace"
     assert data.metadata.name == "my_collection"
@@ -390,7 +398,11 @@ def test_filename_none(_build_docs_blob, populated_collection_root):
     data = CollectionLoader(
         populated_collection_root,
         filename,
-        cfg=SimpleNamespace(run_ansible_doc=True, run_ansible_lint=False),
+        cfg=SimpleNamespace(
+            run_ansible_doc=True,
+            run_ansible_lint=False,
+            ansible_local_tmp=populated_collection_root,
+        ),
     ).load()
     assert data.metadata.namespace == "my_namespace"
     assert data.metadata.name == "my_collection"
@@ -413,7 +425,11 @@ def test_license_file(populated_collection_root):
     data = CollectionLoader(
         populated_collection_root,
         filename=None,
-        cfg=SimpleNamespace(run_ansible_doc=True, run_ansible_lint=False),
+        cfg=SimpleNamespace(
+            run_ansible_doc=True,
+            run_ansible_lint=False,
+            ansible_local_tmp=populated_collection_root,
+        ),
     ).load()
     assert data.metadata.license_file == "LICENSE"
 
@@ -453,7 +469,11 @@ def test_changelog_fail(_build_docs_blob, populated_collection_root, caplog):
     CollectionLoader(
         populated_collection_root,
         filename=None,
-        cfg=SimpleNamespace(run_ansible_doc=True, run_ansible_lint=False),
+        cfg=SimpleNamespace(
+            run_ansible_doc=True,
+            run_ansible_lint=False,
+            ansible_local_tmp=populated_collection_root,
+        ),
     ).load()
     assert (
         "No changelog found. "