From b4cf9d97893383a731b5170c9e44c5fe0c09baa9 Mon Sep 17 00:00:00 2001 From: jctanner Date: Thu, 18 Jan 2024 12:23:08 -0500 Subject: [PATCH 1/5] Enable filter and test plugins doc parsing. (#257) No-Issue Signed-off-by: James Tanner --- galaxy_importer/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/galaxy_importer/constants.py b/galaxy_importer/constants.py index 6beb8953..3d3120f2 100644 --- a/galaxy_importer/constants.py +++ b/galaxy_importer/constants.py @@ -24,12 +24,14 @@ "callback", "cliconf", "connection", + "filter", "httpapi", "inventory", "lookup", "shell", "module", "strategy", + "test", "vars", ] ANSIBLE_DOC_PLUGIN_MAP = {"module": "modules"} From aee9b9e521019036fb461cce5ae875a6de429eb2 Mon Sep 17 00:00:00 2001 From: jctanner Date: Thu, 18 Jan 2024 12:38:46 -0500 Subject: [PATCH 2/5] Release 0.4.19 (#258) Release 0.4.19 No-Issue Signed-off-by: James Tanner --- CHANGES.rst | 16 ++++++++++++++++ CHANGES/2997.bugfix | 1 - CHANGES/3009.feature | 1 - 3 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 CHANGES/2997.bugfix delete mode 100644 CHANGES/3009.feature diff --git a/CHANGES.rst b/CHANGES.rst index bca33735..f5f02aeb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,19 @@ +galaxy-importer 0.4.18 (2024-01-18) +=================================== + +Features +-------- + +- enable docs rendering for filter and test plugins (`AAH-2854 `_) +- Add command to convert markdown to html (`AAH-3009 `_) + + +Bugfixes +-------- + +- Ignore certain linter rules for ruff and pylint inside the EDA tox tests. (`AAH-2997 `_) + + galaxy-importer 0.4.18 (2023-12-06) =================================== diff --git a/CHANGES/2997.bugfix b/CHANGES/2997.bugfix deleted file mode 100644 index ddabdf1a..00000000 --- a/CHANGES/2997.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ignore certain linter rules for ruff and pylint inside the EDA tox tests. \ No newline at end of file diff --git a/CHANGES/3009.feature b/CHANGES/3009.feature deleted file mode 100644 index e5403c6f..00000000 --- a/CHANGES/3009.feature +++ /dev/null @@ -1 +0,0 @@ -Add command to convert markdown to html \ No newline at end of file From 8c45d9c7e55b362c8f901b4907d2dd4b1e94c873 Mon Sep 17 00:00:00 2001 From: jctanner Date: Thu, 18 Jan 2024 13:34:46 -0500 Subject: [PATCH 3/5] Pin the pypi action to a specific version. (#259) Using the master branch is deprecated and it now throws an error saying use a released version. No-Issue Signed-off-by: James Tanner --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8afa0b78..17f84c03 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,6 +70,6 @@ jobs: python -m build --sdist --wheel --outdir dist/ . - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_token }} From 084e138fc6394b20c0b469010cef6abe64417b32 Mon Sep 17 00:00:00 2001 From: jctanner Date: Thu, 18 Jan 2024 14:49:54 -0500 Subject: [PATCH 4/5] Fix the version. (#260) No-Issue Signed-off-by: James Tanner --- galaxy_importer/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy_importer/__init__.py b/galaxy_importer/__init__.py index 2dd55360..8f584e62 100644 --- a/galaxy_importer/__init__.py +++ b/galaxy_importer/__init__.py @@ -1 +1 @@ -__version__ = "0.4.18" +__version__ = "0.4.19" From d4b5e6d12088ba452f129f4824bd049be5543358 Mon Sep 17 00:00:00 2001 From: jctanner Date: Mon, 22 Jan 2024 15:04:10 -0500 Subject: [PATCH 5/5] Use ansible-doc to list the plugins by type instead of crawling the directory (#261) The old finder code didn't find the functions inside test or filter plugins. No-Issue Signed-off-by: James Tanner --- galaxy_importer/loaders/doc_string.py | 59 ++++++++++++++---- tests/integration/conftest.py | 22 +++++++ tests/integration/test_collections.py | 87 +++++++++++++++++++++++++++ tests/unit/test_loader_collection.py | 30 +++++++-- tests/unit/test_loader_doc_string.py | 56 ++++++++++------- 5 files changed, 215 insertions(+), 39 deletions(-) create mode 100644 tests/integration/test_collections.py diff --git a/galaxy_importer/loaders/doc_string.py b/galaxy_importer/loaders/doc_string.py index ebb38e88..facc6b19 100644 --- a/galaxy_importer/loaders/doc_string.py +++ b/galaxy_importer/loaders/doc_string.py @@ -47,9 +47,9 @@ def load(self): return docs for plugin_type in constants.ANSIBLE_DOC_SUPPORTED_TYPES: - plugin_dir_name = constants.ANSIBLE_DOC_PLUGIN_MAP.get(plugin_type, plugin_type) - - plugins = self._get_plugins(os.path.join(self.path, "plugins", plugin_dir_name)) + # use ansible-doc to list all the plugins of this type + found_plugins = self._run_ansible_doc_list(plugin_type) + plugins = sorted(list(found_plugins.keys())) if not plugins: continue @@ -60,6 +60,20 @@ def load(self): return docs + @property + def _collections_path(self): + return "/".join(self.path.split("/")[:-3]) + + @property + def _base_ansible_doc_cmd(self): + return [ + "/usr/bin/env", + f"ANSIBLE_COLLECTIONS_PATHS={self._collections_path}", + f"ANSIBLE_COLLECTIONS_PATH={self._collections_path}", + f"ANSIBLE_LOCAL_TEMP={self.cfg.ansible_local_tmp}", + "ansible-doc", + ] + def _get_plugins(self, plugin_dir): """Get list of fully qualified plugin names inside directory. @@ -79,21 +93,42 @@ def _get_plugins(self, plugin_dir): fq_name_parts.append(os.path.basename(file_path)[:-3]) plugins.append(".".join(fq_name_parts)) + return plugins - def _run_ansible_doc(self, plugin_type, plugins): - collections_path = "/".join(self.path.split("/")[:-3]) - cmd = [ - "/usr/bin/env", - f"ANSIBLE_COLLECTIONS_PATHS={collections_path}", - f"ANSIBLE_LOCAL_TEMP={self.cfg.ansible_local_tmp}", - "ansible-doc", + def _run_ansible_doc_list(self, plugin_type): + """Use ansible-doc to get a list of plugins for the collection by type.""" + cmd = self._base_ansible_doc_cmd + [ + "--list", "--type", plugin_type, "--json", - ] + plugins + self.fq_collection_name, + ] + self.log.debug("CMD: {}".format(" ".join(cmd))) + proc = Popen(cmd, cwd=self._collections_path, stdout=PIPE, stderr=PIPE) + stdout, stderr = proc.communicate() + if proc.returncode != 0: + self.log.error( + 'Error running ansible-doc: cmd="{cmd}" returncode="{rc}" {err}'.format( + cmd=" ".join(cmd), rc=proc.returncode, err=stderr + ) + ) + return {} + return json.loads(stdout) + + def _run_ansible_doc(self, plugin_type, plugins): + cmd = ( + self._base_ansible_doc_cmd + + [ + "--type", + plugin_type, + "--json", + ] + + plugins + ) self.log.debug("CMD: {}".format(" ".join(cmd))) - proc = Popen(cmd, cwd=collections_path, stdout=PIPE, stderr=PIPE) + proc = Popen(cmd, cwd=self._collections_path, stdout=PIPE, stderr=PIPE) stdout, stderr = proc.communicate() if proc.returncode != 0: self.log.error( diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 65a1c63f..246d06ca 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -38,6 +38,28 @@ def local_config(): return {"GALAXY_IMPORTER_CONFIG": config_path} +@pytest.fixture +def local_fast_config(): + """Disable the slow stuff.""" + config = [ + "[galaxy-importer]", + "RUN_ANSIBLE_TEST=False", + "RUN_ANSIBLE_LINT=False", + "ANSIBLE_TEST_LOCAL_IMAGE=False", + "LOCAL_IMAGE_DOCKER=False", + ] + config = "\n".join(config) + + tdir = tempfile.mkdtemp() + atexit.register(clean_files, tdir) + + config_path = os.path.join(tdir, "galaxy-importer.cfg") + with open(config_path, "w") as f: + f.write(config) + + return {"GALAXY_IMPORTER_CONFIG": config_path} + + @pytest.fixture def local_image_config(): config = [ diff --git a/tests/integration/test_collections.py b/tests/integration/test_collections.py new file mode 100644 index 00000000..623407b4 --- /dev/null +++ b/tests/integration/test_collections.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +import copy +import json +import os +import subprocess + + +def test_collection_community_general_import(workdir, local_fast_config): + assert os.path.exists(workdir) + url = ( + "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/" + + "collections/artifacts/community-general-8.2.0.tar.gz" + ) + dst = os.path.join(workdir, os.path.basename(url)) + pid = subprocess.run(f"curl -L -o {dst} {url}", shell=True) + assert pid.returncode == 0 + assert os.path.exists(dst) + + env = copy.deepcopy(dict(os.environ)) + env.update(local_fast_config) + + cmd = f"python3 -m galaxy_importer.main {dst}" + pid = subprocess.run( + cmd, shell=True, cwd=workdir, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + assert pid.returncode == 0, pid.stdout + + # the log should contain all the relevant messages + log = pid.stdout.decode("utf-8") + + # should have no errors + assert "error" not in log.lower() + + # should have skipped ansible-test + assert "skip ansible-test sanity test" in log + + # check for success message + assert "Importer processing completed successfully" in log + + # it should have stored structured data in the pwd + results_file = os.path.join(workdir, "importer_result.json") + assert os.path.exists(results_file) + with open(results_file, "r") as f: + results = json.loads(f.read()) + + # the data should have all the relevant bits + assert results["metadata"]["namespace"] == "community" + assert results["metadata"]["name"] == "general" + assert results["metadata"]["version"] == "8.2.0" + assert results["requires_ansible"] == ">=2.13.0" + + # make sure it found all the files + contents = dict(((x["content_type"], x["name"]), x) for x in results["contents"]) + assert len(contents.keys()) == 831 + + # check a small sample + assert ("test", "a_module") in contents + assert ("module_utils", "version") in contents + assert ("module", "xfconf") in contents + assert ("module", "jabber") in contents + assert ("filter", "time") in contents + assert ("doc_fragments", "nomad") in contents + assert ("connection", "lxc") in contents + assert ("callback", "yaml") in contents + assert ("cache", "yaml") in contents + assert ("become", "pbrun") in contents + assert ("action", "shutdown") in contents + + # make sure it found all the docs + docs_contents = dict( + ((x["content_type"], x["content_name"]), x) for x in results["docs_blob"]["contents"] + ) + assert len(docs_contents.keys()) == 831 + + # check a small sample + assert ("test", "a_module") in docs_contents + assert ("module_utils", "version") in docs_contents + assert ("module", "xfconf") in docs_contents + assert ("module", "jabber") in docs_contents + assert ("filter", "time") in docs_contents + assert ("doc_fragments", "nomad") in docs_contents + assert ("connection", "lxc") in docs_contents + assert ("callback", "yaml") in docs_contents + assert ("cache", "yaml") in docs_contents + assert ("become", "pbrun") in docs_contents + assert ("action", "shutdown") in docs_contents diff --git a/tests/unit/test_loader_collection.py b/tests/unit/test_loader_collection.py index 961fbbb1..fd6feace 100644 --- a/tests/unit/test_loader_collection.py +++ b/tests/unit/test_loader_collection.py @@ -204,7 +204,11 @@ def test_manifest_success(_build_docs_blob, populated_collection_root): data = CollectionLoader( populated_collection_root, filename, - cfg=SimpleNamespace(run_ansible_doc=True, run_ansible_lint=False), + cfg=SimpleNamespace( + run_ansible_doc=True, + run_ansible_lint=False, + ansible_local_tmp=populated_collection_root, + ), ).load() assert data.metadata.namespace == "my_namespace" assert data.metadata.name == "my_collection" @@ -375,7 +379,11 @@ def test_filename_empty_value(_build_docs_blob, populated_collection_root): data = CollectionLoader( populated_collection_root, filename, - cfg=SimpleNamespace(run_ansible_doc=True, run_ansible_lint=False), + cfg=SimpleNamespace( + run_ansible_doc=True, + run_ansible_lint=False, + ansible_local_tmp=populated_collection_root, + ), ).load() assert data.metadata.namespace == "my_namespace" assert data.metadata.name == "my_collection" @@ -390,7 +398,11 @@ def test_filename_none(_build_docs_blob, populated_collection_root): data = CollectionLoader( populated_collection_root, filename, - cfg=SimpleNamespace(run_ansible_doc=True, run_ansible_lint=False), + cfg=SimpleNamespace( + run_ansible_doc=True, + run_ansible_lint=False, + ansible_local_tmp=populated_collection_root, + ), ).load() assert data.metadata.namespace == "my_namespace" assert data.metadata.name == "my_collection" @@ -413,7 +425,11 @@ def test_license_file(populated_collection_root): data = CollectionLoader( populated_collection_root, filename=None, - cfg=SimpleNamespace(run_ansible_doc=True, run_ansible_lint=False), + cfg=SimpleNamespace( + run_ansible_doc=True, + run_ansible_lint=False, + ansible_local_tmp=populated_collection_root, + ), ).load() assert data.metadata.license_file == "LICENSE" @@ -453,7 +469,11 @@ def test_changelog_fail(_build_docs_blob, populated_collection_root, caplog): CollectionLoader( populated_collection_root, filename=None, - cfg=SimpleNamespace(run_ansible_doc=True, run_ansible_lint=False), + cfg=SimpleNamespace( + run_ansible_doc=True, + run_ansible_lint=False, + ansible_local_tmp=populated_collection_root, + ), ).load() assert ( "No changelog found. " diff --git a/tests/unit/test_loader_doc_string.py b/tests/unit/test_loader_doc_string.py index 42902f55..3aca0b61 100644 --- a/tests/unit/test_loader_doc_string.py +++ b/tests/unit/test_loader_doc_string.py @@ -24,6 +24,23 @@ from galaxy_importer import loaders +ANSIBLE_DOC_OUTPUT = json.loads(""" + { + "my_module": { + "return": { + "message": { + "description": ["The output message the sample module generates"] + }, + "original_message": { + "description": ["The original name param that was passed in"], + "type": "str" + } + } + } + } +""") + + @pytest.fixture def doc_string_loader(): cfg = config.Config(config_data=config.ConfigFile.load()) @@ -71,9 +88,12 @@ def test_run_ansible_doc_exception(mocked_popen, doc_string_loader): assert not res -@mock.patch.object(loaders.DocStringLoader, "_run_ansible_doc") -def test_ansible_doc_no_output(mocked_run_ansible_doc, doc_string_loader): - mocked_run_ansible_doc.return_value = "" +@mock.patch("galaxy_importer.loaders.doc_string.constants.ANSIBLE_DOC_SUPPORTED_TYPES", ["module"]) +@mock.patch.object(loaders.DocStringLoader, "_run_ansible_doc_list", return_value={}) +@mock.patch.object(loaders.DocStringLoader, "_run_ansible_doc", return_value={}) +def test_ansible_doc_no_output( + mocked_run_ansible_doc_list, mocked_run_ansible_doc, doc_string_loader +): assert doc_string_loader.load() == {} @@ -344,24 +364,12 @@ def test_transform_doc_strings_nested_suboptions(doc_string_loader): ] -@mock.patch.object(loaders.DocStringLoader, "_run_ansible_doc") -def test_load(mocked_run_ansible_doc, doc_string_loader, tmpdir): - ansible_doc_output = """ - { - "my_module": { - "return": { - "message": { - "description": ["The output message the sample module generates"] - }, - "original_message": { - "description": ["The original name param that was passed in"], - "type": "str" - } - } - } - } - """ - mocked_run_ansible_doc.return_value = json.loads(ansible_doc_output) +@mock.patch("galaxy_importer.loaders.doc_string.constants.ANSIBLE_DOC_SUPPORTED_TYPES", ["module"]) +@mock.patch.object(loaders.DocStringLoader, "_run_ansible_doc_list", return_value={"my_module": {}}) +@mock.patch.object(loaders.DocStringLoader, "_run_ansible_doc", return_value=ANSIBLE_DOC_OUTPUT) +def test_load_function( + mocked_run_ansible_doc_list, mocked_run_ansible_doc, doc_string_loader, tmpdir +): doc_string_loader.path = str(tmpdir) tmpdir.mkdir("plugins").mkdir("modules").join("my_module.py").write("") @@ -386,8 +394,12 @@ def test_load(mocked_run_ansible_doc, doc_string_loader, tmpdir): } +@mock.patch( + "galaxy_importer.loaders.doc_string.constants.ANSIBLE_DOC_SUPPORTED_TYPES", ["inventory"] +) +@mock.patch.object(loaders.DocStringLoader, "_run_ansible_doc_list", return_value={"my_plugin": {}}) @mock.patch("galaxy_importer.loaders.doc_string.Popen") -def test_load_ansible_doc_error(mocked_popen, doc_string_loader, tmpdir): +def test_load_ansible_doc_error(mocked_popen, mocked_doc_list, doc_string_loader, tmpdir): mocked_popen.return_value.communicate.return_value = ( "output", "error that causes exception",