Merge pull request #226 from aboutcode-org/215-lock

Introduce lockfile for download integrity

Author: pombredanne

requirements-dev.txt

@@ -7,7 +7,7 @@ cryptography==37.0.2
 dataclasses==0.8
 docutils==0.18.1
 et-xmlfile==1.1.0
-execnet==1.9.0
+execnet==2.1.1
 importlib-resources==5.4.0
 iniconfig==1.1.1
 isort==5.10.1
@@ -21,14 +21,15 @@ openpyxl==3.0.10
 pathspec==0.9.0
 pkginfo==1.8.3
 platformdirs==2.4.0
-pluggy==1.0.0
+pluggy==1.6.0
 py==1.11.0
 pycodestyle==2.8.0
 pycparser==2.21
 pygments==2.12.0
-pytest==7.0.1
-pytest-forked==1.4.0
-pytest-xdist==2.5.0
+pytest==8.4.0
+pytest-xdist==3.7.0
+pytest-forked==1.6.0
+pytest-rerunfailures==15.1
 readme-renderer==34.0
 requests-toolbelt==0.9.1
 rfc3986==1.5.0
@@ -39,4 +40,4 @@ tqdm==4.64.0
 twine==3.8.0
 typed-ast==1.5.4
 webencodings==0.5.1
-pytest-asyncio==0.21.1
+pytest-asyncio==1.0.0

requirements.txt

@@ -6,6 +6,7 @@ click==8.1.3
 colorama==0.4.5
 commoncode==30.2.0
 dparse2==0.7.0
+fasteners==0.17.3
 idna==3.3
 importlib-metadata==4.12.0
 intbitset==3.1.0
@@ -26,5 +27,5 @@ text-unidecode==1.3
 toml==0.10.2
 urllib3==1.26.11
 zipp==3.8.1
-aiohttp==3.11.14
-aiofiles==23.2.1
+aiohttp==3.12.7
+aiofiles==24.1.0

setup.cfg

@@ -59,6 +59,7 @@ install_requires =
     colorama >= 0.3.9
     commoncode >= 30.0.0
     dparse2 >= 0.7.0
+    fasteners >= 0.17.3
     importlib_metadata >= 4.12.0
     packageurl_python >= 0.9.0
     pkginfo2 >= 30.0.0

src/python_inspector/api.py

@@ -29,7 +29,7 @@
 from _packagedcode.pypi import can_process_dependent_package
 from _packagedcode.pypi import get_resolved_purl
 from python_inspector import dependencies
-from python_inspector import pyinspector_settings as settings
+from python_inspector import pyinspector_settings
 from python_inspector import utils
 from python_inspector import utils_pypi
 from python_inspector.package_data import get_pypi_data_from_purl
@@ -82,7 +82,7 @@ def resolve_dependencies(
     specifiers=tuple(),
     python_version=None,
     operating_system=None,
-    index_urls: tuple[str, ...] = settings.INDEX_URL,
+    index_urls: tuple[str, ...] = pyinspector_settings.INDEX_URL,
     pdt_output=None,
     netrc_file=None,
     max_rounds=200000,
@@ -251,10 +251,10 @@ def resolve_dependencies(
     repos_by_url = {}
     if not use_pypi_json_api:
         # Collect PyPI repos
-        use_only_confed = settings.USE_ONLY_CONFIGURED_INDEX_URLS
+        use_only_confed = pyinspector_settings.USE_ONLY_CONFIGURED_INDEX_URLS
         for index_url in index_urls:
             index_url = index_url.strip("/")
-            if use_only_confed and index_url not in settings.INDEX_URL:
+            if use_only_confed and index_url not in pyinspector_settings.INDEX_URL:
                 if verbose:
                     printer(f"Skipping index URL unknown in settings: {index_url!r}")
                 continue

src/python_inspector/lockfile.py

@@ -0,0 +1,32 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from contextlib import contextmanager
+
+import fasteners
+
+"""
+An interprocess lockfile with a timeout.
+"""
+
+
+class LockTimeout(Exception):
+    pass
+
+
+class FileLock(fasteners.InterProcessLock):
+    @contextmanager
+    def locked(self, timeout):
+        acquired = self.acquire(timeout=timeout)
+        if not acquired:
+            raise LockTimeout(timeout)
+        try:
+            yield
+        finally:
+            self.release()

src/python_inspector/resolve_cli.py

@@ -13,16 +13,17 @@
 
 import click
 
-from python_inspector import pyinspector_settings as settings
+from python_inspector import settings
 from python_inspector import utils_pypi
 from python_inspector.cli_utils import FileOptionType
 from python_inspector.utils import write_output_in_file
 
 TRACE = False
 
-__version__ = "0.13.0"
+__version__ = "0.14.0"
 
 DEFAULT_PYTHON_VERSION = settings.DEFAULT_PYTHON_VERSION
+PYPI_SIMPLE_URL = settings.PYPI_SIMPLE_URL
 
 
 def print_version(ctx, param, value):
@@ -71,7 +72,6 @@ def print_version(ctx, param, value):
     "python_version",
     type=click.Choice(utils_pypi.valid_python_versions),
     metavar="PYVER",
-    default=settings.DEFAULT_PYTHON_VERSION,
     show_default=True,
     required=True,
     help="Python version to use for dependency resolution. One of "
@@ -83,19 +83,18 @@ def print_version(ctx, param, value):
     "operating_system",
     type=click.Choice(utils_pypi.PLATFORMS_BY_OS),
     metavar="OS",
-    default=settings.DEFAULT_OS,
     show_default=True,
     required=True,
     help="OS to use for dependency resolution. One of " + ", ".join(utils_pypi.PLATFORMS_BY_OS),
 )
 @click.option(
     "--index-url",
     "index_urls",
-    envvar="PYINSP_INDEX_URL",
     type=str,
     metavar="INDEX",
     show_default=True,
-    default=tuple(settings.INDEX_URL),
+    # since multiple is True, this is a sequence
+    default=[settings.PYPI_SIMPLE_URL],
     multiple=True,
     help="PyPI simple index URL(s) to use in order of preference. "
     "This option can be used multiple times.",
@@ -123,7 +122,6 @@ def print_version(ctx, param, value):
     "--netrc",
     "netrc_file",
     type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False),
-    envvar="PYINSP_NETRC_FILE",
     metavar="NETRC-FILE",
     hidden=True,
     required=False,
@@ -165,7 +163,6 @@ def print_version(ctx, param, value):
 )
 @click.option(
     "--verbose",
-    envvar="PYINSP_VERBOSE",
     is_flag=True,
     help="Enable verbose debug output.",
 )

src/python_inspector/settings.py

@@ -13,6 +13,9 @@
 from pydantic_settings import BaseSettings
 from pydantic_settings import SettingsConfigDict
 
+DEFAULT_PYTHON_VERSION = "39"
+PYPI_SIMPLE_URL = "https://pypi.org/simple"
+
 
 class Settings(BaseSettings):
     """
@@ -27,28 +30,30 @@ class Settings(BaseSettings):
         env_prefix="PYINSP_",
         case_sensitive=True,
         extra="allow",
+        # never treat data as JSON
+        enable_decoding=False,
     )
 
     # the default Python version to use if none is provided
-    DEFAULT_PYTHON_VERSION: str = "39"
+    DEFAULT_PYTHON_VERSION: str = DEFAULT_PYTHON_VERSION
 
     # the default OS to use if none is provided
     DEFAULT_OS: str = "linux"
 
-    # a list of PyPI simple index URLs. Use a JSON array to represent multiple URLs
-    INDEX_URL: tuple[str, ...] = ("https://pypi.org/simple",)
+    # a string with a tuple of PyPI simple index URLs, each separated by a space
+    INDEX_URL: tuple[str, ...] = (PYPI_SIMPLE_URL,)
 
     # If True, only uses configured INDEX_URLs listed above and ignore other URLs found in requirements
     USE_ONLY_CONFIGURED_INDEX_URLS: bool = False
 
     # a path string where to store the cached downloads. Will be created if it does not exists.
     CACHE_THIRDPARTY_DIR: str = str(Path(Path.home() / ".cache/python_inspector"))
 
-    @field_validator("INDEX_URL")
+    @field_validator("INDEX_URL", mode="before")
     @classmethod
     def validate_index_url(cls, value):
         if isinstance(value, str):
-            return (value,)
+            return tuple(value.split())
         elif isinstance(value, (tuple, list)):
             return tuple(value)
         else:

src/python_inspector/utils_pypi.py

@@ -41,6 +41,7 @@
 from packvers import version as packaging_version
 from packvers.specifiers import SpecifierSet
 
+from python_inspector import lockfile
 from python_inspector import pyinspector_settings as settings
 from python_inspector import utils_pip_compatibility_tags
 
@@ -1650,6 +1651,8 @@ def resolve_relative_url(package_url, url):
 #
 ################################################################################
 
+PYINSP_CACHE_LOCK_TIMEOUT = 120  # in seconds
+
 
 @attr.attributes
 class Cache:
@@ -1681,6 +1684,7 @@ async def get(
         True otherwise as treat as binary. `path_or_url` can be a path or a URL
         to a file.
         """
+        # the cache key is a hash of the normalized path
         cache_key = self.sha256_hash(quote_plus(path_or_url.strip("/")))
         cached = os.path.join(self.directory, cache_key)
 
@@ -1695,8 +1699,13 @@ async def get(
                 echo_func=echo_func,
             )
             wmode = "w" if as_text else "wb"
-            async with aiofiles.open(cached, mode=wmode) as fo:
-                await fo.write(content)
+
+            # acquire lock and wait until timeout to get a lock or die
+            lock_file = os.path.join(self.directory, f"{cache_key}.lockfile")
+
+            with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT):
+                async with aiofiles.open(cached, mode=wmode) as fo:
+                    await fo.write(content)
             return content, cached
         else:
             if TRACE_DEEP:

tests/data/pinned-pdt-requirements.txt-expected.json

@@ -2,7 +2,6 @@
   "headers": {
     "tool_name": "python-inspector",
     "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector",
-    "tool_version": "0.13.0",
     "options": [
       "--index-url https://pypi.org/simple",
       "--json-pdt <file>",

tests/data/pinned-requirements.txt-expected.json

@@ -2,7 +2,6 @@
   "headers": {
     "tool_name": "python-inspector",
     "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector",
-    "tool_version": "0.13.0",
     "options": [
       "--index-url https://pypi.org/simple",
       "--json <file>",