migrate from requirements.txt/setup.py to setup.cfg (localstack#5606)

.circleci/config.yml

@@ -8,7 +8,7 @@ jobs:
     steps:
       - checkout
       - restore_cache:
-          key: python-requirements-{{ checksum "requirements.txt" }}
+          key: python-requirements-{{ checksum "setup.cfg" }}
       - run:
           name: Install prerequisites
           command: |
@@ -23,7 +23,7 @@ jobs:
             mkdir -p target/reports
             mkdir -p target/coverage
       - save_cache:
-          key: python-requirements-{{ checksum "requirements.txt" }}
+          key: python-requirements-{{ checksum "setup.cfg" }}
           paths:
             - "~/.cache/pip"
       - persist_to_workspace:

.github/workflows/asf-updates.yml

@@ -28,7 +28,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: .venv
-          key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('setup.cfg') }}
 
       - name: Install dependencies
         run: make install-dev

.github/workflows/pro-integration.yml

@@ -58,7 +58,7 @@ jobs:
         id: ext-cache
         with:
           path: localstack-ext/.venv
-          key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('localstack-ext/setup.cfg', 'localstack-ext/pyproject.toml', 'localstack/localstack/services/install.py', 'localstack/requirements.txt', 'localstack/localstack/constants.py') }}
+          key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('localstack-ext/setup.cfg', 'localstack-ext/pyproject.toml', 'localstack/localstack/services/install.py', 'localstack/setup.cfg', 'localstack/localstack/constants.py') }}
       - name: Install Python Dependencies for LocalStack Pro
         run: make install
       - name: Cache LocalStack community dependencies (venv, infra)
@@ -69,7 +69,7 @@ jobs:
             localstack/.venv
             localstack/localstack/infra
             localstack/localstack/node_modules
-          key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('localstack-ext/setup.cfg', 'localstack-ext/pyproject.toml', 'localstack/localstack/services/install.py', 'localstack/requirements.txt', 'localstack/localstack/constants.py') }}
+          key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('localstack-ext/setup.cfg', 'localstack-ext/pyproject.toml', 'localstack/localstack/services/install.py', 'localstack/setup.cfg', 'localstack/localstack/constants.py') }}
       - name: Install Dependencies for LocalStack Community # lambda tests look for libraries in this virtualenv
         working-directory: localstack
         run: |

Dockerfile

@@ -141,7 +141,7 @@ RUN mkdir -p /opt/code/localstack/localstack/infra/dynamodb && \
 RUN (virtualenv .venv && source .venv/bin/activate && pip3 install --upgrade pip wheel setuptools)
 
 # add files necessary to install all dependencies
-ADD Makefile setup.py requirements.txt pyproject.toml ./
+ADD Makefile setup.py setup.cfg pyproject.toml ./
 # add the root package init to invalidate docker layers with version bumps
 ADD localstack/__init__.py localstack/
 # add the localstack start scripts (necessary for the installation of the runtime dependencies, i.e. `pip install -e .`)

MANIFEST.in

@@ -1,4 +1,3 @@
-include requirements.txt
 include Makefile
 include LICENSE.txt
 recursive-include localstack/ext *.java

Makefile

@@ -24,7 +24,7 @@ VENV_RUN = . $(VENV_ACTIVATE)
 usage:                    ## Show this help
 	@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/:.*##\s*/##/g' | awk -F'##' '{ printf "%-25s %s\n", $$1, $$2 }'
 
-$(VENV_ACTIVATE): setup.py requirements.txt
+$(VENV_ACTIVATE): setup.py setup.cfg
 	test -d $(VENV_DIR) || $(VENV_BIN) $(VENV_DIR)
 	$(VENV_RUN); $(PIP_CMD) install --upgrade pip setuptools wheel plux
 	touch $(VENV_ACTIVATE)
@@ -46,19 +46,18 @@ install-test: venv        ## Install requirements to run tests into venv
 install-dev: venv         ## Install developer requirements into venv
 	$(VENV_RUN); $(PIP_CMD) install $(PIP_OPTS) -e ".[cli,runtime,test,dev]"
 
-install:                  ## Install full dependencies into venv, and download third-party services
-	(make install-dev && make entrypoints && make init-testlibs) || exit 1
+install: install-dev entrypoints init-testlibs  ## Install full dependencies into venv, and download third-party services
 
 entrypoints:              ## Run setup.py develop to build entry points
-	$(VENV_RUN); rm -f localstack.egg-info/entry_points.txt; python setup.py develop
+	$(VENV_RUN); python setup.py plugins egg_info
 
 init:                     ## Initialize the infrastructure, make sure all libs are downloaded
 	$(VENV_RUN); python -m localstack.services.install libs
 
 init-testlibs:
 	$(VENV_RUN); python -m localstack.services.install testlibs
 
-dist:					  ## Build source and built (wheel) distributions of the current version
+dist: entrypoints        ## Build source and built (wheel) distributions of the current version
 	$(VENV_RUN); pip install --upgrade twine; python setup.py sdist bdist_wheel
 
 publish: clean-dist dist  ## Publish the library to the central PyPi repository

setup.cfg

@@ -20,3 +20,105 @@ classifiers =
 
 [options]
 zip_safe = False
+test_suite = tests
+scripts =
+    bin/localstack
+    bin/localstack.bat
+packages=find:
+
+# dependencies that are required for the cli (via pip install localstack)
+install_requires =
+    boto3>=1.20
+    click>=7.0
+    cachetools>=3.1.1,<4.0.0
+    # dataclasses needed for python3.6 compat
+    dataclasses; python_version < '3.7'
+    #dnspython==1.16.0
+    localstack-client>=1.31
+    localstack-ext>=0.14.0
+    plux>=1.3.1
+    psutil>=5.4.8,<6.0.0
+    python-dotenv>=0.19.1
+    pyyaml>=5.1
+    rich>=10.7.0
+    requests>=2.20.0,<2.26
+    semver>=2.10
+    stevedore>=3.4.0
+    # needed for python3.7 compat (TypedDict, Literal, type hints)
+    typing-extensions; python_version < '3.8'
+    tailer>=0.4.1
+
+[options.packages.find]
+exclude =
+    tests
+    tests.*
+
+[options.package_data]
+* =
+    *.md
+    Makefile
+localstack =
+    package.json
+    utils/kinesis/java/cloud/localstack/*.*
+
+[options.extras_require]
+# required to actually run localstack on the host
+runtime =
+    airspeed>=0.5.18
+    # Use our "ext" version until this bug is fixed: https://github.com/awslabs/amazon-kinesis-client-python/issues/99
+    amazon_kclpy-ext==1.5.1
+    # amazon-kclpy==1.5.1
+    aws-sam-translator>=1.15.1
+    awscli>=1.14.18
+    boto>=2.49.0
+    botocore>=1.12.13
+    cbor2>=5.2.0
+    crontab>=0.22.6
+    cryptography
+    docker==5.0.0
+    flask>=1.0.2
+    flask-cors>=3.0.3,<3.1.0
+    flask_swagger==0.2.12
+    jsonpatch>=1.24,<2.0
+    jsonpath-rw>=1.4.0,<2.0.0
+    localstack-ext[full]>=0.13.2
+    moto-ext[all]==3.0.4
+    opensearch-py>=1.0.0
+    pproxy>=2.7.0
+    #pympler>=0.6
+    pyopenssl>=21.0.0
+    Quart>=0.6.15
+    readerwriterlock>=1.0.7
+    requests-aws4auth==0.9
+    #sasl>=0.2.1
+    Werkzeug>=2.0
+    xmltodict>=0.11.0
+
+# @deprecated - use extra 'runtime' instead.
+full =
+    %(runtime)s
+
+# for running tests and coverage analysis
+test =
+    pytest==6.2.4
+    pytest-httpserver>=1.0.1
+    pytest-rerunfailures==10.0
+    # coverage version should be synced with bin/Dockerfile.base
+    coverage[toml]>=5.5
+
+# for developing localstack
+dev =
+    black==21.6b0
+    coveralls==3.1.0
+    Cython
+    flake8>=3.6.0
+    flake8-black==0.2.3
+    flake8-isort>=4.0.0
+    flake8-quotes>=0.11.0
+    # enables flake8 configuration through pyproject.toml
+    pre-commit==2.13.0
+    pyproject-flake8
+    isort==5.9.1
+    pandoc
+    pypandoc
+    autoflake

setup.py

@@ -1,63 +1,4 @@
 #!/usr/bin/env python
-import re
-from collections import defaultdict
+from setuptools import setup
 
-from plugin.setuptools import load_entry_points
-from setuptools import find_packages, setup
-
-
-def parse_requirements(lines):
-    requirements = defaultdict(list)
-    extra = "install"
-
-    for line in lines:
-        line = line.strip()
-        if line.startswith("# extra="):
-            # all subsequent lines are associated with this extra, until a new extra appears
-            extra = line.split("=")[1]
-            continue
-
-        if line and line[0] == "#" and "#egg=" in line:
-            line = re.search(r"#\s*(.*)", line).group(1)
-
-        if line and line[0] != "#":
-            lib_stripped = line.split(" #")[0].strip()
-            requirements[extra].append(lib_stripped)
-
-    return requirements
-
-
-# define package data
-package_data = {
-    "": ["Makefile", "*.md"],
-    "localstack": [
-        "package.json",
-        "requirements*.txt",
-        "utils/kinesis/java/cloud/localstack/*.*",
-    ],
-}
-
-# determine requirements
-with open("requirements.txt") as f:
-    req = parse_requirements(f.readlines())
-
-install_requires = req["install"]
-
-extras_require = {
-    "cli": req["install"],
-    "runtime": req["runtime"],
-    "test": req["test"],
-    "dev": req["dev"],
-}
-extras_require["full"] = extras_require["cli"] + extras_require["runtime"]  # deprecated
-
-if __name__ == "__main__":
-    setup(
-        scripts=["bin/localstack", "bin/localstack.bat"],
-        packages=find_packages(exclude=("tests", "tests.*")),
-        package_data=package_data,
-        install_requires=install_requires,
-        extras_require=extras_require,
-        entry_points=load_entry_points(exclude=("tests", "tests.*")),
-        test_suite="tests",
-    )
+setup()