diff --git a/.circleci/config.yml b/.circleci/config.yml index 86160a46180b7..4648cdf12c4b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ jobs: steps: - checkout - restore_cache: - key: python-requirements-{{ checksum "requirements.txt" }} + key: python-requirements-{{ checksum "setup.cfg" }} - run: name: Install prerequisites command: | @@ -23,7 +23,7 @@ jobs: mkdir -p target/reports mkdir -p target/coverage - save_cache: - key: python-requirements-{{ checksum "requirements.txt" }} + key: python-requirements-{{ checksum "setup.cfg" }} paths: - "~/.cache/pip" - persist_to_workspace: diff --git a/.github/workflows/asf-updates.yml b/.github/workflows/asf-updates.yml index 2c6f00a6bbc70..acd120196c0a1 100644 --- a/.github/workflows/asf-updates.yml +++ b/.github/workflows/asf-updates.yml @@ -28,7 +28,7 @@ jobs: uses: actions/cache@v2 with: path: .venv - key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('requirements.txt') }} + key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('setup.cfg') }} - name: Install dependencies run: make install-dev diff --git a/.github/workflows/pro-integration.yml b/.github/workflows/pro-integration.yml index 3f07cbf90ccdc..8ea2c7459a5cb 100644 --- a/.github/workflows/pro-integration.yml +++ b/.github/workflows/pro-integration.yml @@ -58,7 +58,7 @@ jobs: id: ext-cache with: path: localstack-ext/.venv - key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('localstack-ext/setup.cfg', 'localstack-ext/pyproject.toml', 'localstack/localstack/services/install.py', 'localstack/requirements.txt', 'localstack/localstack/constants.py') }} + key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('localstack-ext/setup.cfg', 'localstack-ext/pyproject.toml', 'localstack/localstack/services/install.py', 'localstack/setup.cfg', 'localstack/localstack/constants.py') }} - name: Install Python Dependencies for LocalStack Pro run: make install - name: Cache LocalStack community dependencies (venv, infra) @@ -69,7 +69,7 @@ jobs: localstack/.venv localstack/localstack/infra localstack/localstack/node_modules - key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('localstack-ext/setup.cfg', 'localstack-ext/pyproject.toml', 'localstack/localstack/services/install.py', 'localstack/requirements.txt', 'localstack/localstack/constants.py') }} + key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('localstack-ext/setup.cfg', 'localstack-ext/pyproject.toml', 'localstack/localstack/services/install.py', 'localstack/setup.cfg', 'localstack/localstack/constants.py') }} - name: Install Dependencies for LocalStack Community # lambda tests look for libraries in this virtualenv working-directory: localstack run: | diff --git a/Dockerfile b/Dockerfile index 7a40668e28c7f..9b84eb1edf2d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -141,7 +141,7 @@ RUN mkdir -p /opt/code/localstack/localstack/infra/dynamodb && \ RUN (virtualenv .venv && source .venv/bin/activate && pip3 install --upgrade pip wheel setuptools) # add files necessary to install all dependencies -ADD Makefile setup.py requirements.txt pyproject.toml ./ +ADD Makefile setup.py setup.cfg pyproject.toml ./ # add the root package init to invalidate docker layers with version bumps ADD localstack/__init__.py localstack/ # add the localstack start scripts (necessary for the installation of the runtime dependencies, i.e. `pip install -e .`) diff --git a/MANIFEST.in b/MANIFEST.in index aa0dec76fb9cd..47720f78d45d3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ -include requirements.txt include Makefile include LICENSE.txt recursive-include localstack/ext *.java diff --git a/Makefile b/Makefile index 85348fca3bd14..251e360dae0e0 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ VENV_RUN = . $(VENV_ACTIVATE) usage: ## Show this help @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/:.*##\s*/##/g' | awk -F'##' '{ printf "%-25s %s\n", $$1, $$2 }' -$(VENV_ACTIVATE): setup.py requirements.txt +$(VENV_ACTIVATE): setup.py setup.cfg test -d $(VENV_DIR) || $(VENV_BIN) $(VENV_DIR) $(VENV_RUN); $(PIP_CMD) install --upgrade pip setuptools wheel plux touch $(VENV_ACTIVATE) @@ -46,11 +46,10 @@ install-test: venv ## Install requirements to run tests into venv install-dev: venv ## Install developer requirements into venv $(VENV_RUN); $(PIP_CMD) install $(PIP_OPTS) -e ".[cli,runtime,test,dev]" -install: ## Install full dependencies into venv, and download third-party services - (make install-dev && make entrypoints && make init-testlibs) || exit 1 +install: install-dev entrypoints init-testlibs ## Install full dependencies into venv, and download third-party services entrypoints: ## Run setup.py develop to build entry points - $(VENV_RUN); rm -f localstack.egg-info/entry_points.txt; python setup.py develop + $(VENV_RUN); python setup.py plugins egg_info init: ## Initialize the infrastructure, make sure all libs are downloaded $(VENV_RUN); python -m localstack.services.install libs @@ -58,7 +57,7 @@ init: ## Initialize the infrastructure, make sure all libs a init-testlibs: $(VENV_RUN); python -m localstack.services.install testlibs -dist: ## Build source and built (wheel) distributions of the current version +dist: entrypoints ## Build source and built (wheel) distributions of the current version $(VENV_RUN); pip install --upgrade twine; python setup.py sdist bdist_wheel publish: clean-dist dist ## Publish the library to the central PyPi repository diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b5c7580f332c7..0000000000000 --- a/requirements.txt +++ /dev/null @@ -1,83 +0,0 @@ -# dependencies are grouped into blocks that define in which extra they belong to. for example, all requirements below -# the line `# extra=runtime`, can be installed with `pip install -e ".[runtime]"` -# -# install: dependencies that are required for the cli (via pip install localstack) -# runtime: required to actually run localstack on the host -# test: for running tests and coverage analysis -# dev: for developing localstack - -# extra=install -boto3>=1.20 -click>=7.0 -cachetools>=3.1.1,<4.0.0 -# dataclasses needed for python3.6 compat -dataclasses; python_version < '3.7' -#dnspython==1.16.0 -localstack-client>=1.31 -localstack-ext>=0.14.0 -plux>=1.3.1 -psutil>=5.4.8,<6.0.0 -python-dotenv>=0.19.1 -pyyaml>=5.1 -rich>=10.7.0 -requests>=2.20.0,<2.26 -semver>=2.10 -stevedore>=3.4.0 -# needed for python3.7 compat (TypedDict, Literal, type hints) -typing-extensions; python_version < '3.8' -tailer>=0.4.1 - -# extra=runtime -airspeed>=0.5.18 -# Use our "ext" version until this bug is fixed: https://github.com/awslabs/amazon-kinesis-client-python/issues/99 -amazon_kclpy-ext==1.5.1 -# amazon-kclpy==1.5.1 -aws-sam-translator>=1.15.1 -awscli>=1.14.18 -boto>=2.49.0 -botocore>=1.12.13 -cbor2>=5.2.0 -crontab>=0.22.6 -cryptography -docker==5.0.0 -flask>=1.0.2 -flask-cors>=3.0.3,<3.1.0 -flask_swagger==0.2.12 -jsonpatch>=1.24,<2.0 -jsonpath-rw>=1.4.0,<2.0.0 -localstack-ext[full]>=0.13.2 -moto-ext[all]==3.0.4 -opensearch-py>=1.0.0 -pproxy>=2.7.0 -#pympler>=0.6 -pyopenssl>=21.0.0 -Quart>=0.6.15 -readerwriterlock>=1.0.7 -requests-aws4auth==0.9 -#sasl>=0.2.1 -Werkzeug>=2.0 -xmltodict>=0.11.0 - -# extra=test -# these requirements are used for testing, also during docker build -pytest==6.2.4 -pytest-httpserver>=1.0.1 -pytest-rerunfailures==10.0 -# coverage version should be synced with bin/Dockerfile.base -coverage[toml]>=5.5 - -# extra=dev -black==21.6b0 -coveralls==3.1.0 -Cython -flake8>=3.6.0 -flake8-black==0.2.3 -flake8-isort>=4.0.0 -flake8-quotes>=0.11.0 -# enables flake8 configuration through pyproject.toml -pre-commit==2.13.0 -pyproject-flake8 -isort==5.9.1 -pandoc -pypandoc -autoflake diff --git a/setup.cfg b/setup.cfg index 08970c31d1b1a..e36d5a9ffe198 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,3 +20,105 @@ classifiers = [options] zip_safe = False +test_suite = tests +scripts = + bin/localstack + bin/localstack.bat +packages=find: + +# dependencies that are required for the cli (via pip install localstack) +install_requires = + boto3>=1.20 + click>=7.0 + cachetools>=3.1.1,<4.0.0 + # dataclasses needed for python3.6 compat + dataclasses; python_version < '3.7' + #dnspython==1.16.0 + localstack-client>=1.31 + localstack-ext>=0.14.0 + plux>=1.3.1 + psutil>=5.4.8,<6.0.0 + python-dotenv>=0.19.1 + pyyaml>=5.1 + rich>=10.7.0 + requests>=2.20.0,<2.26 + semver>=2.10 + stevedore>=3.4.0 + # needed for python3.7 compat (TypedDict, Literal, type hints) + typing-extensions; python_version < '3.8' + tailer>=0.4.1 + +[options.packages.find] +exclude = + tests + tests.* + +[options.package_data] +* = + *.md + Makefile +localstack = + package.json + utils/kinesis/java/cloud/localstack/*.* + +[options.extras_require] +# required to actually run localstack on the host +runtime = + airspeed>=0.5.18 + # Use our "ext" version until this bug is fixed: https://github.com/awslabs/amazon-kinesis-client-python/issues/99 + amazon_kclpy-ext==1.5.1 + # amazon-kclpy==1.5.1 + aws-sam-translator>=1.15.1 + awscli>=1.14.18 + boto>=2.49.0 + botocore>=1.12.13 + cbor2>=5.2.0 + crontab>=0.22.6 + cryptography + docker==5.0.0 + flask>=1.0.2 + flask-cors>=3.0.3,<3.1.0 + flask_swagger==0.2.12 + jsonpatch>=1.24,<2.0 + jsonpath-rw>=1.4.0,<2.0.0 + localstack-ext[full]>=0.13.2 + moto-ext[all]==3.0.4 + opensearch-py>=1.0.0 + pproxy>=2.7.0 + #pympler>=0.6 + pyopenssl>=21.0.0 + Quart>=0.6.15 + readerwriterlock>=1.0.7 + requests-aws4auth==0.9 + #sasl>=0.2.1 + Werkzeug>=2.0 + xmltodict>=0.11.0 + +# @deprecated - use extra 'runtime' instead. +full = + %(runtime)s + +# for running tests and coverage analysis +test = + pytest==6.2.4 + pytest-httpserver>=1.0.1 + pytest-rerunfailures==10.0 + # coverage version should be synced with bin/Dockerfile.base + coverage[toml]>=5.5 + +# for developing localstack +dev = + black==21.6b0 + coveralls==3.1.0 + Cython + flake8>=3.6.0 + flake8-black==0.2.3 + flake8-isort>=4.0.0 + flake8-quotes>=0.11.0 + # enables flake8 configuration through pyproject.toml + pre-commit==2.13.0 + pyproject-flake8 + isort==5.9.1 + pandoc + pypandoc + autoflake diff --git a/setup.py b/setup.py index ad15a62a3cd4e..c82334553632d 100755 --- a/setup.py +++ b/setup.py @@ -1,63 +1,4 @@ #!/usr/bin/env python -import re -from collections import defaultdict +from setuptools import setup -from plugin.setuptools import load_entry_points -from setuptools import find_packages, setup - - -def parse_requirements(lines): - requirements = defaultdict(list) - extra = "install" - - for line in lines: - line = line.strip() - if line.startswith("# extra="): - # all subsequent lines are associated with this extra, until a new extra appears - extra = line.split("=")[1] - continue - - if line and line[0] == "#" and "#egg=" in line: - line = re.search(r"#\s*(.*)", line).group(1) - - if line and line[0] != "#": - lib_stripped = line.split(" #")[0].strip() - requirements[extra].append(lib_stripped) - - return requirements - - -# define package data -package_data = { - "": ["Makefile", "*.md"], - "localstack": [ - "package.json", - "requirements*.txt", - "utils/kinesis/java/cloud/localstack/*.*", - ], -} - -# determine requirements -with open("requirements.txt") as f: - req = parse_requirements(f.readlines()) - -install_requires = req["install"] - -extras_require = { - "cli": req["install"], - "runtime": req["runtime"], - "test": req["test"], - "dev": req["dev"], -} -extras_require["full"] = extras_require["cli"] + extras_require["runtime"] # deprecated - -if __name__ == "__main__": - setup( - scripts=["bin/localstack", "bin/localstack.bat"], - packages=find_packages(exclude=("tests", "tests.*")), - package_data=package_data, - install_requires=install_requires, - extras_require=extras_require, - entry_points=load_entry_points(exclude=("tests", "tests.*")), - test_suite="tests", - ) +setup()