Switch to new Lambda provider implementation (localstack#6724)

Co-authored-by: Dominik Schubert <[email protected]>
F944 · Mar 23, 2023 · 88dd9b3 · 88dd9b3
1 parent ee34ca6
commit 88dd9b3
Show file tree

Hide file tree

Showing 36 changed files with 359 additions and 326 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -78,102 +78,19 @@ jobs:
           paths:
             - repo/target/coverage/
 
-  itest-lambda-docker:
+  itest-lambda-legacy-local:
     executor: ubuntu-machine-amd64
     working_directory: /tmp/workspace/repo
     steps:
       - attach_workspace:
           at: /tmp/workspace
       - run:
-          name: Pull Lambda runtimes
-          command: |
-            sudo useradd localstack -s /bin/bash
-            docker pull -q lambci/lambda:nodejs12.x
-            docker pull -q localstack/lambda-js:nodejs14.x
-            docker pull -q lambci/lambda:ruby2.7
-            docker pull -q lambci/lambda:python3.6
-            docker pull -q lambci/lambda:python3.7
-            docker pull -q lambci/lambda:python3.8
-            docker pull -q mlupin/docker-lambda:python3.9
-            docker pull -q lambci/lambda:dotnetcore3.1
-            docker pull -q mlupin/docker-lambda:dotnet6
-            docker pull -q lambci/lambda:provided
-            docker pull -q lambci/lambda:java8
-            docker pull -q lambci/lambda:java8.al2
-            docker pull -q lambci/lambda:java11
-      - run:
-          name: Test Docker client
-          environment:
-            TEST_PATH: "tests/integration/docker_utils"
-            TEST_SKIP_LOCALSTACK_START: 1
-            PYTEST_ARGS: "--reruns 2 --junitxml=target/reports/docker-client.xml -o junit_suite_name='docker-client'"
-            COVERAGE_ARGS: "-p"
-          command: make test-coverage
-      - run:
-          name: Test 'docker' Lambda executor
-          environment:
-            LAMBDA_EXECUTOR: "docker"
-            USE_SSL: 1
-            TEST_PATH: "tests/integration/awslambda/ tests/integration/test_integration.py tests/integration/apigateway/test_apigateway_basic.py"
-            PYTEST_ARGS: "--reruns 2 --junitxml=target/reports/lambda-docker.xml -o junit_suite_name='lambda-docker'"
-            COVERAGE_ARGS: "-p"
-          command: make test-coverage
-      - run:
-          name: Test 'docker-reuse' Lambda executor
-          environment:
-            LAMBDA_EXECUTOR: "docker-reuse"
-            TEST_PATH: "tests/integration/awslambda/ tests/integration/test_integration.py tests/integration/apigateway/test_apigateway_basic.py"
-            PYTEST_ARGS: "--reruns 2 --junitxml=target/reports/lambda-docker-reuse.xml -o junit_suite_name='lambda-docker-reuse'"
-            COVERAGE_ARGS: "-p"
-          command: make test-coverage
-      - run:
-          name: Store coverage results
-          command: mv .coverage.* target/coverage/
-      - persist_to_workspace:
-          root:
-            /tmp/workspace
-          paths:
-            - repo/target/coverage/
-      - store_test_results:
-          path: target/reports/
-
-  itest-lambda-provider:
-    executor: ubuntu-machine-amd64
-    working_directory: /tmp/workspace/repo
-    steps:
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Test ASF Lambda provider
+          name: Test 'local' Lambda executor
           environment:
-            PROVIDER_OVERRIDE_LAMBDA: "asf"
-            TEST_PATH: "tests/integration/awslambda/test_lambda.py tests/integration/awslambda/test_lambda_api.py tests/integration/awslambda/test_lambda_common.py tests/integration/awslambda/test_lambda_integration_sqs.py tests/integration/cloudformation/resources/test_lambda.py tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py tests/integration/awslambda/test_lambda_integration_kinesis.py tests/integration/awslambda/test_lambda_developer_tools.py tests/integration/test_network_configuration.py::TestLambda"
-            PYTEST_ARGS: "--reruns 3 --junitxml=target/reports/lambda_asf.xml -o junit_suite_name='lambda_asf'"
-            COVERAGE_ARGS: "-p"
-          command: make test-coverage
-      - run:
-          name: Store coverage results
-          command: mv .coverage.* target/coverage/
-      - persist_to_workspace:
-          root:
-            /tmp/workspace
-          paths:
-            - repo/target/coverage/
-      - store_test_results:
-          path: target/reports/
-
-  itest-s3-asf-provider:
-    executor: ubuntu-machine-amd64
-    working_directory: /tmp/workspace/repo
-    steps:
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Test ASF S3 provider
-          environment:
-            PROVIDER_OVERRIDE_S3: "asf"
-            TEST_PATH: "tests/integration/s3/ tests/integration/test_network_configuration.py::TestS3"
-            PYTEST_ARGS: "--reruns 3 --junitxml=target/reports/s3_asf.xml -o junit_suite_name='s3_asf'"
+            LAMBDA_EXECUTOR: "local"
+            PROVIDER_OVERRIDE_LAMBDA: "legacy"
+            TEST_PATH: "tests/integration/awslambda/ tests/integration/test_integration.py tests/integration/apigateway/test_apigateway_basic.py tests/integration/cloudformation/resources/test_lambda.py"
+            PYTEST_ARGS: "--reruns 2 --junitxml=target/reports/lambda-docker.xml -o junit_suite_name='legacy-lambda-local'"
             COVERAGE_ARGS: "-p"
           command: make test-coverage
       - run:
@@ -263,13 +180,20 @@ jobs:
       image: << parameters.machine_image >>
     resource_class: << parameters.resource_class >>
     working_directory: /tmp/workspace/repo
-    parallelism: 2
+    parallelism: 4
     steps:
       - attach_workspace:
           at: /tmp/workspace
       - run:
           name: Load docker image
           command: docker load -i target/localstack-docker-image-<< parameters.platform >>.tar
+      - when:
+          condition:
+            equal: [ "amd64", << parameters.platform >>]
+          steps:
+            - run:
+                name: pre-build lambda common test packages
+                command: ./scripts/build_common_test_functions.sh `pwd`/tests/integration/awslambda/functions/common
       - run:
           name: Run integration tests
           # circleci split returns newline separated list, so `tr` is necessary to prevent problems in the Makefile
@@ -278,6 +202,7 @@ jobs:
             PYTEST_ARGS="-o junit_family=legacy --junitxml=target/reports/test-report-<< parameters.platform >>-${CIRCLE_NODE_INDEX}.xml" \
             COVERAGE_FILE="target/coverage/.coverage.<< parameters.platform >>.${CIRCLE_NODE_INDEX}" \
             TEST_PATH=$TEST_FILES \
+            DEBUG=1 \
             make docker-run-tests
       - store_test_results:
           path: target/reports/
@@ -426,13 +351,7 @@ workflows:
       - preflight:
           requires:
             - install
-      - itest-lambda-docker:
-          requires:
-            - preflight
-      - itest-lambda-provider:
-          requires:
-            - preflight
-      - itest-s3-asf-provider:
+      - itest-lambda-legacy-local:
           requires:
             - preflight
       - itest-sfn-v2-provider:
@@ -477,9 +396,7 @@ workflows:
             - docker-build-amd64
       - report:
           requires:
-            - itest-lambda-docker
-            - itest-lambda-provider
-            - itest-s3-asf-provider
+            - itest-lambda-legacy-local
             - itest-sfn-v2-provider
             - docker-test-amd64
             - docker-test-arm64
@@ -490,9 +407,7 @@ workflows:
             branches:
               only: master
           requires:
-            - itest-lambda-docker
-            - itest-lambda-provider
-            - itest-s3-asf-provider
+            - itest-lambda-legacy-local
             - itest-sfn-v2-provider
             - docker-test-amd64
             - docker-test-arm64

diff --git a/.github/workflows/pro-integration.yml b/.github/workflows/pro-integration.yml
@@ -178,48 +178,16 @@ jobs:
         env:
           DEBUG: 1
           DNS_ADDRESS: 0
-          LAMBDA_EXECUTOR: "local"
           LOCALSTACK_API_KEY: "test"
           AWS_SECRET_ACCESS_KEY: "test"
           AWS_ACCESS_KEY_ID: "test"
           AWS_DEFAULT_REGION: "us-east-1"
-          HOST_TMP_FOLDER: /tmp/localstack
           PYTEST_LOGLEVEL: debug
         run: |
           # Remove the host tmp folder (might contain remnant files with different permissions)
           sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp
           source .venv/bin/activate
           python -m pytest --capture=no --reruns 2 --durations=10 --junitxml=target/reports/pytest.xml ../localstack/tests/integration/
-      - name: Run Lambda Tests for lambda executor docker
-        env:
-          DEBUG: 0
-          DNS_ADDRESS: 0
-          LAMBDA_EXECUTOR: "docker"
-          LOCALSTACK_API_KEY: "test"
-          HOST_TMP_FOLDER: /tmp/localstack
-        run: |
-          # Remove the host tmp folder (might contain remnant files with different permissions)
-          sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp
-          source .venv/bin/activate
-          python -m pytest --reruns 2 --durations=10 --show-capture=no --junitxml=target/reports/lambda-docker.xml -o junit_suite_name='lambda-docker' \
-            ../localstack/tests/integration/awslambda/ \
-            ../localstack/tests/integration/test_integration.py \
-            ../localstack/tests/integration/apigateway/test_apigateway_basic.py
-      - name: Run Lambda Tests for lambda executor docker-reuse
-        env:
-          DEBUG: 1
-          DNS_ADDRESS: 0
-          LAMBDA_EXECUTOR: "docker-reuse"
-          LOCALSTACK_API_KEY: "test"
-          HOST_TMP_FOLDER: /tmp/localstack
-        run: |
-          # Remove the host tmp folder (might contain remnant files with different permissions)
-          sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp
-          source .venv/bin/activate
-          python -m pytest --reruns 2 --durations=10 --show-capture=no --junitxml=target/reports/lambda-docker-reuse.xml -o junit_suite_name='lambda-docker-reuse' \
-            ../localstack/tests/integration/awslambda/ \
-            ../localstack/tests/integration/test_integration.py \
-            ../localstack/tests/integration/apigateway/test_apigateway_basic.py
       - name: Publish LocalStack Community Integration Test Results
         uses: EnricoMi/publish-unit-test-result-action@v1
         if: always()

diff --git a/Makefile b/Makefile
@@ -151,9 +151,9 @@ docker-create-push-manifests:	## Create and push manifests for a docker image (d
 docker-run-tests:		  ## Initializes the test environment and runs the tests in a docker container
 	# Remove argparse and dataclasses to fix https://github.com/pytest-dev/pytest/issues/5594
 	# Note: running "install-test-only" below, to avoid pulling in [runtime] extras from transitive dependencies
-	docker run -e LOCALSTACK_INTERNAL_TEST_COLLECT_METRIC=1 --entrypoint= -v `pwd`/tests/:/opt/code/localstack/tests/ -v `pwd`/target/:/opt/code/localstack/target/ \
+	docker run -e LOCALSTACK_INTERNAL_TEST_COLLECT_METRIC=1 --entrypoint= -v `pwd`/tests/:/opt/code/localstack/tests/ -v `pwd`/target/:/opt/code/localstack/target/ -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/localstack:/var/lib/localstack \
 		$(IMAGE_NAME) \
-	    bash -c "make install-test-only && pip uninstall -y argparse dataclasses && DEBUG=$(DEBUG) LAMBDA_EXECUTOR=local PYTEST_LOGLEVEL=debug PYTEST_ARGS='$(PYTEST_ARGS)' COVERAGE_FILE='$(COVERAGE_FILE)' TEST_PATH='$(TEST_PATH)' make test-coverage"
+	    bash -c "make install-test-only && pip uninstall -y argparse dataclasses && DEBUG=$(DEBUG) PYTEST_LOGLEVEL=debug PYTEST_ARGS='$(PYTEST_ARGS)' COVERAGE_FILE='$(COVERAGE_FILE)' TEST_PATH='$(TEST_PATH)' LAMBDA_IGNORE_ARCHITECTURE=1 LAMBDA_INIT_POST_INVOKE_WAIT_MS=50 make test-coverage"
 
 docker-run:        		  ## Run Docker image locally
 	($(VENV_RUN); bin/localstack start)

diff --git a/localstack/config.py b/localstack/config.py
@@ -784,6 +784,7 @@ def get_gateway_listen(gateway_listen: str) -> List[HostAndPort]:
 LAMBDA_INIT_BOOTSTRAP_PATH = os.environ.get("LAMBDA_INIT_BOOTSTRAP_PATH")
 LAMBDA_INIT_DELVE_PATH = os.environ.get("LAMBDA_INIT_DELVE_PATH")
 LAMBDA_INIT_DELVE_PORT = int(os.environ.get("LAMBDA_INIT_DELVE_PORT") or 40000)
+LAMBDA_INIT_POST_INVOKE_WAIT_MS = os.environ.get("LAMBDA_INIT_POST_INVOKE_WAIT_MS")
 # Alternative user or empty string to skip dropping privileges
 LAMBDA_INIT_USER = os.environ.get("LAMBDA_INIT_USER")
 
@@ -926,6 +927,7 @@ def get_gateway_listen(gateway_listen: str) -> List[HostAndPort]:
     "LAMBDA_INIT_BOOTSTRAP_PATH",
     "LAMBDA_INIT_DELVE_PATH",
     "LAMBDA_INIT_DELVE_PORT",
+    "LAMBDA_INIT_POST_INVOKE_WAIT",
     "LAMBDA_INIT_USER",
     "LAMBDA_RUNTIME_IMAGE_MAPPING",
     "LAMBDA_JAVA_OPTS",