From 88dd9b3b3e5b03de245c10a2c930a086efbddd06 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Thu, 23 Mar 2023 17:19:21 +0100 Subject: [PATCH] Switch to new Lambda provider implementation (#6724) Co-authored-by: Dominik Schubert --- .circleci/config.yml | 121 +++----------- .github/workflows/pro-integration.yml | 32 ---- Makefile | 4 +- localstack/config.py | 2 + .../stream_event_source_listener.py | 148 ++++++++++-------- .../invocation/runtime_environment.py | 2 + .../awslambda/invocation/version_manager.py | 15 +- localstack/services/awslambda/lambda_utils.py | 2 +- localstack/services/awslambda/packages.py | 2 +- localstack/services/providers.py | 12 +- localstack/testing/aws/lambda_utils.py | 4 +- .../utils/cloudwatch/cloudwatch_util.py | 3 +- .../utils/container_utils/container_client.py | 17 ++ .../container_utils/docker_cmd_client.py | 24 +++ .../container_utils/docker_sdk_client.py | 14 ++ scripts/build_common_test_functions.sh | 33 ++++ .../apigateway/test_apigateway_basic.py | 7 +- .../test_apigateway_integrations.py | 12 +- .../functions/common/echo/python/Makefile | 5 - .../common/echo/python/src/requirements.txt | 1 - tests/integration/awslambda/test_lambda.py | 28 +--- .../integration/awslambda/test_lambda_api.py | 2 +- .../awslambda/test_lambda_common.py | 7 + .../awslambda/test_lambda_destinations.py | 16 +- .../test_lambda_destinations.snapshot.json | 15 +- ...test_lambda_integration_dynamodbstreams.py | 12 +- .../test_lambda_integration_kinesis.py | 8 +- .../awslambda/test_lambda_integration_sqs.py | 10 +- .../awslambda/test_lambda_runtimes.py | 11 +- .../test_lambda_runtimes.snapshot.json | 11 +- tests/integration/docker_utils/test_docker.py | 52 +++--- tests/integration/test_events.py | 6 +- tests/integration/test_integration.py | 27 ++-- tests/integration/test_logs.py | 4 +- tests/integration/test_sns.py | 12 +- tests/integration/test_sqs.py | 4 +- 36 files changed, 359 insertions(+), 326 deletions(-) create mode 100755 scripts/build_common_test_functions.sh delete mode 100644 tests/integration/awslambda/functions/common/echo/python/src/requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index fd6f4df1769ff..8be169ca0404a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,102 +78,19 @@ jobs: paths: - repo/target/coverage/ - itest-lambda-docker: + itest-lambda-legacy-local: executor: ubuntu-machine-amd64 working_directory: /tmp/workspace/repo steps: - attach_workspace: at: /tmp/workspace - run: - name: Pull Lambda runtimes - command: | - sudo useradd localstack -s /bin/bash - docker pull -q lambci/lambda:nodejs12.x - docker pull -q localstack/lambda-js:nodejs14.x - docker pull -q lambci/lambda:ruby2.7 - docker pull -q lambci/lambda:python3.6 - docker pull -q lambci/lambda:python3.7 - docker pull -q lambci/lambda:python3.8 - docker pull -q mlupin/docker-lambda:python3.9 - docker pull -q lambci/lambda:dotnetcore3.1 - docker pull -q mlupin/docker-lambda:dotnet6 - docker pull -q lambci/lambda:provided - docker pull -q lambci/lambda:java8 - docker pull -q lambci/lambda:java8.al2 - docker pull -q lambci/lambda:java11 - - run: - name: Test Docker client - environment: - TEST_PATH: "tests/integration/docker_utils" - TEST_SKIP_LOCALSTACK_START: 1 - PYTEST_ARGS: "--reruns 2 --junitxml=target/reports/docker-client.xml -o junit_suite_name='docker-client'" - COVERAGE_ARGS: "-p" - command: make test-coverage - - run: - name: Test 'docker' Lambda executor - environment: - LAMBDA_EXECUTOR: "docker" - USE_SSL: 1 - TEST_PATH: "tests/integration/awslambda/ tests/integration/test_integration.py tests/integration/apigateway/test_apigateway_basic.py" - PYTEST_ARGS: "--reruns 2 --junitxml=target/reports/lambda-docker.xml -o junit_suite_name='lambda-docker'" - COVERAGE_ARGS: "-p" - command: make test-coverage - - run: - name: Test 'docker-reuse' Lambda executor - environment: - LAMBDA_EXECUTOR: "docker-reuse" - TEST_PATH: "tests/integration/awslambda/ tests/integration/test_integration.py tests/integration/apigateway/test_apigateway_basic.py" - PYTEST_ARGS: "--reruns 2 --junitxml=target/reports/lambda-docker-reuse.xml -o junit_suite_name='lambda-docker-reuse'" - COVERAGE_ARGS: "-p" - command: make test-coverage - - run: - name: Store coverage results - command: mv .coverage.* target/coverage/ - - persist_to_workspace: - root: - /tmp/workspace - paths: - - repo/target/coverage/ - - store_test_results: - path: target/reports/ - - itest-lambda-provider: - executor: ubuntu-machine-amd64 - working_directory: /tmp/workspace/repo - steps: - - attach_workspace: - at: /tmp/workspace - - run: - name: Test ASF Lambda provider + name: Test 'local' Lambda executor environment: - PROVIDER_OVERRIDE_LAMBDA: "asf" - TEST_PATH: "tests/integration/awslambda/test_lambda.py tests/integration/awslambda/test_lambda_api.py tests/integration/awslambda/test_lambda_common.py tests/integration/awslambda/test_lambda_integration_sqs.py tests/integration/cloudformation/resources/test_lambda.py tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py tests/integration/awslambda/test_lambda_integration_kinesis.py tests/integration/awslambda/test_lambda_developer_tools.py tests/integration/test_network_configuration.py::TestLambda" - PYTEST_ARGS: "--reruns 3 --junitxml=target/reports/lambda_asf.xml -o junit_suite_name='lambda_asf'" - COVERAGE_ARGS: "-p" - command: make test-coverage - - run: - name: Store coverage results - command: mv .coverage.* target/coverage/ - - persist_to_workspace: - root: - /tmp/workspace - paths: - - repo/target/coverage/ - - store_test_results: - path: target/reports/ - - itest-s3-asf-provider: - executor: ubuntu-machine-amd64 - working_directory: /tmp/workspace/repo - steps: - - attach_workspace: - at: /tmp/workspace - - run: - name: Test ASF S3 provider - environment: - PROVIDER_OVERRIDE_S3: "asf" - TEST_PATH: "tests/integration/s3/ tests/integration/test_network_configuration.py::TestS3" - PYTEST_ARGS: "--reruns 3 --junitxml=target/reports/s3_asf.xml -o junit_suite_name='s3_asf'" + LAMBDA_EXECUTOR: "local" + PROVIDER_OVERRIDE_LAMBDA: "legacy" + TEST_PATH: "tests/integration/awslambda/ tests/integration/test_integration.py tests/integration/apigateway/test_apigateway_basic.py tests/integration/cloudformation/resources/test_lambda.py" + PYTEST_ARGS: "--reruns 2 --junitxml=target/reports/lambda-docker.xml -o junit_suite_name='legacy-lambda-local'" COVERAGE_ARGS: "-p" command: make test-coverage - run: @@ -263,13 +180,20 @@ jobs: image: << parameters.machine_image >> resource_class: << parameters.resource_class >> working_directory: /tmp/workspace/repo - parallelism: 2 + parallelism: 4 steps: - attach_workspace: at: /tmp/workspace - run: name: Load docker image command: docker load -i target/localstack-docker-image-<< parameters.platform >>.tar + - when: + condition: + equal: [ "amd64", << parameters.platform >>] + steps: + - run: + name: pre-build lambda common test packages + command: ./scripts/build_common_test_functions.sh `pwd`/tests/integration/awslambda/functions/common - run: name: Run integration tests # circleci split returns newline separated list, so `tr` is necessary to prevent problems in the Makefile @@ -278,6 +202,7 @@ jobs: PYTEST_ARGS="-o junit_family=legacy --junitxml=target/reports/test-report-<< parameters.platform >>-${CIRCLE_NODE_INDEX}.xml" \ COVERAGE_FILE="target/coverage/.coverage.<< parameters.platform >>.${CIRCLE_NODE_INDEX}" \ TEST_PATH=$TEST_FILES \ + DEBUG=1 \ make docker-run-tests - store_test_results: path: target/reports/ @@ -426,13 +351,7 @@ workflows: - preflight: requires: - install - - itest-lambda-docker: - requires: - - preflight - - itest-lambda-provider: - requires: - - preflight - - itest-s3-asf-provider: + - itest-lambda-legacy-local: requires: - preflight - itest-sfn-v2-provider: @@ -477,9 +396,7 @@ workflows: - docker-build-amd64 - report: requires: - - itest-lambda-docker - - itest-lambda-provider - - itest-s3-asf-provider + - itest-lambda-legacy-local - itest-sfn-v2-provider - docker-test-amd64 - docker-test-arm64 @@ -490,9 +407,7 @@ workflows: branches: only: master requires: - - itest-lambda-docker - - itest-lambda-provider - - itest-s3-asf-provider + - itest-lambda-legacy-local - itest-sfn-v2-provider - docker-test-amd64 - docker-test-arm64 diff --git a/.github/workflows/pro-integration.yml b/.github/workflows/pro-integration.yml index 899fb006962fc..0c6ba2114bd8e 100644 --- a/.github/workflows/pro-integration.yml +++ b/.github/workflows/pro-integration.yml @@ -178,48 +178,16 @@ jobs: env: DEBUG: 1 DNS_ADDRESS: 0 - LAMBDA_EXECUTOR: "local" LOCALSTACK_API_KEY: "test" AWS_SECRET_ACCESS_KEY: "test" AWS_ACCESS_KEY_ID: "test" AWS_DEFAULT_REGION: "us-east-1" - HOST_TMP_FOLDER: /tmp/localstack PYTEST_LOGLEVEL: debug run: | # Remove the host tmp folder (might contain remnant files with different permissions) sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp source .venv/bin/activate python -m pytest --capture=no --reruns 2 --durations=10 --junitxml=target/reports/pytest.xml ../localstack/tests/integration/ - - name: Run Lambda Tests for lambda executor docker - env: - DEBUG: 0 - DNS_ADDRESS: 0 - LAMBDA_EXECUTOR: "docker" - LOCALSTACK_API_KEY: "test" - HOST_TMP_FOLDER: /tmp/localstack - run: | - # Remove the host tmp folder (might contain remnant files with different permissions) - sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp - source .venv/bin/activate - python -m pytest --reruns 2 --durations=10 --show-capture=no --junitxml=target/reports/lambda-docker.xml -o junit_suite_name='lambda-docker' \ - ../localstack/tests/integration/awslambda/ \ - ../localstack/tests/integration/test_integration.py \ - ../localstack/tests/integration/apigateway/test_apigateway_basic.py - - name: Run Lambda Tests for lambda executor docker-reuse - env: - DEBUG: 1 - DNS_ADDRESS: 0 - LAMBDA_EXECUTOR: "docker-reuse" - LOCALSTACK_API_KEY: "test" - HOST_TMP_FOLDER: /tmp/localstack - run: | - # Remove the host tmp folder (might contain remnant files with different permissions) - sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp - source .venv/bin/activate - python -m pytest --reruns 2 --durations=10 --show-capture=no --junitxml=target/reports/lambda-docker-reuse.xml -o junit_suite_name='lambda-docker-reuse' \ - ../localstack/tests/integration/awslambda/ \ - ../localstack/tests/integration/test_integration.py \ - ../localstack/tests/integration/apigateway/test_apigateway_basic.py - name: Publish LocalStack Community Integration Test Results uses: EnricoMi/publish-unit-test-result-action@v1 if: always() diff --git a/Makefile b/Makefile index 4cdbfa8deea53..1d80e7c138073 100644 --- a/Makefile +++ b/Makefile @@ -151,9 +151,9 @@ docker-create-push-manifests: ## Create and push manifests for a docker image (d docker-run-tests: ## Initializes the test environment and runs the tests in a docker container # Remove argparse and dataclasses to fix https://github.com/pytest-dev/pytest/issues/5594 # Note: running "install-test-only" below, to avoid pulling in [runtime] extras from transitive dependencies - docker run -e LOCALSTACK_INTERNAL_TEST_COLLECT_METRIC=1 --entrypoint= -v `pwd`/tests/:/opt/code/localstack/tests/ -v `pwd`/target/:/opt/code/localstack/target/ \ + docker run -e LOCALSTACK_INTERNAL_TEST_COLLECT_METRIC=1 --entrypoint= -v `pwd`/tests/:/opt/code/localstack/tests/ -v `pwd`/target/:/opt/code/localstack/target/ -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/localstack:/var/lib/localstack \ $(IMAGE_NAME) \ - bash -c "make install-test-only && pip uninstall -y argparse dataclasses && DEBUG=$(DEBUG) LAMBDA_EXECUTOR=local PYTEST_LOGLEVEL=debug PYTEST_ARGS='$(PYTEST_ARGS)' COVERAGE_FILE='$(COVERAGE_FILE)' TEST_PATH='$(TEST_PATH)' make test-coverage" + bash -c "make install-test-only && pip uninstall -y argparse dataclasses && DEBUG=$(DEBUG) PYTEST_LOGLEVEL=debug PYTEST_ARGS='$(PYTEST_ARGS)' COVERAGE_FILE='$(COVERAGE_FILE)' TEST_PATH='$(TEST_PATH)' LAMBDA_IGNORE_ARCHITECTURE=1 LAMBDA_INIT_POST_INVOKE_WAIT_MS=50 make test-coverage" docker-run: ## Run Docker image locally ($(VENV_RUN); bin/localstack start) diff --git a/localstack/config.py b/localstack/config.py index 14c0b82e6bc3c..d290f3076915e 100644 --- a/localstack/config.py +++ b/localstack/config.py @@ -784,6 +784,7 @@ def get_gateway_listen(gateway_listen: str) -> List[HostAndPort]: LAMBDA_INIT_BOOTSTRAP_PATH = os.environ.get("LAMBDA_INIT_BOOTSTRAP_PATH") LAMBDA_INIT_DELVE_PATH = os.environ.get("LAMBDA_INIT_DELVE_PATH") LAMBDA_INIT_DELVE_PORT = int(os.environ.get("LAMBDA_INIT_DELVE_PORT") or 40000) +LAMBDA_INIT_POST_INVOKE_WAIT_MS = os.environ.get("LAMBDA_INIT_POST_INVOKE_WAIT_MS") # Alternative user or empty string to skip dropping privileges LAMBDA_INIT_USER = os.environ.get("LAMBDA_INIT_USER") @@ -926,6 +927,7 @@ def get_gateway_listen(gateway_listen: str) -> List[HostAndPort]: "LAMBDA_INIT_BOOTSTRAP_PATH", "LAMBDA_INIT_DELVE_PATH", "LAMBDA_INIT_DELVE_PORT", + "LAMBDA_INIT_POST_INVOKE_WAIT", "LAMBDA_INIT_USER", "LAMBDA_RUNTIME_IMAGE_MAPPING", "LAMBDA_JAVA_OPTS", diff --git a/localstack/services/awslambda/event_source_listeners/stream_event_source_listener.py b/localstack/services/awslambda/event_source_listeners/stream_event_source_listener.py index 1996d0d4bcd7b..ea5456ea6ee4d 100644 --- a/localstack/services/awslambda/event_source_listeners/stream_event_source_listener.py +++ b/localstack/services/awslambda/event_source_listeners/stream_event_source_listener.py @@ -184,71 +184,84 @@ def _listen_to_shard_and_invoke_lambda(self, params: Dict): more than max_num_retries """ # TODO: These values will never get updated if the event source mapping configuration changes :( - function_arn = params["function_arn"] - stream_arn = params["stream_arn"] - batch_size = params["batch_size"] - parallelization_factor = params["parallelization_factor"] - lock_discriminator = params["lock_discriminator"] - shard_id = params["shard_id"] - stream_client = params["stream_client"] - shard_iterator = params["shard_iterator"] - failure_destination = params["failure_destination"] - max_num_retries = params["max_num_retries"] - num_invocation_failures = 0 - - while lock_discriminator in self._STREAM_LISTENER_THREADS: - records_response = stream_client.get_records( - ShardIterator=shard_iterator, Limit=batch_size - ) - records = records_response.get("Records") - event_filter_criterias = self._get_lambda_event_filters_for_arn( - function_arn, stream_arn - ) - if len(event_filter_criterias) > 0: - records = filter_stream_records(records, event_filter_criterias) - - should_get_next_batch = True - if records: - payload = self._create_lambda_event_payload(stream_arn, records, shard_id=shard_id) - is_invocation_successful, status_code = self._invoke_lambda( - function_arn, payload, lock_discriminator, parallelization_factor + try: + function_arn = params["function_arn"] + stream_arn = params["stream_arn"] + batch_size = params["batch_size"] + parallelization_factor = params["parallelization_factor"] + lock_discriminator = params["lock_discriminator"] + shard_id = params["shard_id"] + stream_client = params["stream_client"] + shard_iterator = params["shard_iterator"] + failure_destination = params["failure_destination"] + max_num_retries = params["max_num_retries"] + num_invocation_failures = 0 + + while lock_discriminator in self._STREAM_LISTENER_THREADS: + records_response = stream_client.get_records( + ShardIterator=shard_iterator, Limit=batch_size + ) + records = records_response.get("Records") + event_filter_criterias = self._get_lambda_event_filters_for_arn( + function_arn, stream_arn ) - if is_invocation_successful: - should_get_next_batch = True - else: - num_invocation_failures += 1 - if num_invocation_failures >= max_num_retries: + if len(event_filter_criterias) > 0: + records = filter_stream_records(records, event_filter_criterias) + + should_get_next_batch = True + if records: + payload = self._create_lambda_event_payload( + stream_arn, records, shard_id=shard_id + ) + is_invocation_successful, status_code = self._invoke_lambda( + function_arn, payload, lock_discriminator, parallelization_factor + ) + if is_invocation_successful: should_get_next_batch = True - if failure_destination: - first_rec = records[0] - last_rec = records[-1] - ( - first_seq_num, - last_seq_num, - ) = self._get_starting_and_ending_sequence_numbers(first_rec, last_rec) - ( - first_arrival_time, - last_arrival_time, - ) = self._get_first_and_last_arrival_time(first_rec, last_rec) - self._send_to_failure_destination( - shard_id, - first_seq_num, - last_seq_num, - stream_arn, - function_arn, - num_invocation_failures, - status_code, - batch_size, - first_arrival_time, - last_arrival_time, - failure_destination, - ) else: - should_get_next_batch = False - if should_get_next_batch: - shard_iterator = records_response["NextShardIterator"] - num_invocation_failures = 0 - time.sleep(self._POLL_INTERVAL_SEC) + num_invocation_failures += 1 + if num_invocation_failures >= max_num_retries: + should_get_next_batch = True + if failure_destination: + first_rec = records[0] + last_rec = records[-1] + ( + first_seq_num, + last_seq_num, + ) = self._get_starting_and_ending_sequence_numbers( + first_rec, last_rec + ) + ( + first_arrival_time, + last_arrival_time, + ) = self._get_first_and_last_arrival_time(first_rec, last_rec) + self._send_to_failure_destination( + shard_id, + first_seq_num, + last_seq_num, + stream_arn, + function_arn, + num_invocation_failures, + status_code, + batch_size, + first_arrival_time, + last_arrival_time, + failure_destination, + ) + else: + should_get_next_batch = False + if should_get_next_batch: + shard_iterator = records_response["NextShardIterator"] + num_invocation_failures = 0 + time.sleep(self._POLL_INTERVAL_SEC) + except Exception as e: + LOG.error( + "Error while listening to shard / executing lambda with params %s: %s", + params, + e, + exc_info=LOG.isEnabledFor(logging.DEBUG), + ) + raise def _send_to_failure_destination( self, @@ -329,7 +342,16 @@ def _monitor_stream_event_sources(self, *args): max_num_retries = source.get("MaximumRetryAttempts", -1) if max_num_retries < 0: max_num_retries = math.inf - stream_description = self._get_stream_description(stream_client, stream_arn) + try: + stream_description = self._get_stream_description(stream_client, stream_arn) + except Exception as e: + LOG.error( + "Cannot describe target stream %s of event source mapping %s: %s", + stream_arn, + mapping_uuid, + e, + ) + continue if stream_description["StreamStatus"] not in {"ENABLED", "ACTIVE"}: continue shard_ids = [shard["ShardId"] for shard in stream_description["Shards"]] diff --git a/localstack/services/awslambda/invocation/runtime_environment.py b/localstack/services/awslambda/invocation/runtime_environment.py index 0ac118e39e7b0..7eb6c7e93a95e 100644 --- a/localstack/services/awslambda/invocation/runtime_environment.py +++ b/localstack/services/awslambda/invocation/runtime_environment.py @@ -153,6 +153,8 @@ def get_environment_variables(self) -> Dict[str, str]: env_vars["LOCALSTACK_USER"] = "" if config.LAMBDA_INIT_USER: env_vars["LOCALSTACK_USER"] = config.LAMBDA_INIT_USER + if config.LAMBDA_INIT_POST_INVOKE_WAIT_MS: + env_vars["LOCALSTACK_POST_INVOKE_WAIT_MS"] = int(config.LAMBDA_INIT_POST_INVOKE_WAIT_MS) return env_vars # Lifecycle methods diff --git a/localstack/services/awslambda/invocation/version_manager.py b/localstack/services/awslambda/invocation/version_manager.py index 2ad05658c6e7b..3ac6b854f1c0f 100644 --- a/localstack/services/awslambda/invocation/version_manager.py +++ b/localstack/services/awslambda/invocation/version_manager.py @@ -8,6 +8,7 @@ import uuid from concurrent.futures import Future, ThreadPoolExecutor from datetime import datetime +from math import ceil from queue import Queue from typing import TYPE_CHECKING, Dict, List, Optional, Union @@ -19,6 +20,7 @@ StateReasonCode, TooManyRequestsException, ) +from localstack.aws.connect import connect_to from localstack.services.awslambda.invocation.lambda_models import ( Function, FunctionVersion, @@ -86,19 +88,22 @@ class LogHandler: _thread: Optional[FuncThread] _shutdown_event: threading.Event - def __init__(self, role_arn: str) -> None: + def __init__(self, role_arn: str, region: str) -> None: self.role_arn = role_arn + self.region = region self.log_queue = Queue() self._shutdown_event = threading.Event() self._thread = None def run_log_loop(self, *args, **kwargs) -> None: - # TODO: create client + logs_client = connect_to(region_name=self.region).logs while not self._shutdown_event.is_set(): log_item = self.log_queue.get() if log_item is QUEUE_SHUTDOWN: return - store_cloudwatch_logs(log_item.log_group, log_item.log_stream, log_item.logs) + store_cloudwatch_logs( + log_item.log_group, log_item.log_stream, log_item.logs, logs_client=logs_client + ) def start_subscriber(self) -> None: self._thread = FuncThread(self.run_log_loop, name="log_handler") @@ -151,7 +156,7 @@ def __init__( self.function_version = function_version self.function = function self.lambda_service = lambda_service - self.log_handler = LogHandler(function_version.config.role) + self.log_handler = LogHandler(function_version.config.role, function_version.id.region) # invocation tracking self.running_invocations = {} @@ -594,7 +599,7 @@ def process_event_destinations( time_passed = datetime.now() - last_invoke_time enough_time_for_retry = ( event_invoke_config.maximum_event_age_in_seconds - and time_passed.seconds + delay_queue_invoke_seconds + and ceil(time_passed.total_seconds()) + delay_queue_invoke_seconds <= event_invoke_config.maximum_event_age_in_seconds ) diff --git a/localstack/services/awslambda/lambda_utils.py b/localstack/services/awslambda/lambda_utils.py index e030bfaf7ccf1..6d5d444b683fc 100644 --- a/localstack/services/awslambda/lambda_utils.py +++ b/localstack/services/awslambda/lambda_utils.py @@ -54,7 +54,7 @@ # default handler and runtime LAMBDA_DEFAULT_HANDLER = "handler.handler" -LAMBDA_DEFAULT_RUNTIME = LAMBDA_RUNTIME_PYTHON37 # FIXME (?) +LAMBDA_DEFAULT_RUNTIME = LAMBDA_RUNTIME_PYTHON39 # FIXME (?) LAMBDA_DEFAULT_STARTING_POSITION = "LATEST" # List of Dotnet Lambda runtime names diff --git a/localstack/services/awslambda/packages.py b/localstack/services/awslambda/packages.py index 2c9fbb9cdd04c..a781659dacf5a 100644 --- a/localstack/services/awslambda/packages.py +++ b/localstack/services/awslambda/packages.py @@ -9,7 +9,7 @@ LAMBDA_RUNTIME_INIT_URL = "https://github.com/localstack/lambda-runtime-init/releases/download/{version}/aws-lambda-rie-{arch}" -LAMBDA_RUNTIME_DEFAULT_VERSION = "v0.1.14-pre" +LAMBDA_RUNTIME_DEFAULT_VERSION = "v0.1.15-pre" # GO Lambda runtime GO_RUNTIME_VERSION = "0.4.0" diff --git a/localstack/services/providers.py b/localstack/services/providers.py index 7ff540b035785..9376de2766650 100644 --- a/localstack/services/providers.py +++ b/localstack/services/providers.py @@ -139,8 +139,8 @@ def kms(): return Service.for_provider(provider) -@aws_provider(api="lambda") -def awslambda(): +@aws_provider(api="lambda", name="legacy") +def awslambda_legacy(): from localstack.services.awslambda import lambda_starter return Service( @@ -165,6 +165,14 @@ def awslambda_v1(): ) +@aws_provider(api="lambda") +def awslambda(): + from localstack.services.awslambda.provider import LambdaProvider + + provider = LambdaProvider() + return Service.for_provider(provider) + + @aws_provider(api="lambda", name="asf") def awslambda_asf(): from localstack.services.awslambda.provider import LambdaProvider diff --git a/localstack/testing/aws/lambda_utils.py b/localstack/testing/aws/lambda_utils.py index d42f53d68c718..3961a334deb69 100644 --- a/localstack/testing/aws/lambda_utils.py +++ b/localstack/testing/aws/lambda_utils.py @@ -126,10 +126,10 @@ def is_old_local_executor() -> bool: def is_old_provider(): return os.environ.get("TEST_TARGET") != "AWS_CLOUD" and os.environ.get( "PROVIDER_OVERRIDE_LAMBDA" - ) not in ["asf", "v2"] + ) in ["legacy", "v1"] def is_new_provider(): return os.environ.get("TEST_TARGET") != "AWS_CLOUD" and os.environ.get( "PROVIDER_OVERRIDE_LAMBDA" - ) in ["asf", "v2"] + ) not in ["legacy", "v1"] diff --git a/localstack/utils/cloudwatch/cloudwatch_util.py b/localstack/utils/cloudwatch/cloudwatch_util.py index cea34b50b4245..0547afb769503 100644 --- a/localstack/utils/cloudwatch/cloudwatch_util.py +++ b/localstack/utils/cloudwatch/cloudwatch_util.py @@ -96,9 +96,10 @@ def store_cloudwatch_logs( log_output, start_time=None, auto_create_group: Optional[bool] = True, + logs_client=None, ): start_time = start_time or int(time.time() * 1000) - logs_client = aws_stack.connect_to_service("logs") + logs_client = logs_client or aws_stack.connect_to_service("logs") log_output = to_str(log_output) if auto_create_group: diff --git a/localstack/utils/container_utils/container_client.py b/localstack/utils/container_utils/container_client.py index 47009d035a4fb..98539ea6288e6 100644 --- a/localstack/utils/container_utils/container_client.py +++ b/localstack/utils/container_utils/container_client.py @@ -617,6 +617,23 @@ def inspect_image(self, image_name: str, pull: bool = True) -> Dict[str, Union[D """ pass + @abstractmethod + def create_network(self, network_name: str) -> str: + """ + Creates a network with the given name + :param network_name: Name of the network + :return Network ID + """ + pass + + @abstractmethod + def delete_network(self, network_name: str) -> None: + """ + Delete a network with the given name + :param network_name: Name of the network + """ + pass + @abstractmethod def inspect_network(self, network_name: str) -> Dict[str, Union[Dict, str]]: """Get detailed attributes of an network. diff --git a/localstack/utils/container_utils/docker_cmd_client.py b/localstack/utils/container_utils/docker_cmd_client.py index c0049b71abf0b..f46e640877f70 100644 --- a/localstack/utils/container_utils/docker_cmd_client.py +++ b/localstack/utils/container_utils/docker_cmd_client.py @@ -404,6 +404,30 @@ def inspect_image(self, image_name: str, pull: bool = True) -> Dict[str, Union[D return self.inspect_image(image_name, pull=False) raise NoSuchImage(image_name=e.object_id) + def create_network(self, network_name: str) -> str: + cmd = self._docker_cmd() + cmd += ["network", "create", network_name] + try: + return run(cmd).strip() + except subprocess.CalledProcessError as e: + raise ContainerException( + "Docker process returned with errorcode %s" % e.returncode, e.stdout, e.stderr + ) from e + + def delete_network(self, network_name: str) -> None: + cmd = self._docker_cmd() + cmd += ["network", "rm", network_name] + try: + run(cmd) + except subprocess.CalledProcessError as e: + stdout_str = to_str(e.stdout) + if re.match(r".*network (.*) not found.*", stdout_str): + raise NoSuchNetwork(network_name=network_name) + else: + raise ContainerException( + "Docker process returned with errorcode %s" % e.returncode, e.stdout, e.stderr + ) from e + def inspect_network(self, network_name: str) -> Dict[str, Union[Dict, str]]: try: return self._inspect_object(network_name) diff --git a/localstack/utils/container_utils/docker_sdk_client.py b/localstack/utils/container_utils/docker_sdk_client.py index eef65dbc76f3a..9b3b17450404c 100644 --- a/localstack/utils/container_utils/docker_sdk_client.py +++ b/localstack/utils/container_utils/docker_sdk_client.py @@ -327,6 +327,20 @@ def inspect_image(self, image_name: str, pull: bool = True) -> Dict[str, Union[D except APIError as e: raise ContainerException() from e + def create_network(self, network_name: str) -> None: + try: + return self.client().networks.create(name=network_name).id + except APIError as e: + raise ContainerException() from e + + def delete_network(self, network_name: str) -> None: + try: + return self.client().networks.get(network_name).remove() + except NotFound: + raise NoSuchNetwork(network_name) + except APIError as e: + raise ContainerException() from e + def inspect_network(self, network_name: str) -> Dict[str, Union[Dict, str]]: try: return self.client().networks.get(network_name).attrs diff --git a/scripts/build_common_test_functions.sh b/scripts/build_common_test_functions.sh new file mode 100755 index 0000000000000..15db1172c70f4 --- /dev/null +++ b/scripts/build_common_test_functions.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e + +COMMON_DIR=$1 +cd $COMMON_DIR + +for scenario in */ ; do + [ -L "${scenario%/}" ] && continue + cd "$scenario" + FULL_SCENARIO_PATH=`pwd` + + for runtime in */ ; do + [ -L "${runtime%/}" ] && continue + + BUILD_PATH="$FULL_SCENARIO_PATH/$runtime" + echo -n "Making ${scenario}.${runtime} in $BUILD_PATH: " + cd "$BUILD_PATH" + + # skip if zip file exists, otherwise run makefile + [ -f "handler.zip" ] && echo "found handler.zip => skipping" && continue + echo -n "building ..." + make build >/dev/null + + # if no zipfile, package build folder + [ -f "handler.zip" ] && echo "found handler.zip => skipping" && continue + echo -n "packaging handler.zip ..." + cd ./build && zip -r ../handler.zip . && cd - + echo "DONE" + done + + cd $COMMON_DIR +done diff --git a/tests/integration/apigateway/test_apigateway_basic.py b/tests/integration/apigateway/test_apigateway_basic.py index 091f85aa57b96..f5b2d2a4b5588 100644 --- a/tests/integration/apigateway/test_apigateway_basic.py +++ b/tests/integration/apigateway/test_apigateway_basic.py @@ -40,6 +40,7 @@ from localstack.utils.files import load_file from localstack.utils.http import safe_requests as requests from localstack.utils.json import clone +from localstack.utils.platform import get_arch from localstack.utils.strings import short_uid, to_str from localstack.utils.sync import retry from tests.integration.apigateway.apigateway_fixtures import ( @@ -68,7 +69,6 @@ from tests.integration.awslambda.test_lambda import ( TEST_LAMBDA_AWS_PROXY, TEST_LAMBDA_HTTP_RUST, - TEST_LAMBDA_LIBS, TEST_LAMBDA_NODEJS, TEST_LAMBDA_NODEJS_APIGW_502, TEST_LAMBDA_NODEJS_APIGW_INTEGRATION, @@ -1791,9 +1791,7 @@ def connect_api_gateway_to_http( @staticmethod def create_lambda_function(fn_name): - testutil.create_lambda_function( - handler_file=TEST_LAMBDA_PYTHON, libs=TEST_LAMBDA_LIBS, func_name=fn_name - ) + testutil.create_lambda_function(handler_file=TEST_LAMBDA_PYTHON, func_name=fn_name) lambda_client = aws_stack.create_external_boto_client("lambda") lambda_client.get_waiter("function_active_v2").wait(FunctionName=fn_name) @@ -2134,6 +2132,7 @@ def test_import_swagger_api(apigateway_client): @pytest.mark.skipif(not use_docker(), reason="Rust lambdas cannot be executed in local executor") +@pytest.mark.skipif(get_arch() == "arm64", reason="Lambda only available for amd64") def test_apigateway_rust_lambda( apigateway_client, create_rest_apigw, diff --git a/tests/integration/apigateway/test_apigateway_integrations.py b/tests/integration/apigateway/test_apigateway_integrations.py index fc79e4c725ca2..abd2fcd18d1d1 100644 --- a/tests/integration/apigateway/test_apigateway_integrations.py +++ b/tests/integration/apigateway/test_apigateway_integrations.py @@ -29,7 +29,11 @@ create_rest_resource_method, ) from tests.integration.apigateway.conftest import DEFAULT_STAGE_NAME -from tests.integration.awslambda.test_lambda import TEST_LAMBDA_AWS_PROXY, TEST_LAMBDA_HELLO_WORLD +from tests.integration.awslambda.test_lambda import ( + TEST_LAMBDA_AWS_PROXY, + TEST_LAMBDA_HELLO_WORLD, + TEST_LAMBDA_LIBS, +) @pytest.mark.skip_offline @@ -816,7 +820,11 @@ def handler(event, context): "SecurityGroupIds": [security_group], } create_lambda_function( - func_name=func_name, handler_file=lambda_code, timeout=10, VpcConfig=vpc_config + func_name=func_name, + handler_file=lambda_code, + libs=TEST_LAMBDA_LIBS, + timeout=10, + VpcConfig=vpc_config, ) # create resource policy diff --git a/tests/integration/awslambda/functions/common/echo/python/Makefile b/tests/integration/awslambda/functions/common/echo/python/Makefile index aa51d4fea78e8..5b36633677f02 100644 --- a/tests/integration/awslambda/functions/common/echo/python/Makefile +++ b/tests/integration/awslambda/functions/common/echo/python/Makefile @@ -1,12 +1,7 @@ build: mkdir build; \ - python -m venv ./build/.venv; \ - . ./build/.venv/bin/activate; \ cp -r ./src/* build/; \ - pip install --target ./build -r ./build/requirements.txt; \ - deactivate; \ - rm -r ./build/.venv; \ find ./build -exec touch -t 200001010100.00 {} \; .PHONY: build diff --git a/tests/integration/awslambda/functions/common/echo/python/src/requirements.txt b/tests/integration/awslambda/functions/common/echo/python/src/requirements.txt deleted file mode 100644 index f2293605cf1b0..0000000000000 --- a/tests/integration/awslambda/functions/common/echo/python/src/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests diff --git a/tests/integration/awslambda/test_lambda.py b/tests/integration/awslambda/test_lambda.py index 4aaa55e0b5162..2159d02e694cd 100644 --- a/tests/integration/awslambda/test_lambda.py +++ b/tests/integration/awslambda/test_lambda.py @@ -28,7 +28,7 @@ from localstack.utils import files, platform, testutil from localstack.utils.files import load_file from localstack.utils.http import safe_requests -from localstack.utils.platform import is_arm_compatible, standardized_arch +from localstack.utils.platform import get_arch, is_arm_compatible, standardized_arch from localstack.utils.strings import short_uid, to_bytes, to_str from localstack.utils.sync import retry, wait_until from localstack.utils.testutil import create_lambda_archive @@ -96,7 +96,7 @@ Runtime.python3_8, Runtime.python3_9, ] - if not is_old_provider() or use_docker() + if (not is_old_provider() or use_docker()) and get_arch() != "arm64" else [Runtime.python3_9] ) NODE_TEST_RUNTIMES = ( @@ -110,27 +110,15 @@ Runtime.java8_al2, Runtime.java11, ] - if not is_old_provider() or use_docker() + if (not is_old_provider() or use_docker()) and get_arch() != "arm64" else [Runtime.java11] ) - -PROVIDED_TEST_RUNTIMES = [ - Runtime.provided, - # TODO remove skip once we use correct images - pytest.param( - Runtime.provided_al2, - marks=pytest.mark.skipif( - is_old_provider(), reason="curl missing in provided.al2 lambci image" - ), - ), -] - TEST_LAMBDA_LIBS = [ "requests", "psutil", "urllib3", - "chardet", + "charset_normalizer", "certifi", "idna", "pip", @@ -339,6 +327,7 @@ class TestLambdaBehavior: "$..Payload.paths._var_task_uid", ], ) + @pytest.mark.skipif(get_arch() == "arm64", reason="Cannot inspect x86 runtime on arm") @pytest.mark.aws_validated def test_runtime_introspection_x86(self, lambda_client, create_lambda_function, snapshot): func_name = f"test_lambda_x86_{short_uid()}" @@ -439,10 +428,7 @@ def test_ignore_architecture( assert lambda_arch == native_arch @pytest.mark.skipif(is_old_provider(), reason="unsupported in old provider") - @pytest.mark.skipif( - not is_arm_compatible() and not is_aws(), - reason="ARM architecture not supported on this host", - ) + @pytest.mark.skip # TODO remove once is_arch_compatible checks work properly @pytest.mark.aws_validated def test_mixed_architecture(self, lambda_client, create_lambda_function): """Test emulation and interaction of lambda functions with different architectures. @@ -1098,7 +1084,6 @@ def test_invocation_with_qualifier( zip_file = create_lambda_archive( load_file(TEST_LAMBDA_PYTHON), get_content=True, - libs=TEST_LAMBDA_LIBS, runtime=Runtime.python3_9, ) s3_client.upload_fileobj(BytesIO(zip_file), s3_bucket, bucket_key) @@ -1154,7 +1139,6 @@ def test_upload_lambda_from_s3( zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON), get_content=True, - libs=TEST_LAMBDA_LIBS, runtime=Runtime.python3_9, ) s3_client.upload_fileobj(BytesIO(zip_file), s3_bucket, bucket_key) diff --git a/tests/integration/awslambda/test_lambda_api.py b/tests/integration/awslambda/test_lambda_api.py index abb0f2d30e6c2..40c70884c5dd0 100644 --- a/tests/integration/awslambda/test_lambda_api.py +++ b/tests/integration/awslambda/test_lambda_api.py @@ -2953,7 +2953,7 @@ def test_create_multiple_lambda_permissions( create_lambda_function( func_name=function_name, - runtime=Runtime.python3_7, + runtime=Runtime.python3_9, handler_file=TEST_LAMBDA_PYTHON_ECHO, ) diff --git a/tests/integration/awslambda/test_lambda_common.py b/tests/integration/awslambda/test_lambda_common.py index 755fde49b88c3..3be5bb3ebd392 100644 --- a/tests/integration/awslambda/test_lambda_common.py +++ b/tests/integration/awslambda/test_lambda_common.py @@ -1,6 +1,7 @@ import json import logging import os +import platform import time import zipfile @@ -42,6 +43,9 @@ def snapshot_transformers(snapshot): condition=is_old_provider(), reason="Local executor does not support the majority of the runtimes", ) +@pytest.mark.skipif( + condition=platform.machine() != "x86_64", reason="build process doesn't support arm64 right now" +) class TestLambdaRuntimesCommon: """ Directly correlates to the structure found in tests.integration.awslambda.functions.common @@ -248,6 +252,9 @@ def test_runtime_wrapper_invoke(self, lambda_client, multiruntime_lambda, snapsh condition=is_old_provider(), reason="Local executor does not support the majority of the runtimes", ) +@pytest.mark.skipif( + condition=platform.machine() != "x86_64", reason="build process doesn't support arm64 right now" +) class TestLambdaCallingLocalstack: @pytest.mark.multiruntime( scenario="endpointinjection", diff --git a/tests/integration/awslambda/test_lambda_destinations.py b/tests/integration/awslambda/test_lambda_destinations.py index efeda39beea81..5b73c7520e10f 100644 --- a/tests/integration/awslambda/test_lambda_destinations.py +++ b/tests/integration/awslambda/test_lambda_destinations.py @@ -12,7 +12,7 @@ from localstack.utils.strings import short_uid, to_bytes, to_str from localstack.utils.sync import retry, wait_until from tests.integration.awslambda.functions import lambda_integration -from tests.integration.awslambda.test_lambda import TEST_LAMBDA_LIBS, TEST_LAMBDA_PYTHON +from tests.integration.awslambda.test_lambda import TEST_LAMBDA_PYTHON class TestLambdaDLQ: @@ -48,7 +48,6 @@ def test_dead_letter_queue( create_lambda_response = create_lambda_function( handler_file=TEST_LAMBDA_PYTHON, func_name=lambda_name, - libs=TEST_LAMBDA_LIBS, runtime=Runtime.python3_9, DeadLetterConfig={"TargetArn": queue_arn}, role=lambda_su_role, @@ -102,6 +101,7 @@ class TestLambdaDestinationSqs: "$..FunctionArn", "$..approximateInvokeCount", "$..stackTrace", + "$..Messages..Body.responsePayload.requestId", ], ) @pytest.mark.parametrize( @@ -135,6 +135,7 @@ def test_assess_lambda_destination_invocation( queue_arn = sqs_queue_arn(queue_url) create_lambda_function( handler_file=TEST_LAMBDA_PYTHON, + runtime=Runtime.python3_9, func_name=lambda_name, role=lambda_su_role, ) @@ -193,6 +194,7 @@ def test_lambda_destination_default_retries( queue_arn = sqs_queue_arn(queue_url) create_lambda_function( handler_file=TEST_LAMBDA_PYTHON, + runtime=Runtime.python3_9, func_name=lambda_name, role=lambda_su_role, ) @@ -269,7 +271,7 @@ def test_retries( create_lambda_function( handler_file=os.path.join(os.path.dirname(__file__), "./functions/lambda_echofail.py"), func_name=fn_name, - libs=TEST_LAMBDA_LIBS, + runtime=Runtime.python3_9, role=lambda_su_role, ) lambda_client.put_function_event_invoke_config( @@ -378,7 +380,6 @@ def test_maxeventage( create_lambda_function( handler_file=os.path.join(os.path.dirname(__file__), "./functions/lambda_echofail.py"), func_name=fn_name, - libs=TEST_LAMBDA_LIBS, role=lambda_su_role, ) lambda_client.put_function_event_invoke_config( @@ -435,7 +436,10 @@ def get_msg_from_q(): msg = retry(get_msg_from_q, retries=15, sleep=3) snapshot.match("no_retry_failure_message", msg) - assert get_filtered_event_count() == 1 + def _assert_event_count(count: int): + assert get_filtered_event_count() == count + + retry(_assert_event_count, retries=5, sleep=1, count=1) # 1 attempt in total (no retries) # now we increase the max event age to give it a bit of a buffer for the actual lambda execution (60s + 30s buffer = 90s) # one retry should now be attempted since there's enough time left @@ -460,7 +464,7 @@ def get_msg_from_q(): msg_retried = retry(get_msg_from_q, retries=15, sleep=3) snapshot.match("single_retry_failure_message", msg_retried) - assert get_filtered_event_count() == 2 # 2 attempts in total (1 retry) + retry(_assert_event_count, retries=5, sleep=1, count=2) # 2 attempts in total (1 retry) # class TestLambdaDestinationSns: diff --git a/tests/integration/awslambda/test_lambda_destinations.snapshot.json b/tests/integration/awslambda/test_lambda_destinations.snapshot.json index 0f2e96bfdd0f4..48a1ee495ed91 100644 --- a/tests/integration/awslambda/test_lambda_destinations.snapshot.json +++ b/tests/integration/awslambda/test_lambda_destinations.snapshot.json @@ -121,7 +121,7 @@ } }, "tests/integration/awslambda/test_lambda_destinations.py::TestLambdaDestinationSqs::test_assess_lambda_destination_invocation[payload0]": { - "recorded-date": "22-03-2023, 18:42:11", + "recorded-date": "23-03-2023, 00:15:17", "recorded-content": { "put_function_event_invoke_config": { "DestinationConfig": { @@ -184,7 +184,7 @@ } }, "tests/integration/awslambda/test_lambda_destinations.py::TestLambdaDestinationSqs::test_assess_lambda_destination_invocation[payload1]": { - "recorded-date": "22-03-2023, 18:42:16", + "recorded-date": "23-03-2023, 00:15:20", "recorded-content": { "put_function_event_invoke_config": { "DestinationConfig": { @@ -226,6 +226,7 @@ "responsePayload": { "errorMessage": "Test exception (this is intentional)", "errorType": "Exception", + "requestId": "", "stackTrace": [ " File \"/var/task/handler.py\", line 54, in handler\n raise Exception(\"Test exception (this is intentional)\")\n" ] @@ -403,7 +404,7 @@ } }, "tests/integration/awslambda/test_lambda_destinations.py::TestLambdaDestinationSqs::test_retries": { - "recorded-date": "27-02-2023, 16:12:55", + "recorded-date": "22-03-2023, 18:50:18", "recorded-content": { "queue_destination_payload": { "Messages": [ @@ -434,6 +435,7 @@ "responsePayload": { "errorMessage": "intentional failure", "errorType": "Exception", + "requestId": "", "stackTrace": [ " File \"/var/task/handler.py\", line 7, in handler\n raise Exception(\"intentional failure\")\n" ] @@ -452,7 +454,7 @@ } }, "tests/integration/awslambda/test_lambda_destinations.py::TestLambdaDestinationSqs::test_maxeventage": { - "recorded-date": "27-02-2023, 16:14:27", + "recorded-date": "22-03-2023, 18:52:05", "recorded-content": { "no_retry_failure_message": { "Attributes": { @@ -481,6 +483,7 @@ "responsePayload": { "errorMessage": "intentional failure", "errorType": "Exception", + "requestId": "", "stackTrace": [ " File \"/var/task/handler.py\", line 7, in handler\n raise Exception(\"intentional failure\")\n" ] @@ -517,6 +520,7 @@ "responsePayload": { "errorMessage": "intentional failure", "errorType": "Exception", + "requestId": "", "stackTrace": [ " File \"/var/task/handler.py\", line 7, in handler\n raise Exception(\"intentional failure\")\n" ] @@ -529,7 +533,7 @@ } }, "tests/integration/awslambda/test_lambda_destinations.py::TestLambdaDestinationSqs::test_lambda_destination_default_retries": { - "recorded-date": "22-03-2023, 19:02:29", + "recorded-date": "23-03-2023, 00:18:52", "recorded-content": { "put_function_event_invoke_config": { "DestinationConfig": { @@ -570,6 +574,7 @@ "responsePayload": { "errorMessage": "Test exception (this is intentional)", "errorType": "Exception", + "requestId": "", "stackTrace": [ " File \"/var/task/handler.py\", line 54, in handler\n raise Exception(\"Test exception (this is intentional)\")\n" ] diff --git a/tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py b/tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py index f44e0e8ed3362..01a4b5eb71460 100644 --- a/tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py +++ b/tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py @@ -4,11 +4,9 @@ import pytest +from localstack.aws.api.lambda_ import Runtime from localstack.services.awslambda.lambda_api import INVALID_PARAMETER_VALUE_EXCEPTION -from localstack.services.awslambda.lambda_utils import ( - LAMBDA_RUNTIME_PYTHON37, - LAMBDA_RUNTIME_PYTHON39, -) +from localstack.services.awslambda.lambda_utils import LAMBDA_RUNTIME_PYTHON39 from localstack.testing.aws.lambda_utils import ( _await_dynamodb_table_active, _await_event_source_mapping_enabled, @@ -197,7 +195,7 @@ def test_disabled_dynamodb_event_source_mapping( create_lambda_function( func_name=function_name, handler_file=TEST_LAMBDA_PYTHON_ECHO, - runtime=LAMBDA_RUNTIME_PYTHON37, + runtime=Runtime.python3_9, role=lambda_su_role, ) dynamodb_create_table_result = dynamodb_create_table( @@ -462,7 +460,7 @@ def test_dynamodb_event_filter( create_lambda_function( handler_file=TEST_LAMBDA_PYTHON_ECHO, func_name=function_name, - runtime=LAMBDA_RUNTIME_PYTHON37, + runtime=Runtime.python3_9, role=lambda_su_role, ) table_creation_response = dynamodb_create_table(table_name=table_name, partition_key="id") @@ -559,7 +557,7 @@ def test_dynamodb_invalid_event_filter( create_lambda_function( handler_file=TEST_LAMBDA_PYTHON_ECHO, func_name=function_name, - runtime=LAMBDA_RUNTIME_PYTHON37, + runtime=Runtime.python3_9, role=lambda_su_role, ) dynamodb_create_table(table_name=table_name, partition_key="id") diff --git a/tests/integration/awslambda/test_lambda_integration_kinesis.py b/tests/integration/awslambda/test_lambda_integration_kinesis.py index d5ce42f628560..db239094a7256 100644 --- a/tests/integration/awslambda/test_lambda_integration_kinesis.py +++ b/tests/integration/awslambda/test_lambda_integration_kinesis.py @@ -7,10 +7,8 @@ import pytest from localstack import config -from localstack.services.awslambda.lambda_utils import ( - LAMBDA_RUNTIME_PYTHON37, - LAMBDA_RUNTIME_PYTHON39, -) +from localstack.aws.api.lambda_ import Runtime +from localstack.services.awslambda.lambda_utils import LAMBDA_RUNTIME_PYTHON39 from localstack.testing.aws.lambda_utils import ( _await_event_source_mapping_enabled, _await_event_source_mapping_state, @@ -387,7 +385,7 @@ def test_kinesis_event_source_mapping_with_on_failure_destination_config( create_lambda_function( handler_file=TEST_LAMBDA_PYTHON, func_name=function_name, - runtime=LAMBDA_RUNTIME_PYTHON37, + runtime=Runtime.python3_9, role=role_arn, ) kinesis_client.create_stream(StreamName=kinesis_name, ShardCount=1) diff --git a/tests/integration/awslambda/test_lambda_integration_sqs.py b/tests/integration/awslambda/test_lambda_integration_sqs.py index 4281593dca8de..499f6c2a1a8c3 100644 --- a/tests/integration/awslambda/test_lambda_integration_sqs.py +++ b/tests/integration/awslambda/test_lambda_integration_sqs.py @@ -5,12 +5,12 @@ import pytest from botocore.exceptions import ClientError +from localstack.aws.api.lambda_ import Runtime from localstack.services.awslambda.lambda_api import ( BATCH_SIZE_RANGES, INVALID_PARAMETER_VALUE_EXCEPTION, ) from localstack.services.awslambda.lambda_utils import ( - LAMBDA_RUNTIME_PYTHON37, LAMBDA_RUNTIME_PYTHON38, LAMBDA_RUNTIME_PYTHON39, ) @@ -297,7 +297,7 @@ def test_sqs_queue_as_lambda_dead_letter_queue( lambda_creation_response = create_lambda_function( func_name=function_name, handler_file=TEST_LAMBDA_PYTHON, - runtime=LAMBDA_RUNTIME_PYTHON37, + runtime=Runtime.python3_9, role=lambda_su_role, DeadLetterConfig={"TargetArn": dlq_queue_arn}, ) @@ -878,7 +878,7 @@ def test_sqs_event_source_mapping( create_lambda_function( func_name=function_name, handler_file=TEST_LAMBDA_PYTHON_ECHO, - runtime=LAMBDA_RUNTIME_PYTHON37, + runtime=Runtime.python3_9, role=lambda_su_role, ) queue_url_1 = sqs_create_queue(QueueName=queue_name_1) @@ -984,7 +984,7 @@ def test_sqs_event_filter( create_lambda_function( func_name=function_name, handler_file=TEST_LAMBDA_PYTHON_ECHO, - runtime=LAMBDA_RUNTIME_PYTHON37, + runtime=Runtime.python3_9, role=lambda_su_role, ) queue_url_1 = sqs_create_queue(QueueName=queue_name_1) @@ -1060,7 +1060,7 @@ def test_sqs_invalid_event_filter( create_lambda_function( func_name=function_name, handler_file=TEST_LAMBDA_PYTHON_ECHO, - runtime=LAMBDA_RUNTIME_PYTHON37, + runtime=Runtime.python3_9, role=lambda_su_role, ) queue_url_1 = sqs_create_queue(QueueName=queue_name_1) diff --git a/tests/integration/awslambda/test_lambda_runtimes.py b/tests/integration/awslambda/test_lambda_runtimes.py index 29803227d7c27..2b0ac302609a6 100644 --- a/tests/integration/awslambda/test_lambda_runtimes.py +++ b/tests/integration/awslambda/test_lambda_runtimes.py @@ -24,7 +24,6 @@ PYTHON_TEST_RUNTIMES, TEST_LAMBDA_JAVA_MULTIPLE_HANDLERS, TEST_LAMBDA_JAVA_WITH_LIB, - TEST_LAMBDA_LIBS, TEST_LAMBDA_NODEJS_ES6, TEST_LAMBDA_PYTHON, TEST_LAMBDA_PYTHON_UNHANDLED_ERROR, @@ -309,7 +308,14 @@ def check_logs(): retry(check_logs, retries=20) @pytest.mark.skip_snapshot_verify( - condition=is_old_provider, paths=["$..Code.RepositoryType", "$..Tags"] + condition=is_old_provider, + paths=[ + "$..Code.RepositoryType", + "$..Tags", + "$..Configuration.RuntimeVersionConfig", + "$..Configuration.SnapStart", + "$..Statement.Condition.ArnLike", + ], ) # TODO maybe snapshot payload as well def test_java_lambda_subscribe_sns_topic( @@ -429,7 +435,6 @@ def test_handler_in_submodule(self, lambda_client, create_lambda_function, runti zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON), get_content=True, - libs=TEST_LAMBDA_LIBS, runtime=runtime, file_name="localstack_package/def/main.py", ) diff --git a/tests/integration/awslambda/test_lambda_runtimes.snapshot.json b/tests/integration/awslambda/test_lambda_runtimes.snapshot.json index 625ae9f572d9d..9229fd06960dc 100644 --- a/tests/integration/awslambda/test_lambda_runtimes.snapshot.json +++ b/tests/integration/awslambda/test_lambda_runtimes.snapshot.json @@ -922,7 +922,7 @@ } }, "tests/integration/awslambda/test_lambda_runtimes.py::TestJavaRuntimes::test_java_lambda_subscribe_sns_topic": { - "recorded-date": "09-09-2022, 22:37:37", + "recorded-date": "22-03-2023, 17:40:56", "recorded-content": { "get-function": { "Code": { @@ -952,6 +952,13 @@ "RevisionId": "", "Role": "arn:aws:iam::111111111111:role/", "Runtime": "java11", + "RuntimeVersionConfig": { + "RuntimeVersionArn": "arn:aws:lambda:::runtime:" + }, + "SnapStart": { + "ApplyOn": "None", + "OptimizationStatus": "Off" + }, "State": "Active", "Timeout": 30, "TracingConfig": { @@ -975,7 +982,7 @@ "Resource": "arn:aws:lambda::111111111111:function:", "Condition": { "ArnLike": { - "AWS:SourceArn": "arn:aws:sns::111111111111:" + "AWS:SourceArn": "arn:aws:sns::111111111111:" } } }, diff --git a/tests/integration/docker_utils/test_docker.py b/tests/integration/docker_utils/test_docker.py index 7898055867e7a..e4e510d0e02df 100644 --- a/tests/integration/docker_utils/test_docker.py +++ b/tests/integration/docker_utils/test_docker.py @@ -4,14 +4,13 @@ import os import re import time -from subprocess import CalledProcessError from typing import NamedTuple import pytest from localstack import config from localstack.config import in_docker -from localstack.utils.common import is_ipv4_address, safe_run, save_file, short_uid, to_str +from localstack.utils.common import is_ipv4_address, save_file, short_uid, to_str from localstack.utils.container_utils.container_client import ( AccessDenied, ContainerClient, @@ -85,7 +84,7 @@ def _create_container(image_name: str, **kwargs): @pytest.fixture -def create_network(): +def create_network(docker_client: ContainerClient): """ Uses the factory as fixture pattern to wrap the creation of networks as a factory that removes the networks after the fixture is cleaned up. @@ -93,7 +92,7 @@ def create_network(): networks = [] def _create_network(network_name: str): - network_id = safe_run([config.DOCKER_CMD, "network", "create", network_name]).strip() + network_id = docker_client.create_network(network_name=network_name) networks.append(network_id) return network_id @@ -102,9 +101,9 @@ def _create_network(network_name: str): for network in networks: try: LOG.debug("Removing network %s", network) - safe_run([config.DOCKER_CMD, "network", "remove", network]) - except CalledProcessError: - pass + docker_client.delete_network(network_name=network) + except ContainerException as e: + LOG.debug("Error while cleaning up network %s: %s", network, e) class TestDockerClient: @@ -311,7 +310,9 @@ def test_get_network_multiple_networks( ): network_name = f"test-network-{short_uid()}" network_id = create_network(network_name) - safe_run(["docker", "network", "connect", network_id, dummy_container.container_id]) + docker_client.connect_container_to_network( + network_name=network_id, container_name_or_id=dummy_container.container_id + ) docker_client.start_container(dummy_container.container_id) networks = docker_client.get_networks(dummy_container.container_id) assert network_name in networks @@ -323,7 +324,9 @@ def test_get_container_ip_for_network( ): network_name = f"test-network-{short_uid()}" network_id = create_network(network_name) - safe_run(["docker", "network", "connect", network_id, dummy_container.container_id]) + docker_client.connect_container_to_network( + network_name=network_id, container_name_or_id=dummy_container.container_id + ) docker_client.start_container(dummy_container.container_id) result_bridge_network = docker_client.get_container_ipv4_for_network( container_name_or_id=dummy_container.container_id, container_network="bridge" @@ -685,7 +688,7 @@ def test_get_container_entrypoint_non_existing_image(self, docker_client: Contai def test_get_container_entrypoint_not_pulled_image(self, docker_client: ContainerClient): try: docker_client.get_image_cmd("alpine", pull=False) - safe_run([config.DOCKER_CMD, "rmi", "alpine"]) + docker_client.remove_image("alpine") except ContainerException: pass entrypoint = docker_client.get_image_entrypoint("alpine") @@ -698,7 +701,7 @@ def test_get_container_command(self, docker_client: ContainerClient): def test_get_container_command_not_pulled_image(self, docker_client: ContainerClient): try: docker_client.get_image_cmd("alpine", pull=False) - safe_run([config.DOCKER_CMD, "rmi", "alpine"]) + docker_client.remove_image("alpine") except ContainerException: pass command = docker_client.get_image_cmd("alpine") @@ -865,7 +868,7 @@ def test_stream_logs(self, docker_client: ContainerClient): def test_pull_docker_image(self, docker_client: ContainerClient): try: docker_client.get_image_cmd("alpine", pull=False) - safe_run([config.DOCKER_CMD, "rmi", "alpine"]) + docker_client.remove_image("alpine") except ContainerException: pass with pytest.raises(NoSuchImage): @@ -882,7 +885,7 @@ def test_pull_non_existent_docker_image(self, docker_client: ContainerClient): def test_pull_docker_image_with_tag(self, docker_client: ContainerClient): try: docker_client.get_image_cmd("alpine", pull=False) - safe_run([config.DOCKER_CMD, "rmi", "alpine"]) + docker_client.remove_image("alpine") except ContainerException: pass with pytest.raises(NoSuchImage): @@ -895,7 +898,7 @@ def test_pull_docker_image_with_tag(self, docker_client: ContainerClient): def test_pull_docker_image_with_hash(self, docker_client: ContainerClient): try: docker_client.get_image_cmd("alpine", pull=False) - safe_run([config.DOCKER_CMD, "rmi", "alpine"]) + docker_client.remove_image("alpine") except ContainerException: pass with pytest.raises(NoSuchImage): @@ -918,8 +921,8 @@ def test_pull_docker_image_with_hash(self, docker_client: ContainerClient): @pytest.mark.skip_offline def test_run_container_automatic_pull(self, docker_client: ContainerClient): try: - safe_run([config.DOCKER_CMD, "rmi", "alpine"]) - except CalledProcessError: + docker_client.remove_image("alpine") + except ContainerException: pass message = "test message" stdout, _ = docker_client.run_container("alpine", command=["echo", message], remove=True) @@ -1008,8 +1011,8 @@ def test_build_image( @pytest.mark.skip_offline def test_run_container_non_existent_image(self, docker_client: ContainerClient): try: - safe_run([config.DOCKER_CMD, "rmi", "alpine"]) - except CalledProcessError: + docker_client.remove_image("alpine") + except ContainerException: pass with pytest.raises(NoSuchImage): stdout, _ = docker_client.run_container( @@ -1035,8 +1038,8 @@ def test_is_container_running(self, docker_client: ContainerClient, dummy_contai @pytest.mark.skip_offline def test_docker_image_names(self, docker_client: ContainerClient): try: - safe_run([config.DOCKER_CMD, "rmi", "alpine"]) - except CalledProcessError: + docker_client.remove_image("alpine") + except ContainerException: pass assert "alpine:latest" not in docker_client.get_docker_image_names() assert "alpine" not in docker_client.get_docker_image_names() @@ -1220,6 +1223,15 @@ def test_remove_image_raises_for_nonexistent_image(self, docker_client: Containe class TestDockerNetworking: + def test_network_lifecycle(self, docker_client: ContainerClient): + network_name = f"test-network-{short_uid()}" + network_id = docker_client.create_network(network_name=network_name) + assert network_name == docker_client.inspect_network(network_name=network_name)["Name"] + assert network_id == docker_client.inspect_network(network_name=network_name)["Id"] + docker_client.delete_network(network_name=network_name) + with pytest.raises(NoSuchNetwork): + docker_client.inspect_network(network_name=network_name) + def test_get_container_ip_with_network( self, docker_client: ContainerClient, create_container, create_network ): diff --git a/tests/integration/test_events.py b/tests/integration/test_events.py index 74180840e249d..09b2f4b8f6c93 100644 --- a/tests/integration/test_events.py +++ b/tests/integration/test_events.py @@ -1653,7 +1653,7 @@ def test_put_target_id_validation( @pytest.mark.aws_validated def test_should_ignore_schedules_for_put_event( - self, create_lambda_function, lambda_client, events_client, logs_client + self, create_lambda_function, lambda_client, events_client, logs_client, cleanups ): """Regression test for https://github.com/localstack/localstack/issues/7847""" fn_name = f"test-event-fn-{short_uid()}" @@ -1675,9 +1675,13 @@ def test_should_ignore_schedules_for_put_event( events_client.put_rule( Name="ScheduledLambda", ScheduleExpression="rate(1 minute)" ) # every minute, can't go lower than that + cleanups.append(lambda: events_client.delete_rule(Name="ScheduledLambda")) events_client.put_targets( Rule="ScheduledLambda", Targets=[{"Id": "calllambda1", "Arn": fn_arn}] ) + cleanups.append( + lambda: events_client.remove_targets(Rule="ScheduledLambda", Ids=["calllambda1"]) + ) events_client.put_events( Entries=[ diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 17f71dd193497..48f089d8a5718 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -28,7 +28,6 @@ from .awslambda.functions import lambda_integration from .awslambda.test_lambda import ( PYTHON_TEST_RUNTIMES, - TEST_LAMBDA_LIBS, TEST_LAMBDA_PUT_ITEM_FILE, TEST_LAMBDA_PYTHON, TEST_LAMBDA_PYTHON_ECHO, @@ -53,7 +52,7 @@ def handler(event, *args): @pytest.fixture(scope="class") -def scheduled_test_lambda(lambda_client): +def scheduled_test_lambda(lambda_client, events_client): # Note: create scheduled Lambda here - assertions will be run in test_scheduled_lambda() below.. # create test Lambda @@ -68,12 +67,14 @@ def scheduled_test_lambda(lambda_client): # create scheduled Lambda function rule_name = f"rule-{short_uid()}" - events = aws_stack.create_external_boto_client("events") - events.put_rule(Name=rule_name, ScheduleExpression="rate(1 minutes)") - events.put_targets(Rule=rule_name, Targets=[{"Id": f"target-{short_uid()}", "Arn": func_arn}]) + target_id = f"target-{short_uid()}" + events_client.put_rule(Name=rule_name, ScheduleExpression="rate(1 minutes)") + events_client.put_targets(Rule=rule_name, Targets=[{"Id": target_id, "Arn": func_arn}]) yield scheduled_lambda_name + events_client.remove_targets(Rule=rule_name, Ids=[target_id]) + events_client.delete_rule(Name=rule_name) testutil.delete_lambda_function(scheduled_lambda_name) @@ -207,10 +208,12 @@ def _assert_objects_created(): def test_lambda_streams_batch_and_transactions( self, dynamodb_client, + lambda_client, dynamodbstreams_client, kinesis_create_stream, dynamodb_create_table, create_lambda_function, + cleanups, ): ddb_lease_table_suffix = "-kclapp2" table_name = short_uid() + "lsbat" + ddb_lease_table_suffix @@ -251,13 +254,15 @@ def process_records(records, shard_id): # deploy test lambda connected to DynamoDB Stream create_lambda_function( handler_file=TEST_LAMBDA_PYTHON, - libs=TEST_LAMBDA_LIBS, func_name=lambda_ddb_name, - event_source_arn=ddb_event_source_arn, - starting_position="TRIM_HORIZON", - delete=True, envvars={"KINESIS_STREAM_NAME": stream_name}, ) + uuid = lambda_client.create_event_source_mapping( + FunctionName=lambda_ddb_name, + EventSourceArn=ddb_event_source_arn, + StartingPosition="TRIM_HORIZON", + )["UUID"] + cleanups.append(lambda: lambda_client.delete_event_source_mapping(UUID=uuid)) # submit a batch with writes dynamodb_client.batch_write_item( @@ -552,9 +557,7 @@ def test_kinesis_lambda_forward_chain( s3_client.create_bucket(Bucket=TEST_BUCKET_NAME) # deploy test lambdas connected to Kinesis streams - zip_file = testutil.create_lambda_archive( - load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS - ) + zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True) lambda_1_resp = create_lambda_function( func_name=lambda1_name, zip_file=zip_file, diff --git a/tests/integration/test_logs.py b/tests/integration/test_logs.py index e797f6a4e68ee..0a41f068c9481 100644 --- a/tests/integration/test_logs.py +++ b/tests/integration/test_logs.py @@ -13,8 +13,7 @@ from localstack.utils import testutil from localstack.utils.aws import arns from localstack.utils.common import now_utc, poll_condition, retry, short_uid - -from .awslambda.test_lambda import TEST_LAMBDA_LIBS, TEST_LAMBDA_PYTHON_ECHO +from tests.integration.awslambda.test_lambda import TEST_LAMBDA_PYTHON_ECHO logs_role = { "Statement": { @@ -234,7 +233,6 @@ def test_put_subscription_filter_lambda( test_lambda_name = f"test-lambda-function-{short_uid()}" create_lambda_function( handler_file=TEST_LAMBDA_PYTHON_ECHO, - libs=TEST_LAMBDA_LIBS, func_name=test_lambda_name, runtime=Runtime.python3_9, ) diff --git a/tests/integration/test_sns.py b/tests/integration/test_sns.py index 53a2f59c71429..8e28530a2edf2 100644 --- a/tests/integration/test_sns.py +++ b/tests/integration/test_sns.py @@ -18,7 +18,6 @@ from localstack import config from localstack.aws.accounts import get_aws_account_id from localstack.aws.api.lambda_ import Runtime -from localstack.services.awslambda.lambda_utils import LAMBDA_RUNTIME_PYTHON37 from localstack.services.sns.constants import PLATFORM_ENDPOINT_MSGS_ENDPOINT from localstack.services.sns.provider import SnsProvider from localstack.testing.aws.util import is_aws_cloud @@ -30,7 +29,7 @@ from localstack.utils.testutil import check_expected_lambda_log_events_length from .awslambda.functions import lambda_integration -from .awslambda.test_lambda import TEST_LAMBDA_LIBS, TEST_LAMBDA_PYTHON, TEST_LAMBDA_PYTHON_ECHO +from .awslambda.test_lambda import TEST_LAMBDA_PYTHON, TEST_LAMBDA_PYTHON_ECHO LOG = logging.getLogger(__name__) @@ -108,7 +107,7 @@ def test_python_lambda_subscribe_sns_topic( lambda_creation_response = create_lambda_function( func_name=function_name, handler_file=TEST_LAMBDA_PYTHON_ECHO, - runtime=Runtime.python3_7, + runtime=Runtime.python3_9, role=lambda_su_role, ) lambda_arn = lambda_creation_response["CreateFunctionResponse"]["FunctionArn"] @@ -700,7 +699,7 @@ def test_sns_topic_as_lambda_dead_letter_queue( lambda_creation_response = create_lambda_function( func_name=function_name, handler_file=TEST_LAMBDA_PYTHON, - runtime=LAMBDA_RUNTIME_PYTHON37, + runtime=Runtime.python3_9, role=lambda_su_role, DeadLetterConfig={"TargetArn": dlq_topic_arn}, ) @@ -831,9 +830,8 @@ def test_redrive_policy_lambda_subscription( lambda_name = f"test-{short_uid()}" lambda_arn = create_lambda_function( func_name=lambda_name, - libs=TEST_LAMBDA_LIBS, handler_file=TEST_LAMBDA_PYTHON, - runtime=LAMBDA_RUNTIME_PYTHON37, + runtime=Runtime.python3_9, role=lambda_su_role, )["CreateFunctionResponse"]["FunctionArn"] @@ -3618,7 +3616,7 @@ def test_delivery_lambda( lambda_creation_response = create_lambda_function( func_name=function_name, handler_file=TEST_LAMBDA_PYTHON_ECHO, - runtime=Runtime.python3_7, + runtime=Runtime.python3_9, role=lambda_su_role, ) lambda_arn = lambda_creation_response["CreateFunctionResponse"]["FunctionArn"] diff --git a/tests/integration/test_sqs.py b/tests/integration/test_sqs.py index b3330508b8e9e..ffd9f947255b1 100644 --- a/tests/integration/test_sqs.py +++ b/tests/integration/test_sqs.py @@ -20,7 +20,7 @@ from localstack.utils.common import poll_condition, retry, short_uid, to_str from .awslambda.functions import lambda_integration -from .awslambda.test_lambda import TEST_LAMBDA_LIBS, TEST_LAMBDA_PYTHON +from .awslambda.test_lambda import TEST_LAMBDA_PYTHON TEST_POLICY = """ { @@ -946,7 +946,6 @@ def test_delete_message_batch_from_lambda( lambda_name = f"lambda-{short_uid()}" create_lambda_function( func_name=lambda_name, - libs=TEST_LAMBDA_LIBS, handler_file=TEST_LAMBDA_PYTHON, runtime=Runtime.python3_9, ) @@ -2229,7 +2228,6 @@ def test_dead_letter_queue_execution_lambda_mapping_preserves_id( lambda_name = "lambda-{}".format(short_uid()) create_lambda_function( func_name=lambda_name, - libs=TEST_LAMBDA_LIBS, handler_file=TEST_LAMBDA_PYTHON, runtime=Runtime.python3_9, )