Optimize lambda code and deployment

database/codes.py

@@ -214,9 +214,19 @@ class TypeOfDecisionMaker(CodeBase):
 
 
 def get_code(session: Session, code_class: Type[CodeBase], value: str) -> CodeBase:
+    """
+    Get code object by value.
+    """
     return session.query(code_class).filter_by(value=value).first()
 
 
+def get_code_uri(code_class: Type[CodeBase], value: str) -> str:
+    """
+    Get code URI by value, without querying the database.
+    """
+    return code_class(value=value).uri
+
+
 decisions_by_status = {
     # Some lifecycle statuses require decisions, some don't.
     # Plan decision code depends on lifecycle status:

infra/Makefile

@@ -17,11 +17,16 @@ update-koodistot-loader:
 	docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(prefix)-koodistot_loader:latest
 	aws lambda update-function-code --function-name $(prefix)-koodistot_loader --image-uri $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(prefix)-koodistot_loader:latest
 
+## With ryhti client, we need to publish new version and update alias for provisioned concurrency.
+## Also, we have to wait until update is complete to publish the new version.
 update-ryhti-client:
 	aws ecr get-login-password --region $(AWS_REGION) | docker login --username AWS --password-stdin $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com
 	docker tag hame-ryhti-ryhti_client:latest $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(prefix)-ryhti_client:latest
 	docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(prefix)-ryhti_client:latest
 	aws lambda update-function-code --function-name $(prefix)-ryhti_client --image-uri $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(prefix)-ryhti_client:latest
+	while aws lambda get-function --function-name hame-dev-ryhti_client --query "Configuration.LastUpdateStatus" | grep "InProgress" ; do sleep 1; echo "Waiting for AWS to complete function update..."; done
+	aws lambda publish-version --function-name $(prefix)-ryhti_client --query "Version" | \
+		xargs -I {} aws lambda update-alias --function-name $(prefix)-ryhti_client --name live --function-version {}
 
 update-mml-loader:
 	aws ecr get-login-password --region $(AWS_REGION) | docker login --username AWS --password-stdin $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com

infra/api.tf

@@ -12,17 +12,11 @@ resource "aws_api_gateway_rest_api" "lambda_api" {
             Action = "execute-api:Invoke",
             # TODO: should we only add EC2 here??
             Principal = "*",
-            Resource = [
-                "execute-api:/*"
-            ]
         },
         {
             Effect = "Deny",
             Action = "execute-api:Invoke",
             Principal = "*",
-            Resource = [
-                "execute-api:/*"
-            ],
             Condition = {
                 StringNotEquals = {
                    "aws:SourceVpce": aws_vpc_endpoint.lambda_api.id,
@@ -71,10 +65,10 @@ resource "aws_api_gateway_deployment" "api_deployment" {
   rest_api_id = aws_api_gateway_rest_api.lambda_api.id
 
   triggers = {
-    redeployment = sha1(jsonencode([
-      aws_api_gateway_resource.ryhti_client.id,
-      aws_api_gateway_method.ryhti_call.id,
-      aws_api_gateway_integration.lambda_integration.id,
+    redeployment = sha1(join(",",[
+      jsonencode(aws_api_gateway_resource.ryhti_client),
+      jsonencode(aws_api_gateway_method.ryhti_call),
+      jsonencode(aws_api_gateway_integration.lambda_integration),
     ]))
   }
 

infra/iam.tf

@@ -105,12 +105,17 @@ resource "aws_iam_policy" "lambda_update_policy" {
           "Resource": "*"
       },
       {
+        # Lambda upload user is used in Github actions for both updating the function
+        # AND invoking the db manager after update.
         "Effect" : "Allow",
         "Action" : [
+          "lambda:GetFunction",
           "lambda:CreateFunction",
           "lambda:UpdateFunctionCode",
           "lambda:InvokeFunction",
-          "lambda:UpdateFunctionConfiguration"
+          "lambda:UpdateFunctionConfiguration",
+          "lambda:PublishVersion",
+          "lambda:UpdateAlias"
          ],
         "Resource" : [
           aws_lambda_function.db_manager.arn,
@@ -223,3 +228,32 @@ resource "aws_iam_role_policy_attachment" "ssm-policy-attachment" {
   role       = aws_iam_role.ec2-role.name
   policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
 }
+
+# We need an extra policy to allow calling ryhti client URL from the EC2 server
+# without authentication or user role.
+resource "aws_iam_policy" "ec2-invoke-ryhti-client" {
+  name        = "${var.prefix}-ec2_invoke_ryhti_client_policy"
+  path        = "/"
+  description = "EC2 Ryhti client invoke policy"
+
+  policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        # Only allow calling Ryhti client lambda
+        "Effect" : "Allow",
+        "Action" : [
+          "lambda:InvokeFunction",
+         ],
+        "Resource" : [
+          aws_lambda_function.ryhti_client.arn,
+        ]
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "ec2-invoke-ryhti-client-attachment" {
+  role       = aws_iam_role.ec2-role.name
+  policy_arn = aws_iam_policy.ec2-invoke-ryhti-client.arn
+}

infra/lambda.tf

@@ -85,6 +85,7 @@ resource "aws_lambda_function" "ryhti_client" {
   function_name = "${var.prefix}-ryhti_client"
   image_uri     = "${aws_ecr_repository.ryhti_client.repository_url}:latest"
   package_type  = "Image"
+  memory_size   = 256
   timeout       = 120
 
   role = aws_iam_role.lambda_exec.arn
@@ -125,6 +126,23 @@ resource "aws_ecr_repository" "ryhti_client" {
   tags = merge(local.default_tags, { Name = "${var.prefix}-ryhti_client" })
 }
 
+# For reasons unknown, provisioned concurrency requires an alias and qualifier
+# for lambda function, just for the fun of it. $LATEST is not an alias itself.
+resource "aws_lambda_alias" "ryhti_client_live" {
+  name             = "live"
+  description      = "Alias to latest ryhti client"
+  function_name    = "${var.prefix}-ryhti_client"
+  function_version = aws_lambda_function.ryhti_client.version
+}
+
+resource "aws_lambda_provisioned_concurrency_config" "ryhti_client" {
+  function_name = aws_lambda_alias.ryhti_client_live.function_name
+  # Assume only one run at a time for now
+  provisioned_concurrent_executions = 1
+  # Should we use ARN, it changes with every lambda deploy?
+  qualifier = aws_lambda_alias.ryhti_client_live.name
+}
+
 resource "aws_lambda_permission" "cloudwatch_call_ryhti_client" {
     action = "lambda:InvokeFunction"
     function_name = aws_lambda_function.ryhti_client.function_name

requirements-dev.in

@@ -7,6 +7,7 @@ mypy
 pep8-naming
 pre-commit
 pytest
+pytest-dotenv
 python-dotenv
 pytest-docker
 requests-mock

requirements-dev.txt

@@ -78,10 +78,15 @@ pytest==7.4.4
     # via
     #   -r requirements-dev.in
     #   pytest-docker
+    #   pytest-dotenv
 pytest-docker==2.0.1
     # via -r requirements-dev.in
-python-dotenv==1.0.0
+pytest-dotenv==0.5.2
     # via -r requirements-dev.in
+python-dotenv==1.0.0
+    # via
+    #   -r requirements-dev.in
+    #   pytest-dotenv
 pyyaml==6.0.1
     # via pre-commit
 requests==2.31.0