[NRL-1922] Fixup more Sonarqube issues

.github/workflows/release.yml

@@ -60,7 +60,7 @@ jobs:
         run: |
           DOWNLOAD_URL="https://github.com/anchore/syft/releases/download/v${{ env.SYFT_VERSION }}/syft_${{ env.SYFT_VERSION }}_linux_${{ steps.os-arch.outputs.arch }}.tar.gz"
           echo "Downloading: ${DOWNLOAD_URL}"
-          curl -L -o syft.tar.gz "${DOWNLOAD_URL}"
+          curl --proto '=https' --tlsv1.2 --location --output syft.tar.gz "${DOWNLOAD_URL}"
           tar -xzf syft.tar.gz
           chmod +x syft
           # Add to PATH for subsequent steps
@@ -111,7 +111,7 @@ jobs:
 
       - name: Upload SBOM to release
         if: ${{ github.event.release.tag_name }}
-        uses: svenstaro/upload-release-action@v2.11.3
+        uses: svenstaro/upload-release-action@b98a3b12e86552593f3e4e577ca8a62aa2f3f22b # v2.11.4
         with:
           file: sbom.spdx.json
           asset_name: sbom-${{ github.event.release.tag_name }}

Dockerfile.ci-build

@@ -1,5 +1,6 @@
 FROM ubuntu:22.04
 
+
 RUN apt update && \
     apt upgrade -y && \
     DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y \
@@ -30,7 +31,8 @@ RUN apt update && \
         zip \
         zlib1g-dev && \
     apt clean && \
-    rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/* # NOSONAR (S6500) - Auto installing the defined packages is acceptable here
+
 
 WORKDIR /root
 RUN git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch v0.13.1 && \

layer/nrlf/core/validators.py

@@ -41,7 +41,6 @@ def validate_type(type_: Optional[RequestQueryType], pointer_types: List[str]) -
     return type_.root in pointer_types
 
 
-# TODO - Validate category is in set permissions once permissioning by category is done.
 def validate_category(categories: Optional[RequestQueryCategory]) -> bool:
     """
     Validates if the given category is valid.

pyproject.toml

@@ -104,7 +104,7 @@ env = [
     "AUTH_STORE=auth-store",
     "TABLE_NAME=unit-test-document-pointer"
 ]
-pythonpath = [".", "./scripts"]
+pythonpath = [".", "./scripts", "./terraform/account-wide-infrastructure/modules/glue/src"]
 
 [tool.datamodel-codegen]
 target-python-version = "3.12"

terraform/account-wide-infrastructure/dev/aws-backup.tf

@@ -1,39 +1,22 @@
 
-resource "aws_s3_bucket" "backup_reports" {
+resource "aws_s3_bucket" "backup_reports" { # NOSONAR (S6258) - Logging not required for this bucket
   bucket_prefix = "${local.prefix}-backup-reports"
 }
 
-resource "aws_s3_bucket_public_access_block" "backup_reports" {
-  bucket = aws_s3_bucket.backup_reports.id
-
-  block_public_acls       = true
-  block_public_policy     = true
-  ignore_public_acls      = true
-  restrict_public_buckets = true
-}
-
-resource "aws_s3_bucket_server_side_encryption_configuration" "backup_reports" {
-  bucket = aws_s3_bucket.backup_reports.bucket
-
-  rule {
-    apply_server_side_encryption_by_default {
-      sse_algorithm = "AES256"
-    }
-  }
-}
-
-resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" {
+resource "aws_s3_bucket_policy" "backup_reports_https_only" {
   bucket = aws_s3_bucket.backup_reports.id
 
   policy = jsonencode({
     Version = "2012-10-17"
-    Id      = "backup_reports_bucket_policy"
+    Id      = "backup_reports_https_only_policy"
     Statement = [
       {
-        Sid       = "HTTPSOnly"
-        Effect    = "Deny"
-        Principal = "*"
-        Action    = "s3:*"
+        Sid    = "HTTPSOnly"
+        Effect = "Deny"
+        Principal = {
+          "AWS" : "*"
+        }
+        Action = "s3:*"
         Resource = [
           aws_s3_bucket.backup_reports.arn,
           "${aws_s3_bucket.backup_reports.arn}/*",
@@ -43,7 +26,18 @@ resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" {
             "aws:SecureTransport" = "false"
           }
         }
-      },
+      }
+    ]
+  })
+}
+
+resource "aws_s3_bucket_policy" "backup_reports_write_access" {
+  bucket = aws_s3_bucket.backup_reports.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Id      = "backup_reports_write_access_policy"
+    Statement = [
       {
         Sid    = "AllowBackupReportsWrite"
         Effect = "Allow"
@@ -64,6 +58,24 @@ resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" {
   })
 }
 
+resource "aws_s3_bucket_public_access_block" "backup_reports" {
+  bucket = aws_s3_bucket.backup_reports.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "backup_reports" {
+  bucket = aws_s3_bucket.backup_reports.bucket
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
 
 resource "aws_s3_bucket_ownership_controls" "backup_reports" {
   bucket = aws_s3_bucket.backup_reports.id

terraform/account-wide-infrastructure/mgmt/s3.tf

@@ -1,4 +1,4 @@
-resource "aws_s3_bucket" "ci_data" {
+resource "aws_s3_bucket" "ci_data" { # NOSONAR (S6258) - Logging not required for this bucket
   bucket = "${local.prefix}--ci-data"
 }
 

terraform/account-wide-infrastructure/modules/athena/s3.tf

@@ -1,13 +1,13 @@
-resource "aws_s3_bucket" "athena" {
+resource "aws_s3_bucket" "athena" { # NOSONAR (S6258) - Logging not required for this bucket
   bucket = "${var.name_prefix}-athena"
 }
 
-resource "aws_s3_bucket_policy" "athena" {
-  bucket = "${var.name_prefix}-athena"
+resource "aws_s3_bucket_policy" "athena-https-only" {
+  bucket = aws_s3_bucket.athena.id
 
   policy = jsonencode({
     Version = "2012-10-17"
-    Id      = "athena-policy"
+    Id      = "athena-https-only-policy"
     Statement = [
       {
         Sid    = "HTTPSOnly"
@@ -25,7 +25,18 @@ resource "aws_s3_bucket_policy" "athena" {
             "aws:SecureTransport" = "false"
           }
         }
-      },
+      }
+    ]
+  })
+}
+
+resource "aws_s3_bucket_policy" "athena-access" {
+  bucket = aws_s3_bucket.athena.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Id      = "athena-access-policy"
+    Statement = [
       {
         Sid : "AllowAthenaAccess",
         Effect : "Allow",

terraform/account-wide-infrastructure/modules/glue/s3.tf

@@ -1,10 +1,10 @@
 # S3 Bucket for Raw Data
-resource "aws_s3_bucket" "source-data-bucket" {
+resource "aws_s3_bucket" "source-data-bucket" { # NOSONAR (S6258) - Logging not required for this bucket
   bucket = "${var.name_prefix}-source-data-bucket"
 }
 
 resource "aws_s3_bucket_policy" "source-data-bucket" {
-  bucket = "${var.name_prefix}-source-data-bucket"
+  bucket = aws_s3_bucket.source-data-bucket.id
 
   policy = jsonencode({
     Version = "2012-10-17"
@@ -68,18 +68,18 @@ resource "aws_s3_bucket_lifecycle_configuration" "source-data-bucket-lifecycle"
 resource "aws_s3_bucket_versioning" "source-data-bucket-versioning" {
   bucket = aws_s3_bucket.source-data-bucket.id
   versioning_configuration {
-    status = "Disabled"
+    status = "Disabled" # NOSONAR (S6252) - Versioning is not required for this bucket
   }
 }
 
 
 # S3 Bucket for Processed Data
-resource "aws_s3_bucket" "target-data-bucket" {
+resource "aws_s3_bucket" "target-data-bucket" { # NOSONAR (S6258) - Logging not required for this bucket
   bucket = "${var.name_prefix}-target-data-bucket"
 }
 
 resource "aws_s3_bucket_policy" "target-data-bucket" {
-  bucket = "${var.name_prefix}-target-data-bucket"
+  bucket = aws_s3_bucket.target-data-bucket.id
 
   policy = jsonencode({
     Version = "2012-10-17"
@@ -127,12 +127,12 @@ resource "aws_s3_bucket_public_access_block" "target-data-bucket-public-access-b
 }
 
 # S3 Bucket for Code
-resource "aws_s3_bucket" "code-bucket" {
+resource "aws_s3_bucket" "code-bucket" { # NOSONAR (S6258) - Logging not required for this bucket
   bucket = "${var.name_prefix}-code-bucket"
 }
 
 resource "aws_s3_bucket_policy" "code-bucket" {
-  bucket = "${var.name_prefix}-code-bucket"
+  bucket = aws_s3_bucket.code-bucket.id
 
   policy = jsonencode({
     Version = "2012-10-17"

terraform/account-wide-infrastructure/modules/glue/src/pipeline.py

@@ -1,7 +1,10 @@
+import os
 import time
 
 import boto3
 
+AWS_REGION = os.getenv("AWS_REGION", "eu-west-2")
+
 
 class LogPipeline:
     def __init__(
@@ -13,8 +16,8 @@ def __init__(
         target_path,
         host_prefixes,
         job_name,
-        partition_cols=[],
-        transformations=[],
+        partition_cols=None,
+        transformations=None,
     ):
         """Initialize Glue context, Spark session, logger, and paths"""
         self.glue_context = glue_context
@@ -23,12 +26,12 @@ def __init__(
         self.source_path = source_path
         self.target_path = target_path
         self.host_prefixes = host_prefixes
-        self.partition_cols = partition_cols
-        self.transformations = transformations
+        self.partition_cols = partition_cols if partition_cols else []
+        self.transformations = transformations if transformations else []
         self.glue = boto3.client(
             service_name="glue",
-            region_name="eu-west-2",
-            endpoint_url="https://glue.eu-west-2.amazonaws.com",
+            region_name=AWS_REGION,
+            endpoint_url=f"https://glue.{AWS_REGION}.amazonaws.com",
         )
         self.job_name = job_name
         self.name_prefix = "-".join(job_name.split("-")[:4])

terraform/account-wide-infrastructure/modules/glue/tests/test_pipeline.py

@@ -0,0 +1,56 @@
+from moto import mock_aws
+from pipeline import LogPipeline
+
+
+@mock_aws
+def test_pipeline_init_defaults():
+    glue_context = "mock_glue_context"
+    spark = "mock_spark_session"
+    logger = "mock_logger"
+    source_path = "s3://mock-source-path"
+    target_path = "s3://mock-target-path"
+    host_prefixes = ["host1", "host2"]
+    job_name = "test-job-name"
+
+    pipeline = LogPipeline(
+        glue_context, spark, logger, source_path, target_path, host_prefixes, job_name
+    )
+
+    assert pipeline.glue_context == glue_context
+    assert pipeline.spark == spark
+    assert pipeline.logger == logger
+    assert pipeline.source_path == source_path
+    assert pipeline.target_path == target_path
+    assert pipeline.host_prefixes == host_prefixes
+    assert pipeline.job_name == job_name
+    assert pipeline.name_prefix == "test-job-name"
+    assert pipeline.partition_cols == []
+    assert pipeline.transformations == []
+
+
+@mock_aws
+def test_pipeline_init_with_custom_values():
+    glue_context = "mock_glue_context"
+    spark = "mock_spark_session"
+    logger = "mock_logger"
+    source_path = "s3://mock-source-path"
+    target_path = "s3://mock-target-path"
+    host_prefixes = ["host1", "host2"]
+    job_name = "test-job-name"
+    partition_cols = ["col1", "col2"]
+    transformations = ["transformation1", "transformation2"]
+
+    pipeline = LogPipeline(
+        glue_context,
+        spark,
+        logger,
+        source_path,
+        target_path,
+        host_prefixes,
+        job_name,
+        partition_cols=partition_cols,
+        transformations=transformations,
+    )
+
+    assert pipeline.partition_cols == partition_cols
+    assert pipeline.transformations == transformations

terraform/account-wide-infrastructure/modules/metadata-bucket/s3.tf

@@ -1,4 +1,4 @@
-resource "aws_s3_bucket" "metadata_bucket" {
+resource "aws_s3_bucket" "metadata_bucket" { # NOSONAR (S6258) - Logging not required for this bucket
   bucket        = "${var.name_prefix}-metadata"
   force_destroy = false
 }

terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf

@@ -1,4 +1,4 @@
-resource "aws_s3_bucket" "authorization-store" {
+resource "aws_s3_bucket" "authorization-store" { # NOSONAR (S6258) - Logging not required for this bucket
   bucket        = "${var.name_prefix}-authorization-store"
   force_destroy = var.enable_bucket_force_destroy
 
@@ -9,25 +9,6 @@ resource "aws_s3_bucket" "authorization-store" {
   }
 }
 
-resource "aws_s3_bucket_public_access_block" "authorization-store-public-access-block" {
-  bucket = aws_s3_bucket.authorization-store.id
-
-  block_public_acls       = true
-  block_public_policy     = true
-  ignore_public_acls      = true
-  restrict_public_buckets = true
-}
-
-resource "aws_s3_bucket_server_side_encryption_configuration" "authorization-store" {
-  bucket = aws_s3_bucket.authorization-store.bucket
-
-  rule {
-    apply_server_side_encryption_by_default {
-      sse_algorithm = "AES256"
-    }
-  }
-}
-
 resource "aws_s3_bucket_policy" "authorization_store_bucket_policy" {
   bucket = aws_s3_bucket.authorization-store.id
 
@@ -54,6 +35,25 @@ resource "aws_s3_bucket_policy" "authorization_store_bucket_policy" {
   })
 }
 
+resource "aws_s3_bucket_public_access_block" "authorization-store-public-access-block" {
+  bucket = aws_s3_bucket.authorization-store.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "authorization-store" {
+  bucket = aws_s3_bucket.authorization-store.bucket
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
 resource "aws_s3_bucket_versioning" "authorization-store" {
   bucket = aws_s3_bucket.authorization-store.id
   versioning_configuration {