chore: refactor docker images (#4561)

.github/workflows/ci.yaml

@@ -24,7 +24,9 @@ jobs:
     name: Build and test
     runs-on: ubuntu-latest
     outputs:
-      gitsha: ${{ steps.setDockerSHAs.outputs.gitsha }}
+      image_sha: ${{ steps.setDockerSHAs.outputs.image_sha }}
+      image_version: ${{ steps.setDockerSHAs.outputs.image_version }}
+      build_version: ${{ steps.setDockerSHAs.outputs.build_version }}
     timeout-minutes: 20
     steps:
       - name: Checkout
@@ -82,8 +84,17 @@ jobs:
           # We lint/test/build affected but in order to build image, we need to ensure everything
           # is built.  Build anything that hasn't been built yet (takes advantage of nx cache
           # for anything already built from above)
-          npx nx run-many -t build --all
-          echo "GITSHA=`echo $(echo ${{ steps.setNxSHAs.outputs.head }} | cut -c1-8).${{ github.run_number }}.${{ github.run_attempt }}`" >> "$GITHUB_OUTPUT"
+          npx nx run-many -t build
+
+          # build values for docker image tags
+          VERSION_SUFFIX=${{ github.run_number }}.${{ github.run_attempt }}
+          FULL_SHA=${{ steps.setNxSHAs.outputs.head }}
+          # instead of using rev-parse we always just get the first 8 characters but we'll ensure
+          # its unique (just in case 8 is not enough) by appending the VERSION_SUFFIX
+          SHORT_SHA=$(echo ${FULL_SHA} | cut -c1-8)
+          echo "IMAGE_SHA=${FULL_SHA}" >> "$GITHUB_OUTPUT"
+          echo "IMAGE_VERSION=${FULL_SHA}.${VERSION_SUFFIX}" >> "$GITHUB_OUTPUT"
+          echo "BUILD_VERSION=${SHORT_SHA}.${VERSION_SUFFIX}" >> "$GITHUB_OUTPUT"
 
       - name: Set up docker buildx
         uses: docker/setup-buildx-action@v3
@@ -93,11 +104,11 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           tags: |
-            type=raw,value=${{ steps.setDockerSHAs.outputs.gitsha }}
+            type=raw,value=${{ steps.setDockerSHAs.outputs.image_version }}
             type=ref,event=branch
             type=ref,event=pr
           labels: |
-            org.opencontainers.image.version=${{ steps.setDockerSHAs.outputs.gitsha }}
+            org.opencontainers.image.version=${{ steps.setDockerSHAs.outputs.image_version }}
 
       - name: Cache for docker
         id: cache
@@ -125,7 +136,7 @@ jobs:
           load: true
           file: ./apps/platform/Dockerfile
           build-args: |
-            GITSHA=${{ steps.setDockerSHAs.outputs.gitsha }}
+            BUILD_VERSION=${{ steps.setDockerSHAs.outputs.build_version }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           outputs: |
@@ -140,7 +151,7 @@ jobs:
 
       - name: Integration and e2e tests
         env:
-          GITSHA: ${{ steps.setDockerSHAs.outputs.gitsha }}
+          APP_IMAGE: ${{ steps.setDockerSHAs.outputs.image_version }}
         run: |
           ./scripts/seed-etc-hosts.sh
 
@@ -274,29 +285,31 @@ jobs:
       - name: Push image to ECR
         id: pushImageToECR
         env:
-          LOCAL_IMAGE_TAG: ${{ needs.build.outputs.gitsha }}
-          REGISTRY_IMAGE_TAG: ${{ steps.login-ecr.outputs.registry }}/uesio:${{ needs.build.outputs.gitsha }}
+          IMAGE_VERSION_TAG: ${{ needs.build.outputs.image_version }}
+          IMAGE_SHA_TAG: ${{ needs.build.outputs.image_sha }}-latest
+          REGISTRY_IMAGE_TAG_BASE: ${{ steps.login-ecr.outputs.registry }}/uesio
         run: |
-          echo "LOCAL_IMAGE_TAG=${LOCAL_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
-          echo "REGISTRY_IMAGE_TAG=${REGISTRY_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+          REGISTRY_SHA_TAG=${REGISTRY_IMAGE_TAG_BASE}:${IMAGE_SHA_TAG}
+          REGISTRY_VERSION_TAG=${REGISTRY_IMAGE_TAG_BASE}:${IMAGE_VERSION_TAG}
+          echo "REGISTRY_VERSION_TAG=${REGISTRY_VERSION_TAG}" >> "$GITHUB_OUTPUT"
           docker load --input ${{ runner.temp }}/uesio-image.tar
-          docker tag $LOCAL_IMAGE_TAG $REGISTRY_IMAGE_TAG
-          docker image push $REGISTRY_IMAGE_TAG
+          docker tag $IMAGE_VERSION_TAG $REGISTRY_SHA_TAG
+          docker tag $IMAGE_VERSION_TAG $REGISTRY_VERSION_TAG
+          docker image push $REGISTRY_IMAGE_TAG_BASE
 
       - name: Update docker container image tag for dev
         env:
-          LOCAL_IMAGE_TAG: ${{ steps.pushImageToECR.outputs.LOCAL_IMAGE_TAG }}
-          REGISTRY_IMAGE_TAG: ${{ steps.pushImageToECR.outputs.REGISTRY_IMAGE_TAG }}
-          appTaskDefPath: ./aws/dev/ecs/task_definitions/uesio_web.json
-          workerTaskDefPath: ./aws/dev/ecs/task_definitions/uesio_worker.json
+          REGISTRY_VERSION_TAG: ${{ steps.pushImageToECR.outputs.registry_version_tag }}
+          APP_TASK_DEF_PATH: ./aws/dev/ecs/task_definitions/uesio_web.json
+          WORKER_TASK_DEF_PATH: ./aws/dev/ecs/task_definitions/uesio_worker.json
         run: |
-          echo "Docker image SHA updated to $LOCAL_IMAGE_TAG"
-          jq --arg img "$REGISTRY_IMAGE_TAG" '.containerDefinitions[0].image = $img' $appTaskDefPath > tmp1.json
-          jq --arg img "$REGISTRY_IMAGE_TAG" '.containerDefinitions[0].image = $img' $workerTaskDefPath > tmp2.json
-          mv tmp1.json $appTaskDefPath
-          mv tmp2.json $workerTaskDefPath
+          echo "Docker image SHA updated to $REGISTRY_VERSION_TAG"
+          jq --arg img "$REGISTRY_VERSION_TAG" '.containerDefinitions[0].image = $img' $APP_TASK_DEF_PATH > tmp1.json
+          jq --arg img "$REGISTRY_VERSION_TAG" '.containerDefinitions[0].image = $img' $WORKER_TASK_DEF_PATH > tmp2.json
+          mv tmp1.json $APP_TASK_DEF_PATH
+          mv tmp2.json $WORKER_TASK_DEF_PATH
           git config user.name github-actions
           git config user.email [email protected]
-          git add $appTaskDefPath $workerTaskDefPath
-          git commit -m "ci: Auto-update dev image to $LOCAL_IMAGE_TAG"
+          git add $APP_TASK_DEF_PATH $WORKER_TASK_DEF_PATH
+          git commit -m "ci: Auto-update dev image to $REGISTRY_VERSION_TAG"
           git push

.github/workflows/release.yaml

@@ -4,67 +4,66 @@ on:
   push:
     tags:
       - "v*"
-    secrets:
-      AWS_ACCOUNT_ID_DEV:
-        required: true
-      AWS_ECR_ROLE_DEV:
-        required: true
-      AWS_REGION_DEV:
-        required: true
-
-# Request permissions to be able to create releases
-permissions:
-  id-token: write # Needed to assume role using AWS OIDC provider
-  actions: write # Access to Github actions
-  contents: write # This is required for actions/checkout
+
+env:
+  GO_CACHE_INFO_FILE: wf-go-cache-info.txt
 
 jobs:
   docker_publish_to_ghcr:
     name: Publish to GitHub Container Registry
     runs-on: ubuntu-latest
-    permissions: write-all
+    permissions:
+      id-token: write # This is required for requesting a OIDC JWT for AWS
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
+
       - name: Login to GitHub Container Registry
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.AWS_ECR_ROLE_DEV }}
           role-session-name: ecrpull
           aws-region: ${{ secrets.AWS_REGION_DEV }}
+
       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
+
       - name: Pull the latest Docker image from ECR
-        shell: bash
         env:
-          FULL_SHA: ${{ github.sha }}
-          REGISTRY_ID: ${{ secrets.AWS_ACCOUNT_ID_DEV }}
-          IMAGE_TAG: ${{ steps.login-ecr.outputs.registry }}/uesio:${{ github.sha }}
+          SHA_TAG: ${{ github.sha }}-latest
+          ECR_REGISTRY_ID: ${{ secrets.AWS_ACCOUNT_ID_DEV }}
+          ECR_REGISTRY_URI: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY_NAME: uesio
           GHCR_IMAGE_TAG_BASE: ghcr.io/${{ github.repository }}
           RELEASE_TAG: ${{ github.ref_name }}
         run: |
-
           # Make sure that the Docker image for this commit exists in ECR
-          image_exists=$(bash ./scripts/ecr-image-exists.sh $REGISTRY_ID uesio $FULL_SHA)
+          IMAGE_EXISTS=$(bash ./scripts/ecr-image-exists.sh $ECR_REGISTRY_ID $ECR_REPOSITORY_NAME $SHA_TAG)
 
-          if [[ "$image_exists" == "no" ]]; then
-              echo "Docker image does not exist in ECR"
+          if [[ "$IMAGE_EXISTS" == "no" ]]; then
+              echo "Docker image does not exist in ECR for tag $SHA_LATEST_TAG"
               exit 1
           fi
 
-          echo "Pulling Docker image from ECR..."
-          docker pull $IMAGE_TAG
-          echo "Pushing tag $RELEASE_TAG to Github Container Registry..."=
-          docker tag $IMAGE_TAG $GHCR_IMAGE_TAG_BASE:$RELEASE_TAG
-          docker push $GHCR_IMAGE_TAG_BASE:$RELEASE_TAG
-          echo "Pushing latest tag to Github Container Registry..."
-          docker tag $IMAGE_TAG $GHCR_IMAGE_TAG_BASE:latest
-          docker push $GHCR_IMAGE_TAG_BASE:latest
+          ECR_SHA_TAG=${ECR_REGISTRY_URI}/${ECR_REPOSITORY_NAME}:${SHA_TAG}
+          echo "Pulling Docker image $ECR_SHA_TAG from ECR..."
+          docker pull $ECR_SHA_TAG
+
+          echo "Pushing tag $GHCR_IMAGE_TAG_BASE to Github Container Registry..."=
+          docker tag $ECR_SHA_TAG $GHCR_IMAGE_TAG_BASE:$RELEASE_TAG
+          docker tag $ECR_SHA_TAG $GHCR_IMAGE_TAG_BASE:latest
+          docker push --all-tags $GHCR_IMAGE_TAG_BASE
+
   cli_release_artifacts:
     name: Build asset for ${{ matrix.goos }} - ${{ matrix.goarch }}
     runs-on: ${{ matrix.os }}
@@ -92,16 +91,30 @@ jobs:
             artifact_name: uesio-macos-arm64
             asset_name: uesio-macos-arm64
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - name: Setup Go
+
+      # https://github.com/actions/setup-go/issues/358 - cache is shared across jobs by default since the dependency
+      # graph is the same, however each job results in different dependencies being downloaded and the first one
+      # to finish wins with regards to saving the cache.  To workaround, we create a file to include in the graph
+      # that contains information specified to the workflow & job so that each job gets a separate go cache.
+      # Note that the cache key used (https://github.com/actions/setup-go/blob/main/src/cache-restore.ts#L35) by
+      # actions/setup-go is already platform/arch/go-version specific so we only need to further differentiate
+      # by workflow and job.
+      - name: Create go cache info file
+        run: echo "go-cache-${{ github.workflow }}-${{ github.job }}" > ${GO_CACHE_INFO_FILE}
+
+      - name: Setup go
         uses: actions/setup-go@v5
         with:
           go-version-file: go.work
           cache-dependency-path: |
             apps/*/go.sum
             go.work.sum
+            ${{ env.GO_CACHE_INFO_FILE }}
+
       - name: Build CLI
         shell: bash
         env:
@@ -110,12 +123,14 @@ jobs:
           cd apps/cli
           GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o $outputPath
           cd ../..
+
       - name: Upload binary artifact
         uses: actions/upload-artifact@v4
         with:
           name: binaries-${{ matrix.artifact_name }}
           path: |
             apps/cli/dist/bin/${{ matrix.artifact_name }}
+
   upload_release_binaries:
     name: Upload binaries to release
     runs-on: ubuntu-latest
@@ -128,6 +143,7 @@ jobs:
           path: binaries
           pattern: binaries-*
           merge-multiple: true
+
       - name: Upload release artifacts
         uses: Roang-zero1/github-upload-release-artifacts-action@v2
         with:

README.md

@@ -134,12 +134,6 @@ npm run in-docker
 open https://studio.uesio-dev.com:3000
 ```
 
-**NOTE**: Docker Compose aggressively caches, so to force the app to rebuild the image (e.g. to rebuild JS / Go source), use this instead:
-
-```
-npm run in-docker-force-build
-```
-
 # <a id="set-up-ssl"></a> Set up SSL
 
 SSL is optional for local development. It is enabled using by setting the environment variable `UESIO_USE_HTTPS=true`

apps/platform/Dockerfile

@@ -18,9 +18,9 @@ COPY ./apps/platform/platform ./platform
 COPY ./dist/ui ./dist/ui
 COPY ./dist/vendor ./dist/vendor
 
-ARG GITSHA
-ENV GITSHA=${GITSHA}
-LABEL build_version=${GITSHA}
+ARG BUILD_VERSION
+ENV BUILD_VERSION=${BUILD_VERSION}
+LABEL build_version=${BUILD_VERSION}
 
 EXPOSE 3000
 

apps/platform/pkg/cmd/serve.go

@@ -53,9 +53,9 @@ var nameParam = getMetadataItemParam("name")
 var itemParam = fmt.Sprintf("%s/%s", nsParam, nameParam)
 var versionParam = "{version:(?:v[0-9]+\\.[0-9]+\\.[0-9]+)|(?:[a-z0-9]{8,}(?:\\.[0-9]+\\.[0-9]+)?)}"
 
-// Version will either be a Uesio bundle version string, e.g. v1.2.3,
-// an 8-character short Git sha, e.g. abcd1234 or an 8-character short
-// Git sha followed by runnumber.runattempt, e.g. abcd1234.13.3
+// Version will either be a Uesio bundle version string, e.g. v1.2.3, an 8-character
+// short Git sha followed by runnumber.runattempt, e.g. abcd1234.13.3 or the current
+// unix epoch time (e.g., 1738704509)
 var versionedItemParam = fmt.Sprintf("%s/%s/%s", nsParam, versionParam, nameParam)
 
 // Grouping values can either be full Uesio items (e.g. <user>/<app>.<name>) or simple values, e.g. "LISTENER",
@@ -67,8 +67,8 @@ var (
 	staticPrefix = "/static"
 )
 
-// Vendored scripts live under /static but do NOT get the GITSHA of the Uesio app,
-// because they are not expected to change with the GITSHA, but are truly static, immutable
+// Vendored scripts live under /static but do NOT get the version of the Uesio app,
+// because they are not expected to change with the version, but are truly static, immutable
 const vendorPrefix = "/static/vendor"
 
 func serve(cmd *cobra.Command, args []string) {
@@ -82,13 +82,13 @@ func serve(cmd *cobra.Command, args []string) {
 		panic("Failed to obtain working directory")
 	}
 
-	// If we have gitsha, append that to the prefixes to enable us to have versioned assets
-	gitsha := os.Getenv("GITSHA")
+	// If we have BUILD_VERSION, append that to the prefixes to enable us to have versioned assets
+	version := os.Getenv("BUILD_VERSION")
 	cacheSiteBundles := os.Getenv("UESIO_CACHE_SITE_BUNDLES")
 	cacheStaticAssets := false
 	staticAssetsPath := ""
-	if gitsha != "" {
-		staticAssetsPath = "/" + gitsha
+	if version != "" {
+		staticAssetsPath = "/" + version
 	} else if cacheSiteBundles == "true" {
 		staticAssetsPath = fmt.Sprintf("/%d", time.Now().Unix())
 	}