refactor(ci): use data-platform workflows (#286)

* build: refactor charmcraft.yaml to use ST124 notation (#282)

Refactor charmcraft.yaml files to enable the usage of charmcraft-cache
and data-platform-workflows in general.

Close #271

* refactor(ci): use data-platform-workflows in on_pull/push workflows (#284)

* refactor(ci): use data-platform-workflows in on_pull/push workflows

* Enable the usage of `build_charm.yaml` workflow, which allows
  independent and cached charm builds, making the CI more efficient and
  fast.
* Refactor the tests and the integrate workflow to be able to use the
  charm artefacts generated by the `build_charm.yaml` workflow:
  - Refactor integrate.yaml to be ci.yaml, a workflow that now takes
    care for the whole pipeline from generating charm paths, to
    building, to testing, and then releasing. This change will replace
    on_pull_request.yaml.
  - on_push.yaml is now replaced entirely by release, which will run on
    pushes to main or track/* branches. This workflow is in charge of
    building -> testing -> releasing.
  - Refactor individual and repository level integration tests to be
    able to use charm artefacts besides building them.

Close #272

* refactor(ci): Replace release.yaml with promote.yaml (#285)

* Add promote.yaml workflow to replace the old release.yaml
  workflow_dispatch funcionality.
* Remove publish.yaml workflow as the release process is handled by the
  recently refactor release.yaml.

Fix #277

Author: orfeas-k

.github/workflows/integrate.yaml .github/workflows/ci.yaml

@@ -1,19 +1,55 @@
-# reusable workflow triggered by other actions
 name: CI
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
+  pull_request:
+  schedule:
+    - cron: '0 8 * * TUE'
+  # Triggered on push by .github/workflows/release.yaml
   workflow_call:
-    secrets:
-      CHARMCRAFT_CREDENTIALS:
-        required: true
+    outputs:
+      artifact-prefix:
+        description: build_charm.yaml `artifact-prefix` output
+        value: ${{ jobs.build.outputs.artifact-prefix }}
+      charm-paths:
+        description: paths for all charms in this repo
+        value: ${{ jobs.get-charm-paths-channel.outputs.charm-paths }}
+      channel:
+        description: Charmhub channel the charms are released to
+        value: ${{ jobs.get-charm-paths-channel.outputs.charm-channel }}
 
 jobs:
+  get-charm-paths-channel:
+    name: Get charm paths and charmhub channel
+    runs-on: ubuntu-latest
+    outputs:
+      charm-paths: ${{ steps.get-charm-paths.outputs.charm-paths }}
+      charm-channel: ${{ steps.select-channel.outputs.name }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Get paths for all charms in this repo
+        id: get-charm-paths
+        uses: canonical/kubeflow-ci/actions/get-charm-paths@main
+      - name: Select charmhub channel
+        uses: canonical/charming-actions/[email protected]
+        id: select-channel
+
   lib-check:
     name: Check libraries
+    needs:
+      - get-charm-paths-channel
+    strategy:
+      matrix:
+        charm: ${{ fromJSON(needs.get-charm-paths-channel.outputs.charm-paths) }}
     uses: canonical/charmed-kubeflow-workflows/.github/workflows/_quality-checks.yaml@main
     secrets: inherit
     with:
-        charm-path: "."
+      charm-path: ${{ matrix.charm }}
 
   lint:
     name: Lint Code
@@ -23,9 +59,10 @@ jobs:
       matrix:
         charm: [operator, serving, eventing]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: pip install tox
       - run: tox -e ${{ matrix.charm }}-lint
+
   unit:
     name: Unit Test
     runs-on: ubuntu-20.04
@@ -34,19 +71,21 @@ jobs:
       matrix:
         charm: [operator, serving, eventing]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: pip install tox
       - run: tox -e ${{ matrix.charm }}-unit
 
   terraform-checks:
     name: Terraform
+    needs:
+      - get-charm-paths-channel
     uses: canonical/charmed-kubeflow-workflows/.github/workflows/terraform-checks.yaml@main
     strategy:
       fail-fast: false
       matrix:
-        charm: [operator, serving, eventing]
+        charm: ${{ fromJSON(needs.get-charm-paths-channel.outputs.charm-paths) }}
     with:
-      charm-path: ./charms/knative-${{ matrix.charm }}
+      charm-path: ${{ matrix.charm }}
       # Skipping the Terraform apply check as knative-eventing and knative-serving
       # go to Error when knative-operator is not deployed, instead of the expected
       # Blocked or Active. This is currently a limitation of the Terraform re-usable
@@ -55,10 +94,22 @@ jobs:
       # See https://github.com/canonical/knative-operators/issues/156
       apply: false
 
+  build:
+    strategy:
+      matrix:
+        charm: ${{ fromJSON(needs.get-charm-paths-channel.outputs.charm-paths) }}
+    name: Build charm | ${{ matrix.charm }}
+    needs:
+      - get-charm-paths-channel
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    with:
+      path-to-charm-directory: ${{ matrix.charm }}
+
   integration-charm-deployment:
     name: Integration Test
+    needs:
+      - build
     runs-on: ubuntu-20.04
-
     steps:
       # Ideally we'd use self-hosted runners, but this effort is still not stable
       # This action will remove unused software (dotnet, haskell, android libs, codeql,
@@ -69,21 +120,28 @@ jobs:
         uses: jlumbroso/[email protected]
 
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup operator environment
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: microk8s
           channel: 1.31-strict/stable
           juju-channel: 3.6/stable
-          charmcraft-channel: 3.x/stable
           microk8s-addons: "dns storage rbac metallb:10.64.140.43-10.64.140.49"
 
+      - name: Download packed charm(s)
+        id: download-charms
+        timeout-minutes: 5
+        uses: actions/download-artifact@v4
+        with:
+          pattern: ${{ needs.build.outputs.artifact-prefix }}-*
+          merge-multiple: true
+
       - name: Run integration tests
         run: |
           sg snap_microk8s -c "juju add-model knative-test"
-          sg snap_microk8s -c "tox -vve integration -- --model knative-test"
+          sg snap_microk8s -c "tox -vve integration -- --model knative-test --charms-path=${{ github.workspace }}/charms"
 
       - run: kubectl get all -A
         if: failure()
@@ -95,12 +153,13 @@ jobs:
         if: failure()
 
       - uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main
-        if: always()
+        if: failure()
 
   integration-observability:
     name: Observability Integration Test
+    needs:
+      - build
     runs-on: ubuntu-20.04
-
     steps:
       # Ideally we'd use self-hosted runners, but this effort is still not stable
       # This action will remove unused software (dotnet, haskell, android libs, codeql,
@@ -111,25 +170,32 @@ jobs:
         uses: jlumbroso/[email protected]
 
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup operator environment
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: microk8s
           channel: 1.31-strict/stable
           juju-channel: 3.6/stable
-          charmcraft-channel: 3.x/stable
+
+      - name: Download packed charm(s)
+        id: download-charms
+        timeout-minutes: 5
+        uses: actions/download-artifact@v4
+        with:
+          pattern: ${{ needs.build.outputs.artifact-prefix }}-*
+          merge-multiple: true
 
       - name: Run integration tests
         run: |
           sg snap_microk8s -c "juju add-model cos-test"
-          sg snap_microk8s -c "tox -vve cos-integration -- --model cos-test"
+          sg snap_microk8s -c "tox -vve cos-integration -- --model cos-test --charms-path=${{ github.workspace }}/charms"
 
-      - run: kubectl get pod/prometheus-k8s-0 -n knative-test -o=jsonpath='{.status}'
+      - run: kubectl get pod/prometheus-k8s-0 -n cos-test -o=jsonpath='{.status}'
         if: failure()
 
-      - run: kubectl get pod/knative-operator-0 -nknative-test -o=jsonpath='{.status}'
+      - run: kubectl get pod/knative-operator-0 -n cos-test -o=jsonpath='{.status}'
         if: failure()
 
       - run: kubectl get all -A
@@ -139,4 +205,22 @@ jobs:
         if: failure()
 
       - uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main
-        if: always()
+        if: failure()
+
+  release:
+    strategy:
+      matrix:
+        charm: ${{ fromJSON(needs.get-charm-paths-channel.outputs.charm-paths) }}
+    name: Release charm to Charmhub branch | ${{ matrix.charm }}
+    if: ${{ github.event_name == 'pull_request' }}
+    needs:
+      - get-charm-paths-channel
+      - build
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    with:
+      channel: ${{ needs.get-charm-paths-channel.outputs.charm-channel }}
+      artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}
+      path-to-charm-directory: ${{ matrix.charm }}
+      create-git-tags: false
+    secrets:
+      charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }}

.github/workflows/get-charm-paths.sh

@@ -0,0 +1,31 @@
+# reusable workflow triggered manually
+name: Promote charm to other tracks and channels
+
+on:
+  workflow_dispatch:
+    inputs:
+      destination-channel:
+        description: 'Destination Channel'
+        required: true
+      origin-channel:
+        description: 'Origin Channel'
+        required: true
+      charm-name:
+        description: 'Charm subdirectory name'
+        required: true
+
+jobs:
+  promote-charm:
+    name: Promote charm
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Release charm to channel
+        uses: canonical/charming-actions/[email protected]
+        with:
+          credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          destination-channel: ${{ github.event.inputs.destination-channel }}
+          origin-channel: ${{ github.event.inputs.origin-channel }}
+          tag-prefix: ${{ github.event.inputs.charm-name }}
+          charm-path: charms/${{ github.event.inputs.charm-name}}