WIP ci: build plugins

.github/README.md

@@ -0,0 +1,39 @@
+# Misc Docs
+
+## Configuration Files
+
+### `plugins.json`
+
+```json
+{
+  "plugins": {
+    "<plugin-name>": {
+      "dir": "<path-to-plugin>",
+      "target": "<cmake-target-for-plugin>",
+      "test": "<cmake-target-for-test>",
+      "options": "<cmake-options>",
+      "platforms": [
+        {
+          "key": "<platform-key>"
+        },
+        {
+          "key": "<platform-key>",
+          "options": <extra-cmake-options>"
+        },
+        {
+          "key": "<platform-key>",
+          "options": <extra-cmake-options>",
+          "plugin_tag": "<extra-tag-for-plugin>"
+        }
+      }
+    }
+  },
+  "platforms": {
+    "<platform-key>": {
+      "runner": "<github-runner>",
+      "docker_tag": "<docker-tag>",
+      "asset_tag": "<asset-tag>"
+    }
+  }
+}
+```

.github/plugins.json

@@ -0,0 +1,55 @@
+{
+  "plugins": {
+    "wasi_crypto": {
+      "dir": "wasi_crypto",
+      "target": "wasmedgePluginWasiCrypto",
+      "test": "wasiCryptoTests",
+      "options": "-DWASMEDGE_PLUGIN_WASI_CRYPTO=ON",
+      "platforms": [
+        {"key":"manylinux_2_28_x86_64"},
+        {"key":"ubuntu2004_x86_64"}
+      ]
+    },
+    "wasi_nn-openvino": {
+      "dir": "wasi_nn",
+      "target": "wasmedgePluginWasiNN",
+      "test": "wasiNNTests",
+      "options": "-DWASMEDGE_PLUGIN_WASI_NN_BACKEND=OpenVINO",
+      "platforms": [
+        {"key":"ubuntu2004_x86_64"}
+      ]
+    },
+    "wasmedge_stablediffusion": {
+      "dir": "wasmedge_stablediffusion",
+      "target": "wasmedgePluginWasmEdgeStableDiffusion",
+      "test": "wasmedgeStableDiffusionTests",
+      "options": "-DWASMEDGE_PLUGIN_STABLEDIFFUSION=ON",
+      "platforms": [
+        {"key":"manylinux_2_28_x86_64"},
+        {"key":"ubuntu2004_x86_64"},
+        {
+          "key":"ubuntu2004_cuda11",
+          "options": "-DWASMEDGE_PLUGIN_STABLEDIFFUSION_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES='60;61;70' -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc",
+          "plugin_tag": "cuda-11"
+        }
+      ]
+    }
+  },
+  "platforms": {
+    "manylinux_2_28_x86_64": {
+      "runner": "ubuntu-latest",
+      "docker_tag": "manylinux_2_28_x86_64-plugins-deps",
+      "asset_tag": "manylinux_2_28_x86_64"
+    },
+    "ubuntu2004_x86_64": {
+      "runner": "ubuntu-latest",
+      "docker_tag": "ubuntu-20.04-build-clang-plugins-deps",
+      "asset_tag": "ubuntu20.04_x86_64"
+    },
+    "ubuntu2004_cuda11": {
+      "runner": "ubuntu-latest",
+      "docker_tag": "ubuntu-20.04-build-gcc-cuda11",
+      "asset_tag": "ubuntu20.04_x86_64"
+    }
+  }
+}

.github/scripts/parse-plugins.js

@@ -0,0 +1,27 @@
+module.exports.parse = (config) => {
+  let map = new Map();
+  for (const platform_key of Object.keys(config.platforms)) {
+    map.set(platform_key, []);
+  }
+  for (const [plugin_key, plugin] of Object.entries(config.plugins)) {
+    for (const platform of plugin.platforms) {
+      if (!(platform.key in config.platforms))
+        continue;
+      let copy = { ...plugin, ...config.platforms[platform.key] };
+      delete copy.platforms;
+      copy.plugin = plugin_key;
+      copy.plugin_tag = platform.plugin_tag;
+      copy.options = [plugin.options, platform.options].join(" ");
+      map.get(platform.key).push(copy);
+    }
+  }
+  return Object.fromEntries(map);
+};
+
+if (require.main === module) {
+  const { parse } = module.exports;
+  const fs = require("fs");
+  const s = fs.readFileSync("plugins.json");
+  const o = JSON.parse(s);
+  console.log(JSON.stringify(parse(o)));
+}

.github/workflows/build-on-linux.yml

@@ -0,0 +1,72 @@
+---
+name: Build on Linux
+
+on:
+  workflow_call:
+    inputs:
+      plugins:
+        type: string
+        required: true
+      version:
+        type: string
+        required: true
+
+permissions: {}
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(inputs.plugins) }}
+    name: ${{ matrix.plugin }}
+    runs-on: ${{ matrix.runner }}
+    container:
+      image: wasmedge/wasmedge:${{ matrix.docker_tag }}
+    env:
+      bin: lib${{ matrix.target }}.so
+      target: ${{ matrix.test }} # TODO: use matrix.target on release
+      test_dir: build/test/plugins/${{ matrix.dir }}
+      output_dir: build/plugins/${{ matrix.dir }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - id: var
+        run: |
+          prefix="WasmEdge-plugin-${{ matrix.plugin }}"
+          if [[ -n "${{ matrix.plugin_tag }}" ]]; then
+            prefix="${prefix}-${{ matrix.plugin_tag }}"
+          fi
+          echo "artifact=${prefix}-${{ inputs.version }}-${{ matrix.asset_tag }}.tar.gz" >> "${GITHUB_OUTPUT}"
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          repository: 'WasmEdge/WasmEdge' # TODO: checkout plugins from this repository
+      - name: Ensure git safe directory
+        run: |
+          git config --global --add safe.directory "$(pwd)"
+      - name: Build ${{ matrix.plugin }}
+        run: |
+          # TODO: Disable BUILD_TESTS on release
+          # TODO: Enable CXX11_ABI if not manylinux_2_28
+          cmake -Bbuild -GNinja \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DWASMEDGE_BUILD_TESTS=ON  \
+            -DWASMEDGE_BUILD_TOOLS=OFF \
+            -DWASMEDGE_USE_LLVM=OFF \
+            -DWASMEDGE_USE_CXX11_ABI=OFF \
+            "-DOPENSSL_ROOT_DIR=${OpenSSL_DIR}" \
+            ${{ matrix.options }}
+          cmake --build build --target "${target}"
+
+          cp -f "${output_dir}/${bin}" "${bin}"
+          tar -zcvf "${{ steps.var.outputs.artifact }}" "${bin}"
+      - name: Test ${{ matrix.plugin }}
+        run: |
+          cd "${test_dir}" && "./${target}"
+      - name: Upload ${{ steps.var.outputs.artifact }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.var.outputs.artifact }}
+          path: ${{ steps.var.outputs.artifact }}

.github/workflows/build.yml

@@ -0,0 +1,83 @@
+---
+name: Build
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions: {}
+
+jobs:
+  get-versions:
+    uses: ./.github/workflows/get-versions.yml
+
+  parse-plugins:
+    uses: ./.github/workflows/parse-plugins.yml
+
+  linux:
+    needs:
+      - get-versions
+      - parse-plugins
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: manylinux_2_28 (x86_64)
+            plugins: ${{ fromJSON(needs.parse-plugins.outputs.plugins).manylinux_2_28_x86_64 }}
+          - name: ubuntu20.04 (x86_64)
+            plugins: ${{ fromJSON(needs.parse-plugins.outputs.plugins).ubuntu2004_x86_64 }}
+          - name: ubuntu20.04 (x86_64) with CUDA 11.3
+            plugins: ${{ fromJSON(needs.parse-plugins.outputs.plugins).ubuntu2004_cuda11 }}
+    name: ${{ matrix.name }}
+    uses: ./.github/workflows/build-on-linux.yml
+    with:
+      plugins: ${{ toJSON(matrix.plugins) }}
+      version: ${{ needs.get-versions.outputs.version }}
+    secrets: inherit
+
+  check-artifacts:
+    if: ${{ true }} # TODO: check output from `get-versions` and `parse-plugins`
+    needs:
+      - get-versions
+      - parse-plugins
+      - linux
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require("fs");
+            const raw = JSON.parse(fs.readFileSync(".github/plugins.json"));
+
+            core.summary.addHeading('Build Summary');
+
+            let table = [
+              [{data: 'plugin/platform', header: true}],
+            ];
+            let platforms = [];
+            Object.entries(raw.platforms).forEach(([key, platform]) => {
+              table[0].push({data: key, header: true});
+              platforms.push(platform.asset_tag);
+            });
+
+            let artifacts = await github
+              .paginate(github.rest.actions.listWorkflowRunArtifacts, {
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.runId,
+              });
+
+            Object.keys(raw.plugins).forEach((plugin) => {
+              let row = [{data: plugin, header: true}];
+              let filtered = artifacts.filter((artifact) => artifact.name.includes(plugin));
+              platforms.forEach((tag) => {
+                row.push({
+                  data: filtered.some((artifact) => artifact.name.includes(tag)) ? ':ok:' : ':x:'
+                });
+              });
+              table.push(row);
+            });
+
+            core.summary.addTable(table);
+            core.summary.write()

.github/workflows/get-versions.yml

@@ -0,0 +1,25 @@
+---
+name: Get versions
+
+on:
+  workflow_call:
+    outputs:
+      version:
+        value: ${{ jobs.get-versions.outputs.version }}
+
+jobs:
+  get-versions:
+    name: Retrieve version information
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.parse.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          repository: 'WasmEdge/WasmEdge' # TODO: checkout plugins from this repository
+      - id: parse
+        run: |
+          git fetch --tags --force
+          VERSION=$(git describe --match '[0-9].[0-9]*' --tag)
+          echo "version=${VERSION}" >> "${GITHUB_OUTPUT}"

.github/workflows/parse-plugins.yml

@@ -0,0 +1,52 @@
+---
+name: Parse
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - '.github/plugins.json'
+      - '.github/scripts/parse-plugins.js'
+      - '.github/workflows/parse-plugins.yml'
+
+  workflow_call:
+    inputs:
+      workflow_call: # workaround to distinguish ${{ github.event }}
+        type: boolean
+        default: true
+    outputs:
+      plugins:
+        value: ${{ jobs.parse.outputs.plugins }}
+
+jobs:
+  parse:
+    name: Parse configuration file
+    runs-on: ubuntu-latest
+    outputs:
+      plugins: ${{ steps.readfile.outputs.plugins }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: readfile
+        uses: actions/github-script@v7
+        with:
+          result-encoding: string
+          script: |
+            const { parse } = require(".github/scripts/parse-plugins.js");
+            const fs = require("fs");
+            const s = fs.readFileSync(".github/plugins.json");
+            const config = parse(JSON.parse(s));
+            core.setOutput("plugins", config);
+
+  test:
+    if: ${{ !inputs.workflow_call }}
+    needs: parse
+    name: Self-testing
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get the first target from ubuntu2004_cuda11
+        run: |
+          echo "target=${{ fromJSON(needs.parse.outputs.plugins).ubuntu2004_cuda11[0].target }}" >> "${GITHUB_ENV}"
+      - name: Check target
+        run: |
+          echo "${target}"
+          test "${target}" = 'wasmedgePluginWasmEdgeStableDiffusion'