diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..c8bd192 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,47 @@ +name: Create and publish a Docker image + +on: + push: + branches: ["release"] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image + id: push + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9db12f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +tmp*/ +.DS_Store +node_modules +*.log +*.local +.env +.cache diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..74baffc --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": ["denoland.vscode-deno"] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..39811aa --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,20 @@ +{ + "deno.enablePaths": ["."], + "deno.lint": true, + "eslint.workingDirectories": [ + { + "mode": "auto" + } + ], + "[typescript]": { + "editor.defaultFormatter": "denoland.vscode-deno", + "editor.formatOnSave": true, + "editor.formatOnPaste": true + }, + "[javascript]": { + "editor.defaultFormatter": "denoland.vscode-deno", + "editor.formatOnSave": true, + "editor.formatOnPaste": true + }, + "editor.tabSize": 2 +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..64578be --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM amazonlinux:2 + + +RUN yum -y update \ + # systemd is not a hard requirement for Amazon ECS Anywhere, but the installation script currently only supports systemd to run. + # Amazon ECS Anywhere can be used without systemd, if you set up your nodes and register them into your ECS cluster **without** the installation script. + && yum -y install systemd unzip tar xz curl \ + && yum clean all + +RUN groupadd -g 3982 -o elwood_runner +RUN useradd -m -u 3982 -g 3982 -o -s /bin/bash elwood_runner + +RUN mkdir -p /elwood/runner/bin \ + /elwood/runner/workspace \ + /elwood/runner/workspace-bin + +RUN curl -fsSL https://deno.land/install.sh | DENO_DIR=/elwood/runner/deno-data DENO_INSTALL=/elwood/runner /bin/sh + +RUN chown -R elwood_runner:elwood_runner /elwood/runner/workspace + +ENV ELWOOD_RUNNER_ROOT /elwood/runner +ENV ELWOOD_RUNNER_WORKSPACE_DIR /elwood/runner/workspace +ENV ELWOOD_RUNNER_EXECUTION_UID 3982 +ENV ELWOOD_RUNNER_EXECUTION_GID 3982 + +RUN tar --version \ No newline at end of file diff --git a/actions/_core/args.ts b/actions/_core/args.ts new file mode 100644 index 0000000..007ecde --- /dev/null +++ b/actions/_core/args.ts @@ -0,0 +1,16 @@ +export function get(name: string, strict = true): string | undefined { + const envName = `ARG_${name.toUpperCase()}`; + + if (strict && !Deno.env.get(envName)) { + throw new Error(`Missing required environment variable: ${envName}`); + } + + if ( + Deno.permissions.querySync({ name: "env", variable: envName }) + .state !== "granted" + ) { + return ""; + } + + return Deno.env.get(envName) as string; +} diff --git a/actions/_core/command.ts b/actions/_core/command.ts new file mode 100644 index 0000000..c95af99 --- /dev/null +++ b/actions/_core/command.ts @@ -0,0 +1,25 @@ +export async function execute( + bin: string, + options: Deno.CommandOptions = {}, +): Promise { + const cmd = await create(bin, { + stdout: "inherit", + stderr: "inherit", + ...options, + }); + + return await cmd.output(); +} + +export async function create( + bin: string, + options: Deno.CommandOptions = {}, +): Promise { + return await Promise.resolve( + new Deno.Command(bin, { + stdout: "inherit", + stderr: "inherit", + ...options, + }), + ); +} diff --git a/actions/_core/fetch.ts b/actions/_core/fetch.ts new file mode 100644 index 0000000..b4d2f93 --- /dev/null +++ b/actions/_core/fetch.ts @@ -0,0 +1,105 @@ +import { toWritableStream } from "../deps.ts"; +import { normalize } from "./path.ts"; + +export const native = globalThis.fetch; + +// You should be using `request` not `fetch` for everything +// this makes that a bit harder to accidentally use `fetch +// deno-lint-ignore no-unused-vars -- intentional +const fetch = undefined; + +export type RequestOptions = RequestInit & { + saveTo?: string; + asStream?: boolean; +}; + +export type Response = { + data: D | null; + headers: Record; + error: Error | undefined; +}; + +export async function request( + url: string, + init: RequestOptions, +): Promise> { + const response = await native(url, init); + const headers = headersToObject(response.headers); + + if (!response.ok) { + return { + data: null, + headers, + error: new Error(response.statusText), + }; + } + + if (!response.body) { + return { + data: null, + headers, + error: new Error("No body in response"), + }; + } + + let data: T | null = null; + + if (init.asStream) { + return { + data: response.body as T, + headers, + error: undefined, + }; + } + + if (init.saveTo) { + const file = await Deno.open(await normalize(init.saveTo), { + write: true, + create: true, + }); + const writableStream = toWritableStream(file); + await response.body.pipeTo(writableStream); + + data = { + path: init.saveTo, + } as T; + } else if ( + response.headers.get("Content-Type")?.includes("application/json") + ) { + data = await response.json() as T; + } else { + data = await response.text() as T; + } + + return { + data, + headers, + error: undefined, + }; +} + +function _methodProxy(method: RequestInit["method"]): typeof request { + return (url: string, options: Omit) => + request(url, { ...options, method }); +} + +export const get = _methodProxy("GET"); +export const post = _methodProxy("POST"); +export const put = _methodProxy("PUT"); +export const patch = _methodProxy("PATCH"); +export const del = _methodProxy("DELETE"); + +function headersToObject(headers: Headers): Record { + const obj: Record = {}; + + for (const [key, value] of headers.entries()) { + obj[key] = value; + } + + return Array.from(headers.entries()).reduce((acc, [key, value]) => { + return { + ...acc, + [key]: value, + }; + }, {}); +} diff --git a/actions/_core/fs.ts b/actions/_core/fs.ts new file mode 100644 index 0000000..a159f43 --- /dev/null +++ b/actions/_core/fs.ts @@ -0,0 +1,29 @@ +import { type FilePathOrUrl, normalize } from "./path.ts"; + +export async function copy(src: FilePathOrUrl, dest: FilePathOrUrl) { + await Deno.copyFile( + await normalize(src), + await normalize(dest), + ); +} + +export async function mkdir(path: FilePathOrUrl, recursive = true) { + await Deno.mkdir( + await normalize(path), + { recursive }, + ); +} + +export async function rename(from: FilePathOrUrl, to: FilePathOrUrl) { + return await Deno.rename( + await normalize(from), + await normalize(to), + ); +} + +export async function remove(path: FilePathOrUrl, recursive = false) { + return await Deno.remove( + await normalize(path), + { recursive }, + ); +} diff --git a/actions/_core/input.ts b/actions/_core/input.ts new file mode 100644 index 0000000..bfd2560 --- /dev/null +++ b/actions/_core/input.ts @@ -0,0 +1,82 @@ +import { normalize } from "./path.ts"; + +export function get(name: string, strict = true): string { + const inputEnvName = `INPUT_${name.toUpperCase()}`; + + if (strict && !Deno.env.get(inputEnvName)) { + throw new Error(`Missing required environment variable: ${inputEnvName}`); + } + + // if they don't strictly require the input, make sure we have permission + // to read it before trying to read it + if ( + Deno.permissions.querySync({ name: "env", variable: inputEnvName }) + .state !== "granted" + ) { + return ""; + } + + return Deno.env.get(inputEnvName) as string; +} + +export function getOptional( + name: string, + fallback: T | undefined = undefined, +): string | T | undefined { + try { + const value = get(name, false); + + return value === "" ? fallback : value; + } catch { + return fallback; + } +} + +export function getJson( + name: string, + strict = true, +): Record | T[] { + const value = get(name, strict); + + if (value && value.startsWith("json:")) { + return JSON.parse(value.substring(5)); + } + + throw new Error(`Input ${name} is not valid JSON: ${value}`); +} + +export function getOptionalJson( + name: string, + fallback: T | undefined = undefined, +): ReturnType | T | undefined { + try { + return getJson(name, false); + } catch { + return fallback; + } +} + +export function getBoolean(name: string, strict = true): boolean { + const value = get(name, strict); + + if (!value) { + return false; + } + + switch (value.toLowerCase()) { + case "1": + case "yes": + case "true": + return true; + + default: + return false; + } +} + +export async function getNormalizedPath( + name: string, + strict = true, +): Promise { + return await normalize(get(name, strict)); +} diff --git a/actions/_core/install.ts b/actions/_core/install.ts new file mode 100644 index 0000000..406e93e --- /dev/null +++ b/actions/_core/install.ts @@ -0,0 +1,103 @@ +import { + assert, + basename, + expandGlob, + extname, + join, + relative, +} from "../deps.ts"; + +import * as fetch from "./fetch.ts"; +import * as fs from "./fs.ts"; + +export class Install { + readonly #dir = Deno.makeTempDirSync({ + dir: Deno.cwd(), + }); + + join(...args: string[]) { + return join(this.#dir, ...args); + } + + async fetch(src: string): Promise { + const saveTo = this.join(basename(src)); + + await fetch.get(src, { + saveTo, + }); + + return saveTo; + } + + async find(srcGlob: string): Promise { + const found = await expandGlob(this.join(srcGlob)).next(); + + if (!found?.value?.isFile) { + throw new Error(`Unable to find ${srcGlob} in ${this.#dir}`); + } + + return relative(this.#dir, found.value.path); + } + + async move(srcName: string, dest: string) { + await fs.rename(this.join(srcName), dest); + } + + async findAndMove(srcGlob: string, dest: string) { + const found = await this.find(srcGlob); + await this.move(found, dest); + } + + async extract(src: string) { + await extract(src, { + cwd: this.#dir, + }); + } + + async cleanup() { + await Deno.remove(this.#dir, { recursive: true }); + } +} + +export type ExtractOptions = { + cwd?: string; +}; + +export async function extract( + src: string, + options: ExtractOptions = {}, +): Promise { + switch (extname(src)) { + case ".zip": + return await extractZip(src, options); + default: + return await extractTar(src, options); + } +} + +export async function extractTar( + src: string, + options: ExtractOptions = {}, +): Promise { + const cwd = options.cwd ?? Deno.cwd(); + const cmd = new Deno.Command("tar", { + cwd, + args: ["-xf", src], + env: { + PATH: Deno.env.get("PATH") ?? "", + }, + stdout: "inherit", + stderr: "inherit", + }); + + assert((await cmd.output()).code === 0); +} + +export async function extractZip( + src: string, + options: ExtractOptions = {}, +): Promise { + const cwd = options.cwd ?? Deno.cwd(); + const cmd = new Deno.Command("unzip", { cwd, args: ["-q", src] }); + await cmd.output(); +} diff --git a/actions/_core/mod.ts b/actions/_core/mod.ts new file mode 100644 index 0000000..2de7aed --- /dev/null +++ b/actions/_core/mod.ts @@ -0,0 +1,7 @@ +export * as input from "./input.ts"; +export * as path from "./path.ts"; +export * as fs from "./fs.ts"; +export * as fetch from "./fetch.ts"; +export * as install from "./install.ts"; +export * as command from "./command.ts"; +export * as args from "./args.ts"; diff --git a/actions/_core/path.test.ts b/actions/_core/path.test.ts new file mode 100644 index 0000000..47d767c --- /dev/null +++ b/actions/_core/path.test.ts @@ -0,0 +1,19 @@ +import { assertEquals, assertRejects } from "../deps.ts"; + +import { normalize } from "./path.ts"; + +Deno.test("normalize()", async function () { + assertRejects(() => normalize("stage:///a")); + + Deno.env.set("ELWOOD_STAGE_DIR", "/stage-path"); + + assertEquals( + await normalize("/path"), + "/path", + ); + + assertEquals( + await normalize("stage:///this/is/a/path"), + "/stage-path/this/is/a/path", + ); +}); diff --git a/actions/_core/path.ts b/actions/_core/path.ts new file mode 100644 index 0000000..ce4754f --- /dev/null +++ b/actions/_core/path.ts @@ -0,0 +1,54 @@ +import { assert, join } from "../deps.ts"; + +export type FilePathOrUrl = string | URL; + +// deno-lint-ignore require-await +export async function normalize( + pathOrUrl: FilePathOrUrl, +): Promise { + // if the path is absolute, relative or doesn't include + // a protocol, we can assume it's a local path + if ( + typeof pathOrUrl === "string" && ( + pathOrUrl.startsWith("/") || pathOrUrl.startsWith(".") || + !pathOrUrl.includes("://") + ) + ) { + return pathOrUrl; + } + + const url = new URL(pathOrUrl); + + // we have a few non-standard protocols we can handle + // stage:// or file+stage:// maps to the stage directory + // bin:// or file+bin:// maps to the bin directory + switch (url.protocol) { + case "stage:": + case "file+stage:": { + const stageDir = Deno.env.get("ELWOOD_STAGE_DIR"); + assert(stageDir, "ELWOOD_STAGE_DIR is required"); + + return join( + stageDir, + url.hostname, + url.pathname, + ); + } + + case "bin:": + case "file+bin": { + const binDir = Deno.env.get("ELWOOD_BIN_DIR"); + assert(binDir, "ELWOOD_BIN_DIR is required"); + + return join( + binDir, + url.hostname, + url.pathname, + ); + } + + default: { + throw new Error(`Unsupported protocol: ${url.protocol}`); + } + } +} diff --git a/actions/deps.ts b/actions/deps.ts new file mode 100644 index 0000000..619f864 --- /dev/null +++ b/actions/deps.ts @@ -0,0 +1,20 @@ +export { + assert, + assertEquals, + assertRejects, + assertThrows, +} from "jsr:@std/assert"; +export { + basename, + dirname, + extname, + fromFileUrl, + join, + relative, +} from "jsr:@std/path"; +export { toWritableStream } from "jsr:@std/io/to-writable-stream"; +export { expandGlob } from "jsr:@std/fs/expand-glob"; +export { ensureFile } from "jsr:@std/fs/ensure-file"; +export { ensureDir } from "jsr:@std/fs/ensure-dir"; +export { Untar } from "jsr:@std/archive/untar"; +export { copy } from "jsr:@std/io/copy"; diff --git a/actions/echo.ts b/actions/echo.ts new file mode 100644 index 0000000..f719837 --- /dev/null +++ b/actions/echo.ts @@ -0,0 +1,10 @@ +import { input } from "./_core/mod.ts"; + +if (import.meta.main) { + main(); +} + +export async function main() { + const content = input.get("content"); + await Deno.stdout.write(new TextEncoder().encode(content)); +} diff --git a/actions/fs/copy.ts b/actions/fs/copy.ts new file mode 100644 index 0000000..ce78953 --- /dev/null +++ b/actions/fs/copy.ts @@ -0,0 +1,12 @@ +import { fs, input } from "../_core/mod.ts"; + +if (import.meta.main) { + main(); +} + +export async function main() { + const src = await input.getNormalizedPath("src"); + const dest = await input.getNormalizedPath("dest"); + + await fs.copy(src, dest); +} diff --git a/actions/fs/mkdir.ts b/actions/fs/mkdir.ts new file mode 100644 index 0000000..25f7aa3 --- /dev/null +++ b/actions/fs/mkdir.ts @@ -0,0 +1,12 @@ +import { fs, input } from "../_core/mod.ts"; + +if (import.meta.main) { + main(); +} + +export async function main() { + const dir = await input.get("path"); + const recursive = input.getBoolean("recursive"); + + await fs.mkdir(dir, recursive); +} diff --git a/actions/install/ffmpeg.ts b/actions/install/ffmpeg.ts new file mode 100644 index 0000000..944ae12 --- /dev/null +++ b/actions/install/ffmpeg.ts @@ -0,0 +1,56 @@ +import { install } from "../_core/mod.ts"; + +enum Urls { + DarwinFFmpeg = "https://evermeet.cx/ffmpeg/ffmpeg-5.1.2.zip", + DarwinFFprobe = "https://evermeet.cx/ffmpeg/ffprobe-5.1.2.zip", + LinuxAMD = + "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz", + LinuxARM = + "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-arm64-static.tar.xz", +} + +const platforms = ["darwin", "linux"]; + +if (import.meta.main) { + main(); +} + +export async function main() { + if (!platforms.includes(Deno.build.os)) { + throw new Error( + `Unsupported platform: ${Deno.build.os}. Must be ${ + Object.keys( + platforms, + ).join(", ") + }`, + ); + } + + // urls to download and install + const urls: string[] = []; + + switch ([Deno.build.os, Deno.build.arch].join("-")) { + case "darwin-aarch64": + case "darwin-x86_64": + urls.push(Urls.DarwinFFmpeg, Urls.DarwinFFprobe); + break; + case "linux-x86_64": + urls.push(Urls.LinuxAMD); + break; + case "linux-aarch64": + urls.push(Urls.LinuxARM); + break; + default: + throw new Error(`Unsupported platform: ${Deno.build.os}`); + } + + const i = new install.Install(); + + for (const url of urls) { + await i.extract(await i.fetch(url)); + } + + await i.findAndMove(`**/ffmpeg`, `bin://ffmpeg`); + await i.findAndMove(`**/ffprobe`, `bin://ffprobe`); + await i.cleanup(); +} diff --git a/actions/install/whisper.ts b/actions/install/whisper.ts new file mode 100644 index 0000000..e69de29 diff --git a/actions/run.ts b/actions/run.ts new file mode 100644 index 0000000..6c35161 --- /dev/null +++ b/actions/run.ts @@ -0,0 +1,35 @@ +import { assert } from "./deps.ts"; +import { args, command, input } from "./_core/mod.ts"; + +if (import.meta.main) { + main(); +} + +export async function main() { + const cmd = input.getOptional("bin", args.get("bin", false)) ?? "deno"; + const script = input.get("script", false); + const cmdArgs = input.getOptionalJson("args", []); + + if (script) { + const cmd = await command.create("bash", { + stdin: "piped", + }); + + const child = cmd.spawn(); + + // write the script to the childprocess as stdin + child.stdin.getWriter().write(new TextEncoder().encode(script)); + child.stdin.close(); + + Deno.exit((await child.status).code); + } + + assert(cmd, "bin must be provided"); + assert(Array.isArray(cmdArgs), "args must be an array"); + + const result = await command.execute(cmd, { + args: cmdArgs as string[], + }); + + Deno.exit(result.code); +} diff --git a/deno.json b/deno.json new file mode 100644 index 0000000..b431188 --- /dev/null +++ b/deno.json @@ -0,0 +1,11 @@ +{ + "tasks": { + "test": "deno test -A ./**/*.test.ts" + }, + "compilerOptions": { + "strict": true, + "useUnknownInCatchVariables": true, + "noImplicitOverride": true, + "noUncheckedIndexedAccess": true + } +} diff --git a/deno.lock b/deno.lock new file mode 100644 index 0000000..263143b --- /dev/null +++ b/deno.lock @@ -0,0 +1,45 @@ +{ + "version": "3", + "packages": { + "specifiers": { + "jsr:@std/assert@^0.226.0": "jsr:@std/assert@0.226.0", + "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.0", + "jsr:@std/path@^0.225.2": "jsr:@std/path@0.225.2" + }, + "jsr": { + "@std/assert@0.226.0": { + "integrity": "0dfb5f7c7723c18cec118e080fec76ce15b4c31154b15ad2bd74822603ef75b3", + "dependencies": [ + "jsr:@std/internal@^1.0.0" + ] + }, + "@std/internal@1.0.0": { + "integrity": "ac6a6dfebf838582c4b4f61a6907374e27e05bedb6ce276e0f1608fe84e7cd9a" + }, + "@std/path@0.225.2": { + "integrity": "0f2db41d36b50ef048dcb0399aac720a5348638dd3cb5bf80685bf2a745aa506", + "dependencies": [ + "jsr:@std/assert@^0.226.0" + ] + } + } + }, + "remote": { + "https://deno.land/std@0.217.0/assert/assert.ts": "bec068b2fccdd434c138a555b19a2c2393b71dfaada02b7d568a01541e67cdc5", + "https://deno.land/std@0.217.0/assert/assertion_error.ts": "9f689a101ee586c4ce92f52fa7ddd362e86434ffdf1f848e45987dc7689976b8", + "https://deno.land/std@0.217.0/path/_common/assert_path.ts": "dbdd757a465b690b2cc72fc5fb7698c51507dec6bfafce4ca500c46b76ff7bd8", + "https://deno.land/std@0.217.0/path/_common/constants.ts": "dc5f8057159f4b48cd304eb3027e42f1148cf4df1fb4240774d3492b5d12ac0c", + "https://deno.land/std@0.217.0/path/_common/normalize.ts": "684df4aa71a04bbcc346c692c8485594fc8a90b9408dfbc26ff32cf3e0c98cc8", + "https://deno.land/std@0.217.0/path/_common/normalize_string.ts": "dfdf657a1b1a7db7999f7c575ee7e6b0551d9c20f19486c6c3f5ff428384c965", + "https://deno.land/std@0.217.0/path/_os.ts": "8fb9b90fb6b753bd8c77cfd8a33c2ff6c5f5bc185f50de8ca4ac6a05710b2c15", + "https://deno.land/std@0.217.0/path/join.ts": "ae2ec5ca44c7e84a235fd532e4a0116bfb1f2368b394db1c4fb75e3c0f26a33a", + "https://deno.land/std@0.217.0/path/posix/_util.ts": "1e3937da30f080bfc99fe45d7ed23c47dd8585c5e473b2d771380d3a6937cf9d", + "https://deno.land/std@0.217.0/path/posix/join.ts": "744fadcbee7047688696455c7cbb368a9625ffde67fc3058a61c98948fcd04de", + "https://deno.land/std@0.217.0/path/posix/normalize.ts": "baeb49816a8299f90a0237d214cef46f00ba3e95c0d2ceb74205a6a584b58a91", + "https://deno.land/std@0.217.0/path/windows/_util.ts": "d5f47363e5293fced22c984550d5e70e98e266cc3f31769e1710511803d04808", + "https://deno.land/std@0.217.0/path/windows/join.ts": "8d03530ab89195185103b7da9dfc6327af13eabdcd44c7c63e42e27808f50ecf", + "https://deno.land/std@0.217.0/path/windows/normalize.ts": "78126170ab917f0ca355a9af9e65ad6bfa5be14d574c5fb09bb1920f52577780", + "https://deno.land/std@0.224.0/assert/assert.ts": "09d30564c09de846855b7b071e62b5974b001bb72a4b797958fe0660e7849834", + "https://deno.land/std@0.224.0/assert/assertion_error.ts": "ba8752bd27ebc51f723702fac2f54d3e94447598f54264a6653d6413738a8917" + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6f672ab --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +version: '3.8' + +services: + runner: + build: + context: . + dockerfile: Dockerfile + image: runner + container_name: runner + volumes: + - ./src:/elwood/runner/runtime + - ./deno.json:/elwood/runtime/deno.json + - ./actions:/elwood/runner/actions + - ./tmp:/tmp + - ./tmp-workspace/:/elwood/runner/workspace + working_dir: /elwood/runner/workspace + command: + [ + '/elwood/runner/bin/deno', + 'run', + '--config', + '/elwood/runtime/deno.json', + '-A', + '--unstable-worker-options', + '/elwood/runner/runtime/launch.ts', + ] diff --git a/src/constants.ts b/src/constants.ts new file mode 100644 index 0000000..f094a82 --- /dev/null +++ b/src/constants.ts @@ -0,0 +1,15 @@ +import type { RunnerDefinition } from "./types.ts"; + +export const RunnerStatus: Record = { + Pending: "pending", + Running: "running", + Complete: "complete", +} as const; + +export const RunnerResult: Record = { + None: "none", + Success: "success", + Failure: "failure", + Cancelled: "cancelled", + Skipped: "skipped", +} as const; diff --git a/src/deps.ts b/src/deps.ts new file mode 100644 index 0000000..385d069 --- /dev/null +++ b/src/deps.ts @@ -0,0 +1,12 @@ +// ASSERT +export { assert } from "jsr:@std/assert@^0.226.0/assert"; +export { assertEquals } from "jsr:@std/assert@^0.226.0/assert-equals"; +export { assertRejects } from "jsr:@std/assert@^0.226.0/assert-rejects"; + +// PATH +export { basename } from "jsr:@std/path@^0.225.2/basename"; +export { dirname } from "jsr:@std/path@^0.225.2/dirname"; +export { fromFileUrl } from "jsr:@std/path@^0.225.2/from-file-url"; +export { join } from "jsr:@std/path@^0.225.2/join"; +export { toFileUrl } from "jsr:@std/path@^0.225.2/to-file-url"; +export { isAbsolute } from "jsr:@std/path@^0.225.2/is-absolute"; diff --git a/src/launch.ts b/src/launch.ts new file mode 100644 index 0000000..18686ed --- /dev/null +++ b/src/launch.ts @@ -0,0 +1,98 @@ +import { assert, join } from "./deps.ts"; + +import { Manager } from "./manager.ts"; +import type { RunnerDefinition } from "./types.ts"; + +const instructions: RunnerDefinition.Normalized = { + name: "default", + jobs: [ + { + id: "job-1", + name: "default", + steps: [ + { + id: "xx", + name: "xx", + action: "run", + if: "true", + with: { + script: "echo 'hello=world' > $ELWOOD_OUTPUT", + }, + permissions: { + env: [], + read: [], + write: [], + run: ["bash"], + net: false, + }, + }, + // { + // id: "x", + // name: "install-ffmpeg", + // action: "install/ffmpeg", + // with: {}, + // permissions: { + // env: [], + // read: [], + // write: [], + // run: ["tar"], + // net: true, + // }, + // }, + + // { + // id: "step-1", + // name: "copy", + // action: "bin://ffmpeg", + // with: { + // args: ["-version"], + // }, + // permissions: { + // env: [], + // read: ["/tmp"], + // write: [], + // run: [], + // net: false, + // }, + // }, + ], + }, + ], +}; + +if (import.meta.main) { + main(); +} + +async function main() { + const workspaceDir = Deno.env.get("ELWOOD_RUNNER_WORKSPACE_DIR"); + const executionUid = Deno.env.get("ELWOOD_RUNNER_EXECUTION_UID"); + const executionGid = Deno.env.get("ELWOOD_RUNNER_EXECUTION_GID"); + + assert(workspaceDir, "ELWOOD_RUNNER_WORKSPACE_DIR not set"); + assert( + Deno.statSync(workspaceDir)?.isDirectory, + "Workspace dir does not exist", + ); + assert(executionUid, "ELWOOD_RUNNER_EXECUTION_UID not set"); + assert(executionGid, "ELWOOD_RUNNER_EXECUTION_GID not set"); + + const manager = new Manager({ + workspaceDir, + stdActionsPrefix: "file:///elwood/runner/actions", + executionGid: Number(executionGid), + executionUid: Number(executionUid), + }); + + // we're going to cleanup any previous executions + for await (const entry of Deno.readDir(workspaceDir)) { + if (entry.isDirectory) { + await Deno.remove(join(workspaceDir, entry.name), { recursive: true }); + } + } + + await manager.prepare(); + const execution = await manager.executeDefinition(instructions); + + console.log(execution.getCombinedState()); +} diff --git a/src/libs/expression-worker.ts b/src/libs/expression-worker.ts new file mode 100644 index 0000000..cebbeb3 --- /dev/null +++ b/src/libs/expression-worker.ts @@ -0,0 +1,66 @@ +import { Json } from "../types.ts"; + +// Worker API +// these functions are available in the worker's +// execution context +Object.assign(self, { + toJson(value: Json) { + return `json:${JSON.stringify(value)}`; + }, + + fromJson(value: string) { + if (value.startsWith("json:")) { + return JSON.parse(value.substring(5)); + } + + return JSON.parse(value); + }, +}); + +// instal api +// these functions are NOT available to the worker +// function and act only as a way to communicate +// with the parent +const __elwood_internal = { + postMessage(data: unknown) { + // deno-lint-ignore ban-ts-comment + // @ts-ignore + self.postMessage(data); + }, +}; + +// listen for message events which will only +// be sent from the parent +self.addEventListener("message", (e) => { + const event = e as MessageEvent; + + try { + const result = (0, eval)(event.data.code); + + if (result && result.then) { + result.then((value: unknown) => { + __elwood_internal.postMessage({ + type: "evaluate", + result: value, + }); + }) + .cache((err: Error) => { + __elwood_internal.postMessage({ + type: "error", + error: err, + }); + }); + return; + } + + __elwood_internal.postMessage({ + type: result, + result, + }); + } catch (err) { + __elwood_internal.postMessage({ + type: "error", + error: err, + }); + } +}); diff --git a/src/libs/expression.test.ts b/src/libs/expression.test.ts new file mode 100644 index 0000000..f0b29c9 --- /dev/null +++ b/src/libs/expression.test.ts @@ -0,0 +1,66 @@ +import { assertEquals, assertRejects } from "../deps.ts"; + +import { evaluateExpress, normalizeExpressionResult } from "./expression.ts"; + +Deno.test("evaluateExpress()", async function () { + assertEquals( + await evaluateExpress("${{ hello.world }}", { + hello: { world: "I am Elwood" }, + }), + "I am Elwood", + ); + + assertEquals( + await evaluateExpress("hello.world", { + hello: { world: "I am Elwood" }, + }), + "hello.world", + ); + + assertEquals( + await evaluateExpress("${{ 1 + 1 }}", { + hello: { world: "I am Elwood" }, + }), + normalizeExpressionResult(2), + ); + + assertEquals( + await evaluateExpress("${{ hello }}", { + hello: { world: "I am Elwood" }, + }), + normalizeExpressionResult({ world: "I am Elwood" }), + ); + + assertEquals( + await evaluateExpress("${{ 42 === my_age }}", { + my_age: 42, + }), + normalizeExpressionResult(true), + ); + + assertEquals( + await evaluateExpress("${{ null }}"), + normalizeExpressionResult(null), + ); + + assertEquals( + await evaluateExpress("${{ d.one && d.two }}", { + d: { one: true, two: true }, + }), + normalizeExpressionResult(true), + ); + + assertRejects( + () => evaluateExpress("${{ d.one && d.two }}", {}), + ); + + assertEquals( + await evaluateExpress("${{ __elwood_internal }}", {}), + normalizeExpressionResult(undefined), + ); + + assertEquals( + await evaluateExpress("${{ toJson(say) }}", { say: { hello: "world" } }), + normalizeExpressionResult({ hello: "world" }), + ); +}); diff --git a/src/libs/expression.ts b/src/libs/expression.ts new file mode 100644 index 0000000..c0cf148 --- /dev/null +++ b/src/libs/expression.ts @@ -0,0 +1,102 @@ +import { assert } from "../deps.ts"; +import type { Json, JsonObject } from "../types.ts"; + +export enum ExpressionTokens { + Prefix = "${{", + Postfix = "}}", +} + +export async function evaluateExpress( + expression: Json, + state: JsonObject = {}, +): Promise { + assert(typeof state === "object", "State must be an object"); + + // if it's not a string, just return it normalized + if (typeof expression !== "string") { + return normalizeExpressionResult(expression); + } + + const trimmedExpression = expression.trim(); + + // if the expression doesn't start with ${{ + // then we can assume it's a simple string + if ( + !isEvaluableExpression(trimmedExpression) + ) { + return normalizeExpressionResult(expression); + } + + const worker = new Worker(import.meta.resolve("./expression-worker.ts"), { + type: "module", + deno: { + permissions: "none", + }, + }); + + const code = ` + __elwood_internal = undefined; + Object.assign(self, ${JSON.stringify(state)}); + ${trimmedExpression.replace("${{", "").replace("}}", "")} + `; + + worker.postMessage({ + type: "eval", + code, + }); + + return await new Promise((resolve, reject) => { + worker.onmessage = (event) => { + if (event.data.type === "error") { + reject(event.data.error); + } + + // stop + worker.terminate(); + + return resolve(normalizeExpressionResult(event.data.result)); + }; + }); +} + +export function normalizeExpressionResult(value: Json): string { + if (typeof value === "string") { + return value; + } + + return `json:${JSON.stringify(value)}`; +} + +export function isExpressionResultTruthy(value: string): boolean { + if (value.startsWith("json:")) { + const jsonValue = JSON.parse(value.replace("json:", "")); + + if (typeof jsonValue === "string") { + return isExpressionResultTruthy(jsonValue); + } + + if ( + jsonValue === false || jsonValue === null || jsonValue === undefined || + jsonValue === 0 + ) { + return false; + } + + return true; + } + + return value === "true" || value === "1"; +} + +export function isEvaluableExpression(value: string): boolean { + return ( + value.startsWith(ExpressionTokens.Prefix) && + value.endsWith(ExpressionTokens.Postfix) + ); +} + +export function makeEvaluableExpression(value: string): string { + return isEvaluableExpression(value) + ? value + : `${ExpressionTokens.Prefix}${value}${ExpressionTokens.Postfix}`; +} diff --git a/src/libs/folder.ts b/src/libs/folder.ts new file mode 100644 index 0000000..229e60d --- /dev/null +++ b/src/libs/folder.ts @@ -0,0 +1,32 @@ +import { isAbsolute, join } from "../deps.ts"; + +export class Folder { + constructor(readonly path: string) { + Deno.mkdirSync(path, { recursive: true }); + } + + async mkdir(...paths: string[]) { + await Deno.mkdir(this.join(...paths), { recursive: true }); + return new Folder(this.join(...paths)); + } + + join(...paths: string[]) { + return join(this.path, ...paths); + } + + async remove() { + await Deno.remove(this.path, { recursive: true }); + } + + async writeText(fileName: string, content: string): Promise { + const filePath = this.join(fileName); + await Deno.writeTextFile(filePath, content); + return filePath; + } + + async readText(fileName: string): Promise { + return await Deno.readTextFile( + isAbsolute(fileName) ? fileName : this.join(fileName), + ); + } +} diff --git a/src/libs/resolve-action-url.ts b/src/libs/resolve-action-url.ts new file mode 100644 index 0000000..e6bebf1 --- /dev/null +++ b/src/libs/resolve-action-url.ts @@ -0,0 +1,46 @@ +import { basename, dirname, fromFileUrl, join } from "../deps.ts"; + +import type { RunnerDefinition } from "../types.ts"; + +export type ResolveActionUrlOptions = { + stdPrefix: string; +}; + +export async function resolveActionUrl( + action: RunnerDefinition.Step["action"], + options: ResolveActionUrlOptions, +): Promise { + if (action.includes("://")) { + const url = new URL(action); + + switch (url.protocol) { + case "bin:": + return new URL( + `?bin=${url.hostname}`, + await resolveActionUrl("run", options), + ); + + default: + return url; + } + } + + const base = basename(action); + const ext = action.endsWith(".ts") ? "" : ".ts"; + + return new URL( + `${options.stdPrefix}/${join(dirname(action), `${base}${ext}`)}`, + ); +} + +export function resolveActionUrlForDenoCommand(url: URL): string { + switch (url.protocol) { + case "file:": + return fromFileUrl(url); + case "http:": + case "https:": + return url.href; + default: + throw new Error(`Unsupported protocol: ${url.protocol}`); + } +} diff --git a/src/libs/run-deno.test.ts b/src/libs/run-deno.test.ts new file mode 100644 index 0000000..60f6856 --- /dev/null +++ b/src/libs/run-deno.test.ts @@ -0,0 +1,86 @@ +import { assertEquals } from "../deps.ts"; + +import { permissionObjectToFlags } from "./run-deno.ts"; + +Deno.test("permissionObjectToFlags()", function () { + assertEquals( + permissionObjectToFlags({ + env: true, + }), + [ + "--allow-env", + "--deny-sys", + "--deny-hrtime", + "--deny-net", + "--deny-ffi", + "--deny-read", + "--deny-run", + "--deny-write", + ], + ); + + assertEquals( + permissionObjectToFlags({ + env: ["a", "b"], + }), + [ + "--allow-env=a,b", + "--deny-sys", + "--deny-hrtime", + "--deny-net", + "--deny-ffi", + "--deny-read", + "--deny-run", + "--deny-write", + ], + ); + + assertEquals( + permissionObjectToFlags({ + env: "inherit", + }), + [ + "--deny-env", + "--deny-sys", + "--deny-hrtime", + "--deny-net", + "--deny-ffi", + "--deny-read", + "--deny-run", + "--deny-write", + ], + ); + + assertEquals( + permissionObjectToFlags({ + env: false, + }), + [ + "--deny-env", + "--deny-sys", + "--deny-hrtime", + "--deny-net", + "--deny-ffi", + "--deny-read", + "--deny-run", + "--deny-write", + ], + ); + + assertEquals( + permissionObjectToFlags({ + env: true, + ffi: [new URL("https://elwood.dev")], + }), + [ + "--allow-env", + "--deny-sys", + "--deny-hrtime", + "--deny-net", + "--allow-ffi=https://elwood.dev/", + "--deny-read", + "--deny-run", + "--deny-write", + ], + ); +}); diff --git a/src/libs/run-deno.ts b/src/libs/run-deno.ts new file mode 100644 index 0000000..fff6d6c --- /dev/null +++ b/src/libs/run-deno.ts @@ -0,0 +1,68 @@ +export type ExecuteDenoRunOptions = Omit & { + file: string; + permissions?: Deno.PermissionOptionsObject; + args?: string[]; +}; + +export async function executeDenoRun( + options: ExecuteDenoRunOptions, +): Promise { + const { file, permissions, args = [], ...cmdOptions } = options; + + return await executeDenoCommand({ + args: ["run", ...permissionObjectToFlags(permissions ?? {}), ...args, file], + ...cmdOptions, + }); +} + +export type ExecuteDenoCommand = Deno.CommandOptions; + +export async function executeDenoCommand( + options: ExecuteDenoCommand, +): Promise { + console.log("executeDenoCommand", JSON.stringify(options, null, 2)); + + const cmd = new Deno.Command(Deno.execPath(), { + stdout: "inherit", + stderr: "inherit", + ...options, + }); + + return await cmd.output(); +} + +export function permissionObjectToFlags( + options: Deno.PermissionOptionsObject, +): string[] { + const defaults = { + env: false, + sys: false, + hrtime: false, + net: false, + ffi: false, + read: false, + run: false, + write: false, + }; + + console.log(options); + + return Object.entries({ ...defaults, ...options }).reduce( + (acc, [name, value]) => { + if (value === false || value === "inherit") { + return [...acc, `--deny-${name}`]; + } + + if (value === true || (Array.isArray(value) && value.length === 0)) { + return [...acc, `--allow-${name}`]; + } + + if (Array.isArray(value)) { + return [...acc, `--allow-${name}=${value.join(",")}`]; + } + + return acc; + }, + [] as string[], + ); +} diff --git a/src/libs/short-id.ts b/src/libs/short-id.ts new file mode 100644 index 0000000..1c52933 --- /dev/null +++ b/src/libs/short-id.ts @@ -0,0 +1,48 @@ +// borrowed from https://deno.land/x/short_uuid@v3.0.0-rc1/mod.ts?source + +class ShortUniqueId { + counter = 0; + dict = ["t", "k"]; + + dictIndex: number = 0; + dictRange: number[] = []; + lowerBound: number = 0; + upperBound: number = 0; + dictLength: number = 2; + + seq(): string { + return this.sequentialUUID(); + } + + /** + * Generates UUID based on internal counter that's incremented after each ID generation. + * @alias `const uid = new ShortUniqueId(); uid.seq();` + */ + sequentialUUID(): string { + let counterDiv: number; + let counterRem: number; + let id: string = ""; + + counterDiv = this.counter; + + /* tslint:disable no-constant-condition */ + while (true) { + counterRem = counterDiv % this.dictLength; + counterDiv = Math.trunc(counterDiv / this.dictLength); + id += this.dict[counterRem]; + if (counterDiv === 0) { + break; + } + } + /* tslint:enable no-constant-condition */ + this.counter += 1; + + return id; + } +} + +const suid = new ShortUniqueId(); + +export function shortId(prefix = ""): string { + return [prefix, suid.seq()].join("--"); +} diff --git a/src/libs/state.ts b/src/libs/state.ts new file mode 100644 index 0000000..7c504ed --- /dev/null +++ b/src/libs/state.ts @@ -0,0 +1,87 @@ +// deno-lint-ignore-file require-await +import { RunnerResult, RunnerStatus } from "../constants.ts"; +import type { RunnerDefinition } from "../types.ts"; +import { shortId } from "./short-id.ts"; + +export abstract class State implements RunnerDefinition.State { + abstract id: string; + abstract name: string; + + protected _status: RunnerDefinition.Status = "pending"; + protected _result: RunnerDefinition.Result = "none"; + protected _data: RunnerDefinition.State["state"] = { + reason: null, + }; + + #startTime: number | null = null; + + get status() { + return this._status; + } + + get result() { + return this._result; + } + + get state() { + return { + status: this.status, + result: this.result, + ...this._data, + }; + } + + shortId(prefix: string = ""): string { + return shortId(prefix); + } + + setState(name: string, value: V) { + this._data[name] = value; + } + + getCombinedState() { + return { + id: this.id, + name: this.name, + ...this.state, + }; + } + + async fail(reason: string = "") { + this._status = RunnerStatus.Complete; + this._result = RunnerResult.Failure; + this._data.reason = reason; + } + + async succeed(reason: string = "") { + this._status = RunnerStatus.Complete; + this._result = RunnerResult.Success; + this._data.reason = reason; + } + + async skip(reason: string = "") { + this._status = RunnerStatus.Complete; + this._result = RunnerResult.Skipped; + this._data.reason = reason; + } + + start() { + this.#startTime = performance.now(); + } + + stop() { + if (this.#startTime === null) { + console.error("State.stop() called without State.start()"); + } + + if (this.#startTime !== null) { + const end = performance.now(); + + this.setState("timing", { + start: performance.timeOrigin + this.#startTime, + end: performance.timeOrigin + end, + elapsed: end - this.#startTime, + }); + } + } +} diff --git a/src/libs/variables.test.ts b/src/libs/variables.test.ts new file mode 100644 index 0000000..49afccd --- /dev/null +++ b/src/libs/variables.test.ts @@ -0,0 +1,56 @@ +import { assertEquals } from "../deps.ts"; + +import { + parseVariableFile, + replaceVariablePlaceholdersInVariables, +} from "./variables.ts"; + +Deno.test("replaceVariablePlaceholdersInVariables()", async function () { + assertEquals( + await replaceVariablePlaceholdersInVariables({ + ENV_1: "test", + ENV_2: "test $ENV_1", + }), + { + ENV_1: "test", + ENV_2: "test test", + }, + ); + + assertEquals( + await replaceVariablePlaceholdersInVariables({ + ENV_1: "test", + ENV_2: "test $ENV_1 $ENV_1", + }), + { + ENV_1: "test", + ENV_2: "test test test", + }, + ); +}); + +Deno.test("parseVariableFile", async function () { + assertEquals( + await parseVariableFile("name=value"), + { name: "value" }, + ); + + assertEquals( + await parseVariableFile("name<, +) { + return await Promise.resolve( + Object.entries(vars).reduce((acc, [key, value]) => { + let value_ = value; + + for (const [varKey, varValue] of Object.entries(vars)) { + // straight replace + value_ = value_.replaceAll( + new RegExp("\\$" + varKey, "g"), + varValue, + ); + + // if this is an input variable, replace the + // prefix so people can access it directly + if (varKey.includes("INPUT_")) { + value_ = value_.replaceAll( + new RegExp("\\$" + varKey.replace("INPUT_", ""), "g"), + varValue, + ); + } + } + + return { + ...acc, + [key]: value_, + }; + }, {}), + ); +} + +export async function parseVariableFile( + content: string, +): Promise> { + const vars: Record = {}; + const lines = content.split("\n"); + let currentName = ""; + let currentValue = ""; + let eof: string | null = null; + + for (const line of lines) { + if (eof && line.includes(eof)) { + const [end] = line.split(eof); + vars[currentName] = (currentValue + end).trim(); + continue; + } + + if (line.includes("<<")) { + const [name_, eof_] = line.split("<<"); + currentName = name_!.trim(); + eof = eof_!.trim(); + currentValue = ""; + continue; + } + + if (eof) { + currentValue += line + "\n"; + } + + if (!line.includes("=")) { + continue; + } + + const [name, value] = line.split("="); + vars[name!.trim()] = value!.trim(); + } + + return await Promise.resolve(vars); +} diff --git a/src/manager.ts b/src/manager.ts new file mode 100644 index 0000000..ae8aa3d --- /dev/null +++ b/src/manager.ts @@ -0,0 +1,61 @@ +import { Folder } from "./libs/folder.ts"; +import { Execution } from "./runtime/execution.ts"; +import { RunnerDefinition } from "./types.ts"; + +export type ManagerOptions = { + workspaceDir: string; + stdActionsPrefix: string; + executionUid: number; + executionGid: number; +}; + +export class Manager { + public readonly executions = new Map(); + + #workspaceDir: Folder; + + constructor(public readonly options: ManagerOptions) { + this.#workspaceDir = new Folder(options.workspaceDir); + } + + get workspaceDir(): Folder { + return this.#workspaceDir; + } + + async mkdir(inFolder: "workspace", ...parts: string[]): Promise { + switch (inFolder) { + case "workspace": + return await this.#workspaceDir.mkdir(...parts); + default: + throw new Error(`Unknown folder: ${inFolder}`); + } + } + + async prepare(): Promise { + await this.mkdir("workspace"); + } + + async executeDefinition( + def: RunnerDefinition.Normalized, + ): Promise { + const execution = new Execution(this, def, {}); + + this.executions.set(execution.id, execution); + + await execution.prepare(); + + // continue with execution if the state is pending + // if something failed in prepare, status will be complete + if (execution.status === "pending") { + await execution.execute(); + } + + return execution; + } + + async cleanup(): Promise { + for await (const [_, execution] of this.executions) { + await execution.workingDir.remove(); + } + } +} diff --git a/src/runtime/execution.ts b/src/runtime/execution.ts new file mode 100644 index 0000000..0b54d56 --- /dev/null +++ b/src/runtime/execution.ts @@ -0,0 +1,183 @@ +import { assert } from "../deps.ts"; + +import { Manager } from "../manager.ts"; +import type { RunnerDefinition } from "../types.ts"; +import { Job } from "./job.ts"; +import { executeDenoCommand } from "../libs/run-deno.ts"; +import { resolveActionUrlForDenoCommand } from "../libs/resolve-action-url.ts"; +import { State } from "../libs/state.ts"; +import { + executeDenoRun, + type ExecuteDenoRunOptions, +} from "../libs/run-deno.ts"; +import { Folder } from "../libs/folder.ts"; + +export type ExecutionOptions = unknown; + +export class Execution extends State { + readonly id: string; + readonly name = "execution"; + readonly #jobs = new Map(); + + #workingDir: Folder | null = null; + #contextDir: Folder | null = null; + #stageDir: Folder | null = null; + #cacheDir: Folder | null = null; + #binDir: Folder | null = null; + + constructor( + public readonly manager: Manager, + public readonly def: RunnerDefinition.Normalized, + public readonly options: ExecutionOptions, + ) { + super(); + this.id = this.shortId("execution"); + } + + get jobs(): Job[] { + return Array.from(this.#jobs.values()); + } + + get workingDir(): Folder { + assert(this.#workingDir !== null, "Execution not prepared"); + return this.#workingDir; + } + + get stageDir(): Folder { + assert(this.#stageDir !== null, "Execution not prepared"); + return this.#stageDir; + } + get cacheDir(): Folder { + assert(this.#cacheDir !== null, "Execution not prepared"); + return this.#cacheDir; + } + + get contextDir(): Folder { + assert(this.#contextDir !== null, "Execution not prepared"); + return this.#contextDir; + } + + get binDir(): Folder { + assert(this.#binDir !== null, "Execution not prepared"); + return this.#binDir; + } + + async prepare(): Promise { + this.#workingDir = await this.manager.mkdir("workspace", this.id); + this.#contextDir = await this.workingDir.mkdir("context"); + this.#stageDir = await this.workingDir.mkdir("stage"); + this.#cacheDir = await this.workingDir.mkdir("cache"); + this.#binDir = await this.workingDir.mkdir("bin"); + + // write our definition to the working directory + await Deno.writeTextFile( + this.workingDir.join("definition.json"), + JSON.stringify(this.def, null, 2), + ); + + const actionUrls: URL[] = []; + + for (const def of this.def.jobs) { + const job = new Job(this, def); + this.#jobs.set(job.id, job); + await job.prepare(); + + // loop through each job step and compile a list of action URLs + for (const step of job.steps) { + actionUrls.push(step.actionUrl!); + } + } + + // cache each action file + const results = await Promise.all( + actionUrls.map(async (url) => { + console.log(`Preloading action: ${url}`); + + return await executeDenoCommand({ + args: ["cache", resolveActionUrlForDenoCommand(url)], + env: this.getDenoEnv(), + }); + }), + ); + + if (results.some((item) => item.code !== 0)) { + const names = results.map((item, i) => [actionUrls[i], item.code]).filter( + (item) => item[1] !== 0, + ).map((item) => item[0].toString()).join(", "); + + await this.fail(`Failed to cache action files: ${names}`); + } + } + + async execute(): Promise { + console.log(`Executing: ${this.id}`); + + try { + this.start(); + + for (const job of this.jobs) { + if (job.status !== "pending") { + continue; + } + + await job.execute(); + } + + console.log("done jobs"); + + const hasFailure = this.jobs.some((job) => job.result === "failure"); + + if (hasFailure) { + await this.fail("Execution failed"); + return; + } + + await this.succeed(); + } catch (error) { + await this.fail(error.message); + } finally { + this.stop(); + } + } + + getCombinedState() { + return { + ...super.getCombinedState(), + jobs: this.jobs.map((job) => job.getCombinedState()), + }; + } + + getDenoEnv(): Record { + return { + DENO_DIR: this.cacheDir.join("deno"), + }; + } + + async executeDenoRun( + options: ExecuteDenoRunOptions, + ): ReturnType { + const { permissions = {}, env = {}, ...opts } = options; + + return await executeDenoRun({ + ...opts, + uid: this.manager.options.executionUid, + gid: this.manager.options.executionGid, + env: { + ...env, + ...this.getDenoEnv(), + ELWOOD_STAGE_DIR: this.stageDir.path, + ELWOOD_BIN_DIR: this.binDir.path, + PATH: [ + "/usr/local/sbin", + "/usr/local/bin", + "/usr/sbin", + "/usr/bin", + "/sbin", + "/bin", + this.binDir.path, + ].join(":"), + }, + permissions, + }); + } +} diff --git a/src/runtime/job.ts b/src/runtime/job.ts new file mode 100644 index 0000000..1f352bd --- /dev/null +++ b/src/runtime/job.ts @@ -0,0 +1,80 @@ +import { assert } from "../deps.ts"; +import { Execution } from "./execution.ts"; +import type { RunnerDefinition } from "../types.ts"; +import { State } from "../libs/state.ts"; +import { Folder } from "../libs/folder.ts"; +import { Step } from "./step.ts"; + +export class Job extends State { + readonly id: string; + readonly name: string; + + readonly #steps = new Map(); + + #contextDir: Folder | null = null; + + constructor( + public readonly execution: Execution, + public readonly def: RunnerDefinition.Job, + ) { + super(); + this.id = this.shortId("job"); + this.name = def.name; + } + + get steps(): Step[] { + return Array.from(this.#steps.values()); + } + + get contextDir(): Folder { + assert(this.#contextDir !== null, "Working dir not set"); + return this.#contextDir; + } + + async prepare(): Promise { + this.#contextDir = await this.execution.contextDir.mkdir(this.id); + + for (const def of this.def.steps) { + const step = new Step(this, def); + this.#steps.set(step.id, step); + await step.prepare(); + } + } + + async execute(): Promise { + console.log(`Running job: ${this.id}`); + + try { + this.start(); + + for (const step of this.steps) { + if (this.status !== "pending") { + continue; + } + + await step.execute(); + } + + const hasFailure = this.steps.some((job) => job.result === "failure"); + + if (hasFailure) { + await this.fail("Execution failed"); + return; + } + + await this.succeed(); + } catch (error) { + await this.fail(error.message); + } finally { + this.stop(); + } + } + + getCombinedState() { + return { + ...super.getCombinedState(), + definition: this.def, + steps: this.steps.map((step) => step.state), + }; + } +} diff --git a/src/runtime/step.ts b/src/runtime/step.ts new file mode 100644 index 0000000..1bcb9b1 --- /dev/null +++ b/src/runtime/step.ts @@ -0,0 +1,202 @@ +import { Job } from "./job.ts"; +import type { RunnerDefinition } from "../types.ts"; +import { + resolveActionUrl, + resolveActionUrlForDenoCommand, +} from "../libs/resolve-action-url.ts"; +import { State } from "../libs/state.ts"; +import { Folder } from "../libs/folder.ts"; +import { + evaluateExpress, + isExpressionResultTruthy, + makeEvaluableExpression, +} from "../libs/expression.ts"; +import { + parseVariableFile, + replaceVariablePlaceholdersInVariables, +} from "../libs/variables.ts"; + +import { assert } from "../deps.ts"; +import { ExecuteDenoRunOptions } from "../libs/run-deno.ts"; + +export class Step extends State { + readonly id: string; + readonly name: string; + + public actionUrl: URL | null = null; + + #contextDir: Folder | null = null; + + constructor( + public readonly job: Job, + public readonly def: RunnerDefinition.Step, + ) { + super(); + this.id = this.shortId("step"); + this.name = def.name; + } + + get contextDir(): Folder { + assert(this.#contextDir !== null, "Context dir not set"); + return this.#contextDir; + } + + getCombinedState() { + return { + ...super.getCombinedState(), + definition: this.def, + }; + } + + async prepare(): Promise { + this.#contextDir = await this.job.contextDir.mkdir(this.id); + + this.actionUrl = await resolveActionUrl(this.def.action, { + stdPrefix: this.job.execution.manager.options.stdActionsPrefix, + }); + } + + async execute(): Promise { + assert(this.actionUrl, "Action URL not resolved"); + + console.log(`Running step: ${this.name} ${this.actionUrl}`); + + try { + this.start(); + + // check to see if this step should be skipped + const shouldSkip = !isExpressionResultTruthy( + await evaluateExpress(makeEvaluableExpression(this.def.if)), + ); + + if (shouldSkip) { + await this.skip('Step was skipped due to "if" condition'); + return; + } + + const outputFilePath = await this.contextDir.writeText( + this.shortId("set-output"), + "", + ); + const envFilePath = await this.contextDir.writeText( + this.shortId("set-env"), + "", + ); + + const result = await this.job.execution.executeDenoRun({ + ...(await this._getDenoRunOptions({ + env: { + ELWOOD_OUTPUT: outputFilePath, + ELWOOD_ENV: envFilePath, + }, + })), + file: resolveActionUrlForDenoCommand(this.actionUrl), + cwd: this.contextDir.path, + }); + + this.setState( + "output", + await parseVariableFile(await this.contextDir.readText(outputFilePath)), + ); + + this.setState( + "env", + await parseVariableFile(await this.contextDir.readText(envFilePath)), + ); + + switch (result.code) { + case 0: { + await this.succeed(); + break; + } + default: { + await this.fail(`Action failed with code ${result.code}`); + await this.job.fail(`Step ${this.name} failed`); + } + } + } catch (error) { + await this.fail(error.message); + } finally { + this.stop(); + } + } + + async _getDenoRunOptions( + init: Omit = {}, + ): Promise> { + const commandInputEnv = await this._getCommandInputEnv(); + const argsFromActionUrl: Record = {}; + + // if the action has search params + // pass them to the action as ARG_ env variables + if (this.actionUrl?.searchParams) { + for (const [name, value] of this.actionUrl.searchParams.entries()) { + argsFromActionUrl[`ARG_${name.toUpperCase()}`] = value; + } + } + + const env = { + ...(init.env ?? {}), + ...argsFromActionUrl, + ...commandInputEnv, + }; + + // if the value is an array, merge it with the append array + // otherwise return the value. + function _arrayOrTrue( + value: string[] | boolean, + append: Array, + ): string[] | boolean { + if (Array.isArray(value)) { + return [ + ...value, + ...append.filter(Boolean) as string[], + ]; + } + + return value; + } + + return { + ...init, + permissions: { + ...this.def.permissions, + read: _arrayOrTrue(this.def.permissions.read, [ + init.env?.ELWOOD_ENV, + init.env?.ELWOOD_OUTPUT, + this.contextDir.path, + this.job.execution.stageDir.path, + "", + ]), + write: _arrayOrTrue( + this.def.permissions.write, + [ + init.env?.ELWOOD_ENV, + init.env?.ELWOOD_OUTPUT, + this.contextDir.path, + this.job.execution.stageDir.path, + this.job.execution.binDir.path, + ], + ), + env: _arrayOrTrue( + this.def.permissions.env, + [...Object.keys(argsFromActionUrl), ...Object.keys(commandInputEnv)], + ), + }, + env: await replaceVariablePlaceholdersInVariables(env), + }; + } + + async _getCommandInputEnv(): Promise> { + const withDefinition = this.def.with ?? {}; + const inputEnv: Record = {}; + + for (const [key, value] of Object.entries(withDefinition)) { + inputEnv[`INPUT_${key.toLocaleUpperCase()}`] = await evaluateExpress( + value, + ); + } + + return inputEnv; + } +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..d37f57d --- /dev/null +++ b/src/types.ts @@ -0,0 +1,50 @@ +// deno-lint-ignore-file no-namespace + +// deno-lint-ignore no-explicit-any +export type Json = any; +export type JsonObject = Record; + +export namespace RunnerDefinition { + export type Status = "pending" | "running" | "complete"; + export type Result = "none" | "success" | "failure" | "cancelled" | "skipped"; + + export interface State { + status: Status; + result: Result; + state: { + [key: string]: unknown; + reason: string | null; + }; + } + + export interface Normalized { + name: string; + jobs: Job[]; + } + + export interface Job { + id: string; + name: string; + steps: Step[]; + } + + export interface Step { + id: string; + name: string; + action: string; + if: string; + with: Record< + string, + string | string[] | number | number[] | Record + >; + permissions: StepPermission; + } + + export interface StepPermission { + env: string[] | boolean; + read: string[] | boolean; + write: string[] | boolean; + net: string[] | boolean; + run: string[] | boolean; + } +}