AP-856: scaffold a basic flask+celery app

.github/workflows/build.yml

@@ -0,0 +1,81 @@
+name: Build / Test / Push
+
+on:
+  push:
+    branches:
+      - "**"
+  workflow_call:
+  workflow_dispatch:
+
+env:
+  BUILD_SUFFIX: -build-${{ github.run_id }}_${{ github.run_attempt }}
+
+jobs:
+  docker-build:
+    uses: BerkeleyLibrary/.github/.github/workflows/docker-build.yml@3.1.0
+    with:
+      image: ghcr.io/${{ github.repository }}
+    secrets: inherit
+
+  test:
+    runs-on: ubuntu-24.04
+    needs: docker-build
+    env:
+      COMPOSE_FILE: docker-compose.yml:docker-compose.ci.yml
+      DOCKER_APP_IMAGE: ${{ needs.docker-build.outputs.image }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Set up Docker Compose
+        uses: docker/setup-compose-action@v2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Basic setup
+        run: |
+          ARTIFACTS_DIR="${RUNNER_TEMP}/artifacts"
+          mkdir -p "$ARTIFACTS_DIR"
+          echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> $GITHUB_ENV
+          echo "TEST_START=$(date +%s)" >> $GITHUB_ENV
+      - name: Setup the stack
+        run: |
+          docker compose run app bin/dbinit
+          docker compose up --wait
+      - name: Install testing and linting dependencies
+        run: |
+          docker compose exec app pip install --no-cache-dir -e .[test,lint]
+      - name: Run pytest
+        run: |
+          docker compose exec app pytest
+
+      - name: Copy out artifacts
+        if: ${{ always() }}
+        run: |
+          docker compose cp app:/app/artifacts "${ARTIFACTS_DIR}" || true
+          docker compose logs > "${ARTIFACTS_DIR}/docker-compose-services.log"
+          docker compose config > "${ARTIFACTS_DIR}/docker-compose.merged.yml"
+          docker events --json --since $TEST_START --until `date +%s` | tee "${ARTIFACTS_DIR}/docker-events.json"
+
+      - name: Upload the test report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: quiabo Build Report (${{ github.run_id }}_${{ github.run_attempt }})
+          path: ${{ env.ARTIFACTS_DIR }}
+          if-no-files-found: error
+
+  push:
+    needs:
+      - docker-build
+      - test
+    uses: BerkeleyLibrary/.github/.github/workflows/docker-push.yml@3.1.0
+    with:
+      image: ghcr.io/${{ github.repository }}
+      build-image-arm64: ${{ needs.docker-build.outputs.image-arm64 }}
+      build-image-x64: ${{ needs.docker-build.outputs.image-x64 }}
+    secrets: inherit

.github/workflows/release.yml

@@ -0,0 +1,13 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - '**'
+  workflow_dispatch:
+
+jobs:
+  release:
+    uses: BerkeleyLibrary/.github/.github/workflows/docker-release.yml@main
+    with:
+      image: ghcr.io/${{ github.repository }}

.gitignore

@@ -216,3 +216,7 @@ __marimo__/
 
 # Streamlit
 .streamlit/secrets.toml
+
+# other stuff
+artifacts/*
+uv.lock

Dockerfile

@@ -0,0 +1,47 @@
+FROM python:3.14-slim AS reqs
+
+ENV APP_USER=quiabo
+ENV APP_UID=40098
+ENV VIRTUAL_ENV=/venv
+
+RUN apt-get update -y && apt-get upgrade -y \
+    && apt-get install -y --no-install-recommends \
+        gcc \
+        libpq-dev \
+        libxml2-dev \
+        python3-dev \ 
+        postgresql-client \
+    && rm -rf /var/lib/apt/lists/
+
+RUN groupadd --system --gid $APP_UID $APP_USER \
+    && useradd --home-dir /app --system --uid $APP_UID --gid $APP_USER $APP_USER
+
+RUN mkdir -p /app && mkdir -p /venv
+
+RUN chown -R $APP_USER:$APP_USER /app /venv
+
+USER $APP_USER
+
+RUN python -m venv /venv
+ENV PATH=/venv/bin:$PATH
+
+RUN python -m pip install -U setuptools
+
+WORKDIR /app
+
+COPY pyproject.toml .
+
+FROM reqs AS app
+
+WORKDIR /app
+USER $APP_USER
+
+COPY quiabo quiabo
+COPY bin bin
+COPY README.md README.md
+COPY test test
+RUN pip install --no-cache-dir -e . 
+
+EXPOSE 8000
+
+CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0", "quiabo:app"]

README.md

@@ -1,2 +1,59 @@
-# quiabo
-Backend web service for running OCR processes
+# ![quiabo logo](quiabo/static/quiabo.png) quiabo
+
+`quiabo` is a backend web service for running OCR jobs implemented as a Flask and Celery application.
+
+## Dependencies
+
+Python dependencies are declared in `pyproject.toml`.
+
+## Development
+
+Spin up the application using Docker Compose. There are number of dependencies (Postgres and Redis) as well as Flask/Celery app components (`app`, `worker`, and optionially `flower`). Redis serves as the Celery broker (source of jobs) and Postgres is the Celery results backend.
+
+```bash
+# Build the Docker image for app, worker, and flower
+docker compose build
+
+# Create the postgres database; only needed the first time
+docker compose run --rm app bin/dbinit
+
+# Start the Flask app, which will be running on http://localhost:8000/
+docker compose up --detach
+
+# Optionally start Flower, which is a dashboard for the Celery queue and
+# will be running on http://localhost:5555/
+docker compose up --profile flower --detach
+```
+
+## Testing
+
+Once the stack is started, execute the tests by running `pytest` in one
+of the running containers. Note that testing and linting dependencies are
+not installed by default, so you'll need to do that too.
+
+```bash
+# Install the testing and linting dependencies
+
+docker compose exec app pip install --no-cache-dir -e .[test,lint]
+
+# Run all the tests
+docker compose exec app pytest
+
+# Run tests with a specific marker
+# Example: only run the unit tests
+docker compose exec app pytest -m unit
+```
+
+Test results/reports are written to `./artifacts/pytest`.
+
+## Configuration
+
+`quiabo`'s configuration is handled by environment variables using Flask's
+[`from_prefixed_env()`](https://flask.palletsprojects.com/en/stable/config/#configuring-from-environment-variables) method, using `QUIABO` as the
+prefix. Celery configuration is set using the same method. At a minimum,
+you will need to set the following:
+
+| Environment variable | Purpose | Example |
+| -------------------- | ------- | ------- |
+| `QUIABO_CELERY__broker_url` | Connection URL for the Celery broker (e.g. Redis) | `redis://redis:6379` |
+| `QUIABO_CELERY__result_backend` | SQLAlchemy connection URL for the Celery result backend (e.g. Postgres) | `db+postgresql://postgres:postgres@db:5432/quiabo` |

bin/dbinit

@@ -0,0 +1,20 @@
+#!/bin/sh -e
+
+# Initialise the configured database environment for use with quiabo
+#
+# Copyright © 2026 The Regents of the University of California.  MIT license.
+
+# Use the credentials from the app's environment.
+export PGHOST=${POSTGRES_HOST:-db}
+export PGPORT=${POSTGRES_PORT:-5432}
+export PGUSER=${POSTGRES_USER}
+export PGPASSWORD=${POSTGRES_PASSWORD}
+export PGDATABASE=${POSTGRES_DB}
+
+# Determine if the database needs to be created or not.
+if [ "$(psql -d template1 -t -A -c "SELECT COUNT(*) FROM pg_database WHERE datname='${POSTGRES_DB}';")" = '0' ]; then
+  echo Creating database ${POSTGRES_DB}...
+  createdb
+else
+  echo Database ${POSTGRES_DB} already exists. Bye bye!
+fi

docker-compose.ci.yml

@@ -0,0 +1,22 @@
+services:
+  db:
+    volumes: !reset
+
+  redis:
+    volumes: !reset
+
+  app:
+    build: !reset
+    depends_on: !reset
+    ports: !reset
+    image: ${DOCKER_APP_IMAGE}
+    env_file: !override env.example
+    volumes: !reset
+
+  worker:
+    build: !reset
+    image: ${DOCKER_APP_IMAGE}
+    depends_on: !reset
+    ports: !reset
+    env_file: !override env.example
+    volumes: !reset

docker-compose.yml

@@ -0,0 +1,107 @@
+x-dbconfig:
+  environment: &dbconfig
+    POSTGRES_USER: ${POSTGRES_USER:-quiabo}
+    POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-quiabo}
+    POSTGRES_HOST: ${POSTGRES_HOST:-db}
+    POSTGRES_PORT: ${POSTGRES_PORT:-5432}
+    POSTGRES_DB: ${POSTGRES_DB:-quiabo}
+
+x-quiabo-common:
+  environment: &quiabo-common-environment
+    QUIABO_CELERY__broker_url: ${QUIABO__CELERY__broker_url:-redis://redis:6379}
+    QUIABO_CELERY__result_backend: ${QUIABO__CELERY__result_backend:-db+postgresql://${POSTGRES_USER:-quiabo}:${POSTGRES_PASSWORD:-quiabo}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-quiabo}}
+    QUIABO_CELERY__task_default_queue: ${QUIABO_CELERY__task_default_queue:-quiabo}
+    QUIABO_CELERY__task_ignore_result: ${QUIABO_CELERY__task_ignore_result:-false}
+
+
+services:
+  db:
+    environment:
+      <<: *dbconfig
+    image: postgres:16
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER}"]
+      interval: 10s
+      retries: 5
+      start_period: 5s
+    ports:
+      - 5432:5432
+    restart: always
+    volumes:
+      - postgres-db-volume:/var/lib/postgresql/data
+
+  app:
+    build:
+      context: .
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    environment:
+      <<: 
+        - *quiabo-common-environment
+        - *dbconfig
+    init: true
+    restart: always
+    ports:
+      - 8000:8000
+    volumes:
+      - ./quiabo:/app/quiabo:rw
+
+  worker:
+    build:
+      context: .
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    environment:
+      <<:
+        - *quiabo-common-environment
+        - *dbconfig
+    init: true
+    restart: always
+    command: celery -A quiabo.celery_app worker --loglevel INFO
+    volumes:
+      - ./quiabo:/app/quiabo:rw
+
+  flower:
+    build:
+      context: .
+    profiles:
+      - flower
+    depends_on:
+      redis:
+        condition: service_healthy
+    environment:
+      <<: *quiabo-common-environment
+    init: true
+    restart: always
+    command: celery -A quiabo.celery_app flower
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    ports:
+      - 127.0.0.1:5555:5555
+    volumes:
+      - ./quiabo:/app/quiabo:rw
+
+  redis:
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 30s
+      retries: 50
+      start_period: 30s
+    image: redis:8
+    ports:
+      - 6379:6379
+    restart: always
+
+volumes:
+  postgres-db-volume:

env.example

@@ -0,0 +1,3 @@
+POSTGRES_USER=root
+POSTGRES_PASSWORD=root
+POSTGRES_DB=quiabo