Merge pull request #1183 from datajoint/docker-img-refactor

Refactor Docker image and package builds

Author: dimitri-yatsenko

.devcontainer/Dockerfile

@@ -8,6 +8,6 @@ RUN \
     pip uninstall datajoint -y
 
 USER root
-ENV DJ_HOST fakeservices.datajoint.io
+ENV DJ_HOST db
 ENV DJ_USER root
 ENV DJ_PASS password

.devcontainer/devcontainer.json

@@ -5,7 +5,7 @@
 	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
 	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
 	"dockerComposeFile": [
-		"../LNX-docker-compose.yml",
+		"../docker-compose.yaml",
 		"docker-compose.yml"
 	],
 	// The 'service' property is the name of the service for the container that VS Code should
@@ -28,7 +28,7 @@
 	// "runServices": [],
 	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
 	"shutdownAction": "stopCompose",
-	"onCreateCommand": "python3 -m pip install -e .",
+	"onCreateCommand": "python3 -m pip install -q -e .[test]",
 	"features": {
 		"ghcr.io/devcontainers/features/git:1": {},
 		"ghcr.io/devcontainers/features/docker-in-docker:2": {},

.github/workflows/development.yaml

@@ -29,10 +29,13 @@ jobs:
       COMPOSE_HTTP_TIMEOUT: "120"
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{matrix.py_ver}}
       - name: Validate version and release notes
         run: |
           DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
-          RELEASE_BODY=$(python -c \
+          RELEASE_BODY=$(python3 -c \
             'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \
           )
           echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
@@ -41,8 +44,8 @@ jobs:
           echo "EOF" >> $GITHUB_ENV
       - name: Build pip artifacts
         run: |
-          export HOST_UID=$(id -u)
-          docker compose -f docker-compose-build.yaml up --exit-code-from app --build
+          python3 -m pip install build
+          python3 -m build .
           echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
       - if: matrix.py_ver == '3.9' && matrix.distro == 'debian'
         name: Add pip artifacts
@@ -58,6 +61,8 @@ jobs:
         py_ver: ["3.9"]
         mysql_ver: ["8.0", "5.7"]
         include:
+          - py_ver: "3.12"
+            mysql_ver: "8.0"
           - py_ver: "3.11"
             mysql_ver: "8.0"
           - py_ver: "3.10"
@@ -70,12 +75,6 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{matrix.py_ver}}
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 black
-      - name: Run syntax tests
-        run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
       - name: Run primary tests
         env:
           PY_VER: ${{matrix.py_ver}}
@@ -87,7 +86,7 @@ jobs:
           COMPOSE_HTTP_TIMEOUT: "120"
         run: |
           export HOST_UID=$(id -u)
-          docker compose -f LNX-docker-compose.yml up --build --exit-code-from app
+          docker compose --profile test up --quiet-pull --build --exit-code-from djtest djtest
   lint:
     runs-on: ubuntu-latest
     strategy:
@@ -101,8 +100,8 @@ jobs:
           python-version: ${{matrix.py_ver}}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install flake8 black==24.2.0
+          python3 -m pip install --upgrade pip
+          python3 -m pip install ".[test]"
       - name: Run syntax tests
         run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
       - name: Run style tests
@@ -138,7 +137,7 @@ jobs:
           export PACKAGE=datajoint
           export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git
           export HOST_UID=$(id -u)
-          docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build
+          docker compose -f docs/docker-compose.yaml up --quiet-pull --exit-code-from docs --build
           git push origin gh-pages
   publish-release:
     if: |
@@ -219,7 +218,7 @@ jobs:
       - name: Publish pip release
         run: |
           export HOST_UID=$(id -u)
-          docker compose -f docker-compose-build.yaml run \
+          docker compose run --build --quiet-pull \
             -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \
             sh -c "pip install twine && python -m twine upload dist/*"
       - name: Login to DockerHub

CHANGELOG.md

@@ -1,13 +1,14 @@
 ## Release notes
 
-### 0.14.3 -- TBD
+### 0.14.3 -- Sep 20, 2024
 - Added - `dj.Top` restriction ([#1024](https://github.com/datajoint/datajoint-python/issues/1024)) PR [#1084](https://github.com/datajoint/datajoint-python/pull/1084)
 - Fixed - Added encapsulating double quotes to comply with [DOT language](https://graphviz.org/doc/info/lang.html) - PR [#1177](https://github.com/datajoint/datajoint-python/pull/1177)
 - Added - Datajoint python CLI ([#940](https://github.com/datajoint/datajoint-python/issues/940)) PR [#1095](https://github.com/datajoint/datajoint-python/pull/1095)
 - Added - Ability to set hidden attributes on a table - PR [#1091](https://github.com/datajoint/datajoint-python/pull/1091)
 - Added - Ability to specify a list of keys to populate - PR [#989](https://github.com/datajoint/datajoint-python/pull/989)
 - Fixed - fixed topological sort [#1057](https://github.com/datajoint/datajoint-python/issues/1057)- PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184)
 - Fixed - .parts() not always returning parts [#1103](https://github.com/datajoint/datajoint-python/issues/1103)- PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184)
+- Changed - replace `setup.py` with `pyproject.toml` PR [#1183](https://github.com/datajoint/datajoint-python/pull/1183)
 
 ### 0.14.2 -- Aug 19, 2024
 - Added - Migrate nosetests to pytest - PR [#1142](https://github.com/datajoint/datajoint-python/pull/1142)

Dockerfile

@@ -1,9 +1,24 @@
-ARG IMAGE=djbase
+ARG IMAGE=mambaorg/micromamba:1.5-bookworm-slim
+FROM ${IMAGE}
+
+ARG CONDA_BIN=micromamba
 ARG PY_VER=3.9
-ARG DISTRO=debian
-FROM datajoint/${IMAGE}:py${PY_VER}-${DISTRO}
-COPY --chown=anaconda:anaconda ./setup.py ./datajoint.pub ./requirements.txt /main/
-COPY --chown=anaconda:anaconda ./datajoint /main/datajoint
+ARG HOST_UID=1000
+
+RUN ${CONDA_BIN} install --no-pin -qq -y -n base -c conda-forge \
+        python=${PY_VER} pip setuptools git graphviz pydot && \
+    ${CONDA_BIN} clean -qq -afy
+ENV PATH="$PATH:/home/mambauser/.local/bin"
+
+COPY --chown=${HOST_UID:-1000}:mambauser ./pyproject.toml ./README.md ./LICENSE.txt /main/
+COPY --chown=${HOST_UID:-1000}:mambauser ./datajoint /main/datajoint
+
+VOLUME /src
+WORKDIR /src
+USER root
 RUN \
-    pip install --no-cache-dir /main && \
+    chown -R ${HOST_UID:-1000}:mambauser /main && \
+    chown -R ${HOST_UID:-1000}:mambauser /src && \
+    ${CONDA_BIN} run -n base pip install -q --no-cache-dir /main && \
     rm -r /main/*
+USER ${MAMBA_USER}

LNX-docker-compose.yml

@@ -136,7 +136,7 @@ def __call__(
         format=None,
         as_dict=None,
         squeeze=False,
-        download_path="."
+        download_path=".",
     ):
         """
         Fetches the expression results from the database into an np.array or list of dictionaries and

MANIFEST.in

@@ -27,7 +27,7 @@ def __init__(
         *,
         secure=False,
         proxy_server=None,
-        **_
+        **_,
     ):
         # from https://docs.min.io/docs/python-client-api-reference
         self.client = minio.Minio(

datajoint/fetch.py

@@ -0,0 +1,79 @@
+# HOST_UID=$(id -u) PY_VER=3.11 DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) docker compose --profile test up --build --exit-code-from djtest djtest
+services:
+  db:
+    image: datajoint/mysql:${MYSQL_VER:-8.0}
+    environment:
+      - MYSQL_ROOT_PASSWORD=${DJ_PASS:-password}
+    command: mysqld --default-authentication-plugin=mysql_native_password
+    # ports:
+    #   - "3306:3306"
+    # volumes:
+    #   - ./mysql/data:/var/lib/mysql
+    healthcheck:
+      test: [ "CMD", "mysqladmin", "ping", "-h", "localhost" ]
+      timeout: 30s
+      retries: 5
+      interval: 15s
+  minio:
+    image: minio/minio:${MINIO_VER:-RELEASE.2022-08-11T04-37-28Z}
+    environment:
+      - MINIO_ACCESS_KEY=datajoint
+      - MINIO_SECRET_KEY=datajoint
+    # ports:
+    #   - "9000:9000"
+    # volumes:
+    #   - ./minio/config:/root/.minio
+    #   - ./minio/data:/data
+    command: server --address ":9000" /data
+    healthcheck:
+      test:
+        - "CMD"
+        - "curl"
+        - "--fail"
+        - "http://minio:9000/minio/health/live"
+      timeout: 30s
+      retries: 5
+      interval: 15s
+  app:
+    image: datajoint/datajoint:${DJ_VERSION:-latest}
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        PY_VER: ${PY_VER:-3.8}
+        HOST_UID: ${HOST_UID:-1000}
+    depends_on:
+      db:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+    environment:
+      - DJ_HOST=db
+      - DJ_USER=root
+      - DJ_PASS=password
+      - DJ_TEST_HOST=db
+      - DJ_TEST_USER=datajoint
+      - DJ_TEST_PASSWORD=datajoint
+      - S3_ENDPOINT=minio:9000
+      - S3_ACCESS_KEY=datajoint
+      - S3_SECRET_KEY=datajoint
+      - S3_BUCKET=datajoint.test
+      - PYTHON_USER=dja
+      - JUPYTER_PASSWORD=datajoint
+    working_dir: /src
+    user: ${HOST_UID:-1000}:mambauser
+    volumes:
+      - .:/src
+  djtest:
+    extends:
+      service: app
+    profiles: ["test"]
+    command:
+      - sh
+      - -c
+      - |
+        set -e
+        pip install -q -e ".[test]"
+        pip freeze | grep datajoint
+        pytest --cov-report term-missing --cov=datajoint tests
+