Merge pull request #2318 from valory-xyz/fix/build-image

Fix: Support building `linux/amd64,linux/arm64,linux/arm/v7` agent images

Author: OjusWiZard

.github/workflows/release.yml

@@ -157,8 +157,8 @@ jobs:
         run: |
           # export `TAG` variable
           source env.sh
-          docker buildx build --platform linux/amd64,linux/arm64 -t valory/open-autonomy-user:$TAG deployments/Dockerfiles/autonomy-user --push
-          docker buildx build --platform linux/amd64,linux/arm64 -t valory/open-autonomy-user:latest deployments/Dockerfiles/autonomy-user --push
+          docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t valory/open-autonomy-user:$TAG deployments/Dockerfiles/autonomy-user --push
+          docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t valory/open-autonomy-user:latest deployments/Dockerfiles/autonomy-user --push
 
   publish-helper-images:
     name: Publish Helper Images

autonomy/cli/build_images.py

@@ -72,7 +72,18 @@
     type=str,
     help="Specify the target architecture platform for the image.",
 )
+@click.option(
+    "--builder",
+    type=str,
+    help='Override the configured docker builder instance (default "default").',
+)
 @click.option("--push", is_flag=True, help="Push image to docker hub.", default=False)
+@click.option(
+    "--pre-install-command",
+    type=str,
+    help="Run the command before installing dependencies.",
+    default=None,
+)
 @image_author_option
 def build_image(  # pylint: disable=too-many-arguments
     agent: Optional[PublicId],
@@ -85,6 +96,8 @@ def build_image(  # pylint: disable=too-many-arguments
     image_author: Optional[str] = None,
     platform: Optional[str] = None,
     push: bool = False,
+    builder: Optional[str] = None,
+    pre_install_command: Optional[str] = None,
 ) -> None:
     """Build runtime images for autonomous agents."""
     if dev:
@@ -103,4 +116,6 @@ def build_image(  # pylint: disable=too-many-arguments
             dockerfile=dockerfile,
             platform=platform,
             push=push,
+            builder=builder,
+            pre_install_command=pre_install_command,
         )

autonomy/cli/helpers/image.py

@@ -39,6 +39,8 @@ def build_image(  # pylint: disable=too-many-arguments
     dockerfile: Optional[Path] = None,
     platform: Optional[str] = None,
     push: bool = False,
+    builder: Optional[str] = None,
+    pre_install_command: Optional[str] = None,
 ) -> None:
     """Build agent/service image."""
     extra_dependencies = extra_dependencies or ()
@@ -58,4 +60,6 @@ def build_image(  # pylint: disable=too-many-arguments
         dockerfile=dockerfile,
         platform=platform,
         push=push,
+        builder=builder,
+        pre_install_command=pre_install_command,
     )

autonomy/data/Dockerfiles/agent/Dockerfile

@@ -6,6 +6,9 @@ ARG AEA_AGENT
 ARG AUTHOR
 ARG EXTRA_DEPENDENCIES
 
+ARG PRE_INSTALL_COMMAND=""
+RUN /bin/sh -c "${PRE_INSTALL_COMMAND}"
+
 RUN aea init --reset --remote --ipfs --author ${AUTHOR}
 
 WORKDIR /root

autonomy/deploy/image.py

@@ -65,6 +65,8 @@ def build_image(  # pylint: disable=too-many-arguments,too-many-locals
     dockerfile: Optional[Path] = None,
     platform: Optional[str] = None,
     push: bool = False,
+    builder: Optional[str] = None,
+    pre_install_command: Optional[str] = None,
 ) -> None:
     """Command to build images from for skaffold deployment."""
 
@@ -94,20 +96,28 @@ def build_image(  # pylint: disable=too-many-arguments,too-many-locals
             f"AUTHOR={get_default_author_from_cli_config()}",
             "--build-arg",
             f"AEA_AGENT={str(agent)}",
-            "--build-arg",
-            f"EXTRA_DEPENDENCIES={generate_dependency_flag_var(extra_dependencies or ())}",
             "--tag",
             tag,
             "--no-cache",
         ]
+        + (
+            [
+                "--build-arg",
+                f"EXTRA_DEPENDENCIES={generate_dependency_flag_var(extra_dependencies)}",
+            ]
+            if extra_dependencies
+            else []
+        )
+        + (
+            ["--build-arg", f"PRE_INSTALL_COMMAND={pre_install_command}"]
+            if pre_install_command
+            else []
+        )
         + (["--platform", platform] if platform else [])
-        + []
         + (["--push"] if push else [])
-        + []
         + (["--pull"] if pull else [])
-        + [
-            path,
-        ]
+        + (["--builder", builder] if builder else [])
+        + [path]
     )
 
     build_process.check_returncode()

deployments/Dockerfiles/autonomy-user/Dockerfile

@@ -1,19 +1,25 @@
 # Base image
-FROM python:3.10-slim-bullseye as base
+FROM python:3.10-slim-bullseye AS base
 ENV VIRTUAL_ENV=/opt/venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-RUN apt-get update -y && apt-get install gcc -y && rm -rf /var/lib/apt/lists/*
+RUN apt-get update -y && apt-get install gcc git make libssl-dev libffi-dev -y && rm -rf /var/lib/apt/lists/*
 
 # Install library dependencies into our venv
 RUN python3 -m venv $VIRTUAL_ENV
 COPY requirements.txt .
+RUN pip install --upgrade pip
 RUN pip install -r requirements.txt
 
 # Customize base image with agent deps
-FROM python:3.10-slim-bullseye as agent_deps
+FROM python:3.10-slim-bullseye AS agent_deps
 ARG AUTHOR=user
 
+# Install docker
+VOLUME /var/run/docker.sock
+RUN apt-get update -y && apt-get install curl -y
+RUN curl -sSL https://get.docker.com/ | sh
+
 ENV VIRTUAL_ENV=/opt/venv
 COPY --from=base $VIRTUAL_ENV $VIRTUAL_ENV
 

deployments/Dockerfiles/autonomy/Dockerfile

@@ -2,9 +2,12 @@ ARG AEA_VERSION=latest
 
 FROM valory/open-aea-user:${AEA_VERSION}
 
+VOLUME /var/run/docker.sock
 RUN apt update
 
-RUN apt install git net-tools sudo -y
+RUN apt install git net-tools sudo curl -y
+
+RUN curl -sSL https://get.docker.com/ | sh
 
 COPY scripts /root/scripts
 

deployments/Dockerfiles/autonomy/scripts/install.sh

@@ -13,6 +13,7 @@ aea fetch $AEA_AGENT --alias agent || exit 1
 cd agent
 
 echo "Building the deployments host dependencies."
+pip install --upgrade pip
 aea build || exit 1
 echo "Successfully built the host dependencies."
 

docs/api/cli/build_images.md

@@ -52,10 +52,21 @@ Build images.
     type=str,
     help="Specify the target architecture platform for the image.",
 )
+@click.option(
+    "--builder",
+    type=str,
+    help='Override the configured docker builder instance (default "default").',
+)
 @click.option("--push",
               is_flag=True,
               help="Push image to docker hub.",
               default=False)
+@click.option(
+    "--pre-install-command",
+    type=str,
+    help="Run the command before installing dependencies.",
+    default=None,
+)
 @image_author_option
 def build_image(agent: Optional[PublicId],
                 service_dir: Optional[Path],
@@ -66,7 +77,9 @@ def build_image(agent: Optional[PublicId],
                 version: Optional[str] = None,
                 image_author: Optional[str] = None,
                 platform: Optional[str] = None,
-                push: bool = False) -> None
+                push: bool = False,
+                builder: Optional[str] = None,
+                pre_install_command: Optional[str] = None) -> None
 ```
 
 Build runtime images for autonomous agents.

docs/api/cli/helpers/image.md

@@ -17,7 +17,9 @@ def build_image(agent: Optional[PublicId],
                 extra_dependencies: Optional[Tuple[Dependency, ...]] = None,
                 dockerfile: Optional[Path] = None,
                 platform: Optional[str] = None,
-                push: bool = False) -> None
+                push: bool = False,
+                builder: Optional[str] = None,
+                pre_install_command: Optional[str] = None) -> None
 ```
 
 Build agent/service image.

docs/api/deploy/image.md

@@ -36,7 +36,9 @@ def build_image(agent: PublicId,
                 extra_dependencies: Optional[Tuple[Dependency, ...]] = None,
                 dockerfile: Optional[Path] = None,
                 platform: Optional[str] = None,
-                push: bool = False) -> None
+                push: bool = False,
+                builder: Optional[str] = None,
+                pre_install_command: Optional[str] = None) -> None
 ```
 
 Command to build images from for skaffold deployment.

tox.ini

@@ -954,3 +954,5 @@ websocket-client: >=0.59.0
 sniffio: >=1.3.0
 ; licence is MIT, but the tool does not detect it
 attrs: ==25.3.0
+; licence is GPL-compatible, but the tool does not detect it
+typing-extensions: ==4.13.0