Pluggable backends and tokenizers, with XLA/goMLX backend

.dockerignore

@@ -1 +1,2 @@
-**/.git
+.git
+/models

.github/workflows/release.yaml

@@ -48,10 +48,6 @@ jobs:
         with:
           name: onnxruntime-linux-x64.so
           path: ./artifacts/onnxruntime-linux-x64.so
-      - uses: actions/upload-artifact@v4
-        with:
-          name: onnxruntime-linux-x64-gpu
-          path: ./artifacts/onnxruntime-linux-x64-gpu
       - uses: actions/upload-artifact@v4
         with:
           name: hugot-cli-linux-x64
@@ -120,16 +116,10 @@ jobs:
         with:
           name: hugot-cli-linux-x64
           path: .
-      - uses: actions/download-artifact@v4
-        with:
-          name: onnxruntime-linux-x64-gpu
-          path: ./onnxruntime-linux-x64-gpu
       - name: Display structure of downloaded files
         run: ls -R
-      - name: zip onnxruntime gpu
-        run: zip onnxruntime-linux-x64-gpu.zip ./onnxruntime-linux-x64-gpu -r
       - uses: ncipollo/release-action@v1
         with:
-          artifacts: "libtokenizers.a, onnxruntime-linux-x64.so, onnxruntime-linux-x64-gpu.zip, hugot-cli-linux-x64"
+          artifacts: "libtokenizers.a, onnxruntime-linux-x64.so, hugot-cli-linux-x64"
           generateReleaseNotes: true
           skipIfReleaseExists: true

.github/workflows/test.yaml

@@ -38,10 +38,6 @@ jobs:
         with:
           name: onnxruntime-linux-x64.so
           path: ./artifacts/onnxruntime-linux-x64.so
-      - uses: actions/upload-artifact@v4
-        with:
-          name: onnxruntime-linux-x64-gpu
-          path: ./artifacts/onnxruntime-linux-x64-gpu
       - uses: actions/upload-artifact@v4
         with:
           name: hugot-cli-linux-x64
@@ -52,4 +48,4 @@ jobs:
         with:
           report_paths: './testTarget/unit/*.xml'
           fail_on_failure: true
-          require_tests: true
+          require_tests: true

Dockerfile

@@ -4,45 +4,40 @@ ARG GO_VERSION=1.23.3
 ARG ONNXRUNTIME_VERSION=1.20.0
 ARG BUILD_PLATFORM=linux/amd64
 
-#--- runtime layer with all hugot dependencies for cpu and gpu ---
+#--- runtime layer with all hugot dependencies for cpu ---
 
 FROM --platform=$BUILD_PLATFORM public.ecr.aws/amazonlinux/amazonlinux:2023 AS hugot-runtime
 ARG GO_VERSION
 ARG ONNXRUNTIME_VERSION
 
-ENV PATH="$PATH:/usr/local/go/bin"
+ENV PATH="$PATH:/usr/local/go/bin" \
+    GOPJRT_NOSUDO=1
 
+COPY ./scripts/download-onnxruntime.sh /download-onnxruntime.sh
 RUN --mount=src=./go.mod,dst=/go.mod \
-    --mount=src=./scripts/download-onnxruntime.sh,dst=/download-onnxruntime.sh \
-    dnf -y install gcc jq bash tar xz gzip glibc-static libstdc++ wget zip git && \
+    dnf --allowerasing -y install gcc jq bash tar xz gzip glibc-static libstdc++ wget zip git dirmngr sudo which && \
     ln -s /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so && \
-    dnf install -y 'dnf-command(config-manager)' && \
-    dnf config-manager --add-repo https://download.fedoraproject.org/pub/fedora/linux/releases/39/Everything/x86_64/os/ && \
-    # from fedora
-    dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/fedora39/x86_64/cuda-fedora39.repo && \
-    dnf install -y cuda-cudart-12-6 libcublas-12-6 libcurand-12-6 libcufft-12-6 && \
-    # from rhel
-    dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo && \
-    dnf install -y libcudnn9-cuda-12 && \
     dnf clean all && \
-    # NON-PRIVILEDGED USER
-    # create non-priviledged testuser with id: 1000
-    dnf install --disablerepo=* --enablerepo=amazonlinux --allowerasing -y dirmngr sudo which && dnf clean all && \
-    useradd -u 1000 -m testuser && usermod -a -G wheel testuser && \
-    echo "testuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/testuser && \
     # tokenizers
     tokenizer_version=$(grep 'github.com/daulet/tokenizers' /go.mod | awk '{print $2}') && \
     tokenizer_version=$(echo $tokenizer_version | awk -F'-' '{print $NF}') && \
     echo "tokenizer_version: $tokenizer_version" && \
     curl -LO https://github.com/daulet/tokenizers/releases/download/${tokenizer_version}/libtokenizers.linux-amd64.tar.gz && \
     tar -C /usr/lib -xzf libtokenizers.linux-amd64.tar.gz && \
     rm libtokenizers.linux-amd64.tar.gz && \
+    # onnxruntime cpu
+    sed -i 's/\r//g' /download-onnxruntime.sh && chmod +x /download-onnxruntime.sh && \
+    /download-onnxruntime.sh ${ONNXRUNTIME_VERSION} && \
+    # XLA/goMLX
+    curl -sSf https://raw.githubusercontent.com/gomlx/gopjrt/main/cmd/install_linux_amd64_amazonlinux.sh | bash && \
     # go
     curl -LO https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz && \
     tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz && \
     rm go${GO_VERSION}.linux-amd64.tar.gz && \
-    # onnxruntime cpu and gpu
-    /download-onnxruntime.sh ${ONNXRUNTIME_VERSION}
+    # NON-PRIVILEGED USER
+    # create non-privileged testuser with id: 1000
+    useradd -u 1000 -m testuser && usermod -a -G wheel testuser && \
+    echo "testuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/testuser
 
 #--- test layer ---
 
@@ -53,7 +48,7 @@ COPY . /build
 RUN cd /build && \
     chown -R testuser:testuser /build && \
     # cli binary
-    cd /build/cmd && CGO_ENABLED=1 CGO_LDFLAGS="-L/usr/lib/" GOOS=linux GOARCH=amd64 go build -a -o /cli main.go && \
+    cd /build/cmd && CGO_ENABLED=1 CGO_LDFLAGS="-L/usr/lib/" GOOS=linux GOARCH=amd64 go build -a -tags GO,ORT -o /cli main.go && \
     cd / && \
     curl -LO https://github.com/gotestyourself/gotestsum/releases/download/v1.12.0/gotestsum_1.12.0_linux_amd64.tar.gz && \
     tar -xzf gotestsum_1.12.0_linux_amd64.tar.gz --directory /usr/local/bin && \
@@ -66,6 +61,5 @@ ENTRYPOINT ["/entrypoint.sh"]
 FROM --platform=$BUILD_PLATFORM scratch AS artifacts
 
 COPY --from=hugot-build /usr/lib64/onnxruntime.so onnxruntime-linux-x64.so
-COPY --from=hugot-build /usr/lib64/onnxruntime-gpu onnxruntime-linux-x64-gpu
 COPY --from=hugot-build /usr/lib/libtokenizers.a libtokenizers.a
 COPY --from=hugot-build /cli /hugot-cli-linux-x64