change: improvements to cicd

.ci/docker-compose.yaml

@@ -1,6 +1,7 @@
 services:
   hugot:
     image: hugot:$commit_hash
+    container_name: hugot
     build:
       context: ./..
       dockerfile: ./Dockerfile

.github/workflows/release.yaml

@@ -17,10 +17,26 @@ jobs:
         uses: actions/checkout@v4
       - name: Build and run tests
         run: make run-tests
+      - uses: actions/upload-artifact@main
+        with:
+          name: libtokenizers.a
+          path: ./testTarget/libtokenizers.a
+      - uses: actions/upload-artifact@main
+        with:
+          name: onnxruntime.so
+          path: ./testTarget/onnxruntime.so
   release:
     name: Release pushed tag
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/download-artifact@master
+        with:
+          name: libtokenizers
+          path: ./libtokenizers.a
+      - uses: actions/download-artifact@master
+        with:
+          name: onnxruntime.so
+          path: ./onnxruntime.so
       - name: Create release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -29,4 +45,6 @@ jobs:
           gh release create "$tag" \
               --repo="$GITHUB_REPOSITORY" \
               --title="${GITHUB_REPOSITORY#*/} ${tag#v}" \
-              --generate-notes
+              --generate-notes \
+              './onnxruntime.so#onnxruntime.so' \ 
+              './libtokenizers.a#libtokenizers.a'

Dockerfile

@@ -46,7 +46,7 @@ RUN curl -LO https://github.com/gotestyourself/gotestsum/releases/download/v1.11
 COPY . /build
 WORKDIR /build
 RUN go mod download && CGO_ENABLED=1 GOOS=linux GOARCH=amd64 && \
-    mkdir /unittest && go test -c ./pipelines -o /unittest && \
+    mkdir /unittest && go test -c . -o /unittest/pipelines.test && \
     go clean -r -cache -testcache -modcache
 
 # models

hugot_test.go

@@ -31,6 +31,9 @@ func TestTextClassificationPipeline(t *testing.T) {
 		Check(session.Destroy())
 	}(session)
 	modelFolder := os.Getenv("TEST_MODELS_FOLDER")
+	if modelFolder == "" {
+		modelFolder = "./models"
+	}
 	modelPath := path.Join(modelFolder, "distilbert-base-uncased-finetuned-sst-2-english")
 	sentimentPipeline, err := session.NewTextClassificationPipeline(modelPath, "testPipeline")
 	Check(err)
@@ -86,6 +89,9 @@ func TestTokenClassificationPipeline(t *testing.T) {
 	}(session)
 
 	modelFolder := os.Getenv("TEST_MODELS_FOLDER")
+	if modelFolder == "" {
+		modelFolder = "./models"
+	}
 	modelPath := path.Join(modelFolder, "distilbert-NER")
 	pipelineSimple, err2 := session.NewTokenClassificationPipeline(modelPath, "testPipeline", pipelines.WithSimpleAggregation())
 	Check(err2)
@@ -148,6 +154,9 @@ func TestFeatureExtractionPipeline(t *testing.T) {
 	}(session)
 
 	modelFolder := os.Getenv("TEST_MODELS_FOLDER")
+	if modelFolder == "" {
+		modelFolder = "./models"
+	}
 	modelPath := path.Join(modelFolder, "all-MiniLM-L6-v2")
 	pipeline, err := session.NewFeatureExtractionPipeline(modelPath, "testPipeline")
 	Check(err)
@@ -206,10 +215,11 @@ func TestFeatureExtractionPipeline(t *testing.T) {
 		}
 	}
 
-	assert.Greater(t, pipeline.PipelineTimings.NumCalls, 0, "PipelineTimings.NumCalls should be greater than 0")
-	assert.Greater(t, pipeline.PipelineTimings.TotalNS, 0, "PipelineTimings.TotalNS should be greater than 0")
-	assert.Greater(t, pipeline.TokenizerTimings.NumCalls, 0, "TokenizerTimings.NumCalls should be greater than 0")
-	assert.Greater(t, pipeline.TokenizerTimings.TotalNS, 0, "TokenizerTimings.TotalNS should be greater than 0")
+	zero := uint64(0)
+	assert.Greater(t, pipeline.PipelineTimings.NumCalls, zero, "PipelineTimings.NumCalls should be greater than 0")
+	assert.Greater(t, pipeline.PipelineTimings.TotalNS, zero, "PipelineTimings.TotalNS should be greater than 0")
+	assert.Greater(t, pipeline.TokenizerTimings.NumCalls, zero, "TokenizerTimings.NumCalls should be greater than 0")
+	assert.Greater(t, pipeline.TokenizerTimings.TotalNS, zero, "TokenizerTimings.TotalNS should be greater than 0")
 }
 
 // utilities

scripts/run-unit-tests.sh

@@ -22,10 +22,12 @@ fi
 
 # build with compose
 docker compose -f $src_dir/.ci/docker-compose.yaml build
-# (cd $src_dir/.ci && docker compose build)
-
 echo "Running tests for commit hash: $commit_hash"
-
 docker compose -f $src_dir/.ci/docker-compose.yaml up && \
 docker compose -f $src_dir/.ci/docker-compose.yaml logs --no-color >& $test_folder/logs.txt && \
-docker compose -f $src_dir/.ci/docker-compose.yaml rm -fsv
+echo "Extracting lib artifacts"
+id=$(docker ps -aqf "name=hugot")
+echo $id
+docker cp $id:/usr/lib/libtokenizers.a ./testTarget/libtokenizers.a
+docker cp $id:/usr/lib64/onnxruntime.so ./testTarget/onnxruntime.so
+docker compose -f $src_dir/.ci/docker-compose.yaml rm -fsv