Dev2

Makefile

@@ -1,6 +1,6 @@
 ARCH := ${shell uname -m}
-
-LOCAL_IP=${hostname -I | awk '{print $$1}' | xargs}
+VERSION := v0.1.1
+NODE_NAME=${shell hostname}
 
 all: ctl dash spdk runmodel
 
@@ -14,52 +14,53 @@ dash:
 	-rm ./target/dashboard/* -rf
 	cp ./dashboard/* ./target/dashboard -rL
 	cp ./deployment/dashboard.Dockerfile ./target/dashboard/Dockerfile
-	-sudo docker image rm inferx/inferx_dashboard:v0.1.0
-	sudo docker build -t inferx/inferx_dashboard:v0.1.0 ./target/dashboard
-	# sudo docker push inferx/inferx_dashboard:v0.1.0
+	-sudo docker image rm inferx/inferx_dashboard:$(VERSION)
+	sudo docker build -t inferx/inferx_dashboard:$(VERSION) ./target/dashboard
 
 pushdash:
 	# sudo docker login -u inferx
-	sudo docker tag inferx/inferx_dashboard:v0.1.0 inferx/inferx_dashboard:v0.1.0
-	sudo docker push inferx/inferx_dashboard:v0.1.0
+	sudo docker tag inferx/inferx_dashboard:$(VERSION) inferx/inferx_dashboard:$(VERSION)
+	sudo docker push inferx/inferx_dashboard:$(VERSION)
 
 runmodel:
 	mkdir -p ./target/runmodel
 	cp ./script/run_model.py ./target/runmodel
 	cp ./script/run_llava.py ./target/runmodel
 	cp ./script/run_stablediffusion.py ./target/runmodel
 	cp ./deployment/vllm-opai.Dockerfile ./target/runmodel/Dockerfile
-	-sudo docker image rm vllm-openai-upgraded:v0.1.0
-	sudo docker build -t vllm-openai-upgraded:v0.1.0 ./target/runmodel
+	-sudo docker image rm vllm-openai-upgraded:$(VERSION)
+	sudo docker build -t vllm-openai-upgraded:$(VERSION) ./target/runmodel
 
 spdk:
 	mkdir -p ./target/spdk
 	-rm ./target/spdk/* -rf
 	cp ./deployment/spdk.Dockerfile ./target/spdk/Dockerfile
-	-sudo docker image rm inferx/spdk-container:v0.1.0
-	sudo docker build -t inferx/spdk-container:v0.1.0 ./target/spdk
+	-sudo docker image rm inferx/spdk-container:$(VERSION)
+	sudo docker build -t inferx/spdk-container:$(VERSION) ./target/spdk
 
 spdk2:
 	mkdir -p ./target/spdk
 	-rm ./target/spdk/* -rf
 	cp ./deployment/spdk2.Dockerfile ./target/spdk/Dockerfile
 	cp ./deployment/spdk.script ./target/spdk/entrypoint.sh
-	-sudo docker image rm inferx/spdk-container2:v0.1.0
-	sudo docker build -t inferx/spdk-container2:v0.1.0 ./target/spdk
+	-sudo docker image rm inferx/spdk-container2:$(VERSION)
+	sudo docker build -t inferx/spdk-container2:$(VERSION) ./target/spdk
 
 pushspdk:
 	# sudo docker login -u inferx
-	sudo docker tag inferx/spdk-container:v0.1.0 inferx/spdk-container:v0.1.0
-	sudo docker push inferx/spdk-container:v0.1.0
-	sudo docker tag inferx/spdk-container2:v0.1.0 inferx/spdk-container2:v0.1.0
-	sudo docker push inferx/spdk-container2:v0.1.0
+	sudo docker tag inferx/spdk-container:$(VERSION) inferx/spdk-container:$(VERSION)
+	sudo docker push inferx/spdk-container:$(VERSION)
+	sudo docker tag inferx/spdk-container2:$(VERSION) inferx/spdk-container2:$(VERSION)
+	sudo docker push inferx/spdk-container2:$(VERSION)
 sql:
 	sudo cp ./dashboard/sql/create_table.sql /opt/inferx/config
 	sudo cp ./dashboard/sql/secret.sql /opt/inferx/config
 
 run:
 	-sudo pkill -9 inferx
 	@echo "LOCAL_IP=$$(hostname -I | awk '{print $$1}' | xargs)" > .env
+	@echo "Version=$(VERSION)" >> .env
+	@echo "HOSTNAME=$(NODE_NAME)" >> .env
 	sudo docker compose -f docker-compose.yml  build
 	- sudo rm -f /opt/inferx/log/inferx.log
 	- sudo rm -f /opt/inferx/log/onenode.log
@@ -68,11 +69,14 @@ run:
 
 runblob:
 	-sudo pkill -9 inferx
-	@echo "LOCAL_IP=$$(hostname -I | awk '{print $$1}' | xargs)" > .env
+	@echo "LOCAL_IP=$$(hostname -I | tr ' ' '\n' | grep -v '^172\.' | head -n 1 | xargs)" > .env
+	@echo "Version=$(VERSION)" >> .env
+	@echo "HOSTNAME=$(NODE_NAME)" >> .env
 	sudo docker compose -f docker-compose_blob.yml  build
 	- sudo rm -f /opt/inferx/log/inferx.log
 	- sudo rm -f /opt/inferx/log/onenode.log
 	sudo docker compose -f docker-compose_blob.yml up -d --remove-orphans
+	cat .env
 	rm .env
 
 stop:
@@ -82,7 +86,25 @@ stopblob:
 	sudo docker compose -f docker-compose_blob.yml down
 
 rundash:
-	sudo docker run --net=host --name inferx_dashboard -v /etc/letsencrypt/:/etc/letsencrypt/ --rm  inferx/inferx_dashboard:v0.1.0
+	sudo docker run --net=host --name inferx_dashboard --env "KEYCLOAK_URL=http://192.168.0.22:1260/authn" \
+	-v /etc/letsencrypt/:/etc/letsencrypt/ --rm  inferx/inferx_dashboard:$(VERSION)
 
 stopdash:
-	sudo docker stop inferx_dashboard
+	sudo docker stop inferx_dashboard
+
+runkblob:
+	sudo kubectl apply -f k8s/spdk.yaml
+	sudo kubectl apply -f k8s/etcd.yaml
+	sudo kubectl apply -f k8s/secretdb.yaml
+	sudo kubectl apply -f k8s/db-deployment.yaml
+	sudo kubectl apply -f k8s/keycloak_postgres.yaml
+	sudo kubectl apply -f k8s/keycloak.yaml
+	sudo kubectl apply -f k8s/statesvc.yaml
+	sudo kubectl apply -f k8s/scheduler.yaml
+	sudo kubectl apply -f k8s/nodeagent.yaml
+	sudo kubectl apply -f k8s/dashboard.yaml
+	sudo kubectl apply -f k8s/ingress.yaml
+
+stopnodeagent:
+	sudo kubectl delete DaemonSet nodeagent-blob
+	sudo kubectl delete DaemonSet nodeagent-file

config/Aquila-7B.json

@@ -5,7 +5,7 @@
     "name": "Aquila-7B",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "BAAI/Aquila-7B",
@@ -17,7 +17,7 @@
             ],
             "resources": {
                 "CPU": 20000,
-                "Mem": 50000,
+                "Mem": 60000,
                 "GPU": {
                     "Type": "Any",
                     "Count": 2,
@@ -28,6 +28,10 @@
                 [
                     "LD_LIBRARY_PATH",
                     "/usr/local/lib/python3.12/dist-packages/nvidia/cuda_nvrtc/lib/:$LD_LIBRARY_PATH"
+                ],
+                [
+                    "VLLM_CUDART_SO_PATH",
+                    "/usr/local/cuda-12.1/targets/x86_64-linux/lib/libcudart.so.12"
                 ]
             ],
             "mounts": [

config/Baichuan-7B.json

@@ -5,7 +5,7 @@
     "name": "Baichuan-7B",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "baichuan-inc/Baichuan-7B",
@@ -28,6 +28,10 @@
                 [
                     "LD_LIBRARY_PATH",
                     "/usr/local/lib/python3.12/dist-packages/nvidia/cuda_nvrtc/lib/:$LD_LIBRARY_PATH"
+                ],
+                [
+                    "VLLM_CUDART_SO_PATH",
+                    "/usr/local/cuda-12.1/targets/x86_64-linux/lib/libcudart.so.12"
                 ]
             ],
             "mounts": [

config/Baichuan2-13B-Chat-4bits.json

@@ -5,7 +5,7 @@
     "name": "Baichuan2-13B-Chat-4bits",
     "object": {
         "spec": {
-            "image": "vllm-openai-upgraded:v.0.1",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "baichuan-inc/Baichuan2-13B-Chat-4bits",
@@ -16,17 +16,21 @@
             ],
             "resources": {
                 "CPU": 12000,
-                "Mem": 14000,
+                "Mem": 24000,
                 "GPU": {
                     "Type": "Any",
                     "Count": 1,
-                    "vRam": 8000
+                    "vRam": 13800
                 }
             },
             "envs": [
                 [
                     "LD_LIBRARY_PATH",
                     "/usr/local/lib/python3.12/dist-packages/nvidia/cuda_nvrtc/lib/:$LD_LIBRARY_PATH"
+                ],
+                [
+                    "VLLM_CUDART_SO_PATH",
+                    "/usr/local/cuda-12.1/targets/x86_64-linux/lib/libcudart.so.12"
                 ]
             ],
             "mounts": [

config/Baichuan2-7B-Chat.json

@@ -5,7 +5,7 @@
     "name": "Baichuan2-7B-Chat",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "baichuan-inc/Baichuan2-7B-Chat",
@@ -28,6 +28,10 @@
                 [
                     "LD_LIBRARY_PATH",
                     "/usr/local/lib/python3.12/dist-packages/nvidia/cuda_nvrtc/lib/:$LD_LIBRARY_PATH"
+                ],
+                [
+                    "VLLM_CUDART_SO_PATH",
+                    "/usr/local/cuda-12.1/targets/x86_64-linux/lib/libcudart.so.12"
                 ]
             ],
             "mounts": [

config/DeciLM-7B-instruct.json

@@ -5,7 +5,7 @@
     "name": "DeciLM-7B-instruct",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "Deci/DeciLM-7B-instruct",

config/DeciLM-7B.json

@@ -5,7 +5,7 @@
     "name": "DeciLM-7B",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "Deci/DeciLM-7B",

config/DeepSeek-R1-Distill-Llama-8B.json

@@ -5,7 +5,7 @@
     "name": "DeepSeek-R1-Distill-Llama-8B",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "/root/.cache/huggingface/git/DeepSeek-R1-Distill-Llama-8B",
@@ -30,6 +30,10 @@
                 [
                     "LD_LIBRARY_PATH",
                     "/usr/local/lib/python3.12/dist-packages/nvidia/cuda_nvrtc/lib/:$LD_LIBRARY_PATH"
+                ],
+                [
+                    "VLLM_CUDART_SO_PATH",
+                    "/usr/local/cuda-12.1/targets/x86_64-linux/lib/libcudart.so.12"
                 ]
             ],
             "mounts": [

config/DeepSeek-R1-Distill-Qwen-1.5B.json

@@ -5,7 +5,7 @@
     "name": "DeepSeek-R1-Distill-Qwen-1.5B",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "/root/.cache/huggingface/git/DeepSeek-R1-Distill-Qwen-1.5B",
@@ -18,7 +18,7 @@
             ],
             "resources": {
                 "CPU": 20000,
-                "Mem": 50000,
+                "Mem": 60000,
                 "GPU": {
                     "Type": "Any",
                     "Count": 1,
@@ -29,6 +29,10 @@
                 [
                     "LD_LIBRARY_PATH",
                     "/usr/local/lib/python3.12/dist-packages/nvidia/cuda_nvrtc/lib/:$LD_LIBRARY_PATH"
+                ],
+                [
+                    "VLLM_CUDART_SO_PATH",
+                    "/usr/local/cuda-12.1/targets/x86_64-linux/lib/libcudart.so.12"
                 ]
             ],
             "mounts": [

config/DeepSeek-R1-Distill-Qwen-7B.json

@@ -5,7 +5,7 @@
     "name": "DeepSeek-R1-Distill-Qwen-7B",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "/root/.cache/huggingface/git/DeepSeek-R1-Distill-Qwen-7B",
@@ -30,6 +30,10 @@
                 [
                     "LD_LIBRARY_PATH",
                     "/usr/local/lib/python3.12/dist-packages/nvidia/cuda_nvrtc/lib/:$LD_LIBRARY_PATH"
+                ],
+                [
+                    "VLLM_CUDART_SO_PATH",
+                    "/usr/local/cuda-12.1/targets/x86_64-linux/lib/libcudart.so.12"
                 ]
             ],
             "mounts": [

config/EXAONE-3.0-7.8B-Instruct copy.json

@@ -4,7 +4,7 @@
     "namespace": "ns1",
     "name": "gemma-7b",
     "spec": {
-        "image": "vllm/vllm-openai:v0.6.2",
+        "image": "vllm/vllm-openai:v0.7.3",
         "commands": [
             "--model",
             "google/gemma-7b",

config/EXAONE-3.0-7.8B-Instruct.json

@@ -4,7 +4,7 @@
     "namespace": "ns1",
     "name": "EXAONE-3.0-7.8B-Instruct",
     "spec": {
-        "image": "vllm/vllm-openai:v0.6.2",
+        "image": "vllm/vllm-openai:v0.7.3",
         "commands": [
             "--model",
             "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct",

config/Llama-2-13b-hf.json

@@ -5,7 +5,7 @@
     "name": "Llama-2-13b-hf",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "meta-llama/Llama-2-13b-hf",
@@ -18,7 +18,7 @@
             ],
             "resources": {
                 "CPU": 20000,
-                "Mem": 50000,
+                "Mem": 60000,
                 "GPU": {
                     "Type": "Any",
                     "Count": 2,
@@ -29,6 +29,10 @@
                 [
                     "LD_LIBRARY_PATH",
                     "/usr/local/lib/python3.12/dist-packages/nvidia/cuda_nvrtc/lib/:$LD_LIBRARY_PATH"
+                ],
+                [
+                    "VLLM_CUDART_SO_PATH",
+                    "/usr/local/cuda-12.1/targets/x86_64-linux/lib/libcudart.so.12"
                 ]
             ],
             "mounts": [

config/Llama-3.2-3B-Instruct.json

@@ -5,7 +5,7 @@
     "name": "Llama-3.2-3B-Instruct",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "meta-llama/Llama-3.2-3B-Instruct",
@@ -27,6 +27,10 @@
                 [
                     "LD_LIBRARY_PATH",
                     "/usr/local/lib/python3.12/dist-packages/nvidia/cuda_nvrtc/lib/:$LD_LIBRARY_PATH"
+                ],
+                [
+                    "VLLM_CUDART_SO_PATH",
+                    "/usr/local/cuda-12.1/targets/x86_64-linux/lib/libcudart.so.12"
                 ]
             ],
             "mounts": [

config/Llama-3.2-3B-Instruct_2gpu.json

@@ -5,7 +5,7 @@
     "name": "Llama-3.2-3B-Instruct_2gpu",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "meta-llama/Llama-3.2-3B-Instruct",

config/Meta-Llama-3-8B-Instruct.json

@@ -5,7 +5,7 @@
     "name": "Meta-Llama-3-8B-Instruct",
     "object": {
         "spec": {
-            "image": "vllm/vllm-openai:v0.6.2",
+            "image": "vllm/vllm-openai:v0.7.3",
             "commands": [
                 "--model",
                 "meta-llama/Meta-Llama-3-8B-Instruct",