- feature: add uv (#78)

* - feature: added uv to replace direct python usage

* - improve: modularized ensure python method

* - ci: bump npm version

* - fix: uv location

* - fix: missing cargo dep

* - fix: add cargo lock

* - fix: uv location macos

* - improve: use uv wrapper for pip

docker-images/Dockerfile

@@ -7,10 +7,12 @@ SHELL ["/bin/bash", "-c"]
 RUN apt-get update && apt-get install -y chromium
 ENV CHROME_PATH=/usr/bin/chromium
 
-# Add python 3
-RUN apt-get install -y python3 python3-pip python3-venv
+RUN apt-get install -y curl
 
 WORKDIR /app
-RUN python3 -m venv cache/python-venv && source cache/python-venv/bin/activate && pip install pipreqs
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+ENV PATH="/root/.local/bin:${PATH}"
+
+RUN source $HOME/.local/bin/env && uv venv --seed cache/python-venv --python 3.13 && source cache/python-venv/bin/activate && uv pip install pipreqs
 
 ENTRYPOINT ["/tini", "--"]

libs/shinkai-tools-runner/Cargo.toml

@@ -32,6 +32,7 @@ once_cell = { version = "1.20.2" }
 env_logger = "0.11.5"
 anyhow = { version = "1.0.93" }
 chrono = { version = "0.4.38" }
+whoami = "1.5"
 
 [dev-dependencies]
 rstest = "0.23.0"

libs/shinkai-tools-runner/src/tools/deno_runner_options.rs

@@ -9,7 +9,6 @@ use super::{
 pub struct DenoRunnerOptions {
     pub context: ExecutionContext,
     pub deno_binary_path: PathBuf,
-    pub python_binary_path: PathBuf,
     pub code_runner_docker_image_name: String,
     pub force_runner_type: Option<RunnerType>,
     pub shinkai_node_location: ShinkaiNodeLocation,
@@ -19,17 +18,12 @@ impl Default for DenoRunnerOptions {
     fn default() -> Self {
         Self {
             context: ExecutionContext::default(),
-            code_runner_docker_image_name: String::from("dcspark/shinkai-code-runner:0.9.0"),
+            code_runner_docker_image_name: String::from("dcspark/shinkai-code-runner:0.9.1"),
             deno_binary_path: PathBuf::from(if cfg!(windows) {
                 "./shinkai-tools-runner-resources/deno.exe"
             } else {
                 "./shinkai-tools-runner-resources/deno"
             }),
-            python_binary_path: PathBuf::from(if cfg!(windows) {
-                "C://Users/agall/AppData/Local/Microsoft/WindowsApps/python"
-            } else {
-                "python"
-            }),
             force_runner_type: None,
             shinkai_node_location: ShinkaiNodeLocation {
                 protocol: String::from("http"),

libs/shinkai-tools-runner/src/tools/python_runner.rs

@@ -1,7 +1,7 @@
 use serde_json::Value;
 use std::{
     collections::HashMap,
-    path::{self, Path},
+    path::{self, Path, PathBuf},
     sync::Arc,
     time::Duration,
 };
@@ -41,16 +41,67 @@ impl PythonRunner {
         }
     }
 
+    async fn ensure_python_venv(&self, venv_path: PathBuf) -> anyhow::Result<String> {
+        let uv_binary_path_at_host = path::absolute(self.options.uv_binary_path.clone())
+            .unwrap()
+            .to_string_lossy()
+            .to_string();
+
+        log::info!("Creating new venv with uv {:?}", venv_path);
+        let mut venv_command = tokio::process::Command::new(&uv_binary_path_at_host);
+        venv_command
+            .args([
+                "venv",
+                "--seed",
+                venv_path.to_str().unwrap(),
+                "--python",
+                "3.13",
+            ])
+            .stdout(std::process::Stdio::piped())
+            .stderr(std::process::Stdio::piped())
+            .kill_on_drop(true);
+        let venv_output = venv_command.spawn()?.wait_with_output().await?;
+        if !venv_output.status.success() {
+            return Err(anyhow::Error::new(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                String::from_utf8(venv_output.stderr)?,
+            )));
+        }
+
+        log::info!("successfully created Python venv at {:?}", venv_path);
+
+        let python_path_at_venv = venv_path
+            .join(if cfg!(windows) { "Scripts" } else { "bin" })
+            .join(if cfg!(windows) {
+                "python.exe"
+            } else {
+                "python"
+            });
+
+        let python_binary_path = python_path_at_venv.to_string_lossy().to_string();
+        log::info!("python binary path: {}", python_binary_path);
+        Ok(python_binary_path)
+    }
+
     pub async fn check(&self) -> anyhow::Result<Vec<String>> {
         let execution_storage =
             ExecutionStorage::new(self.code.clone(), self.options.context.clone());
         execution_storage.init_for_python(None)?;
 
-        let binary_path = path::absolute(self.options.python_binary_path.clone())
-            .unwrap()
-            .to_string_lossy()
-            .to_string();
-        let mut command = tokio::process::Command::new(binary_path);
+        let python_binary_path: String = self
+            .ensure_python_venv(execution_storage.python_cache_folder_path())
+            .await?;
+
+        log::info!(
+            "using python from host at path: {:?}",
+            python_binary_path.clone()
+        );
+
+        let mut command = tokio::process::Command::new(python_binary_path);
+        println!(
+            "compiling code in folder: {}",
+            execution_storage.code_folder_path.to_str().unwrap()
+        );
         command
             .args([
                 "-m",
@@ -250,7 +301,10 @@ print("</shinkai-code-result>")
         container_envs.push(String::from("-e"));
         container_envs.push(format!("SHINKAI_MOUNT={}", mount_env));
         container_envs.push(String::from("-e"));
-        container_envs.push(format!("SHINKAI_CONTEXT_ID={}", self.options.context.context_id));
+        container_envs.push(format!(
+            "SHINKAI_CONTEXT_ID={}",
+            self.options.context.context_id
+        ));
         container_envs.push(String::from("-e"));
         container_envs.push(format!(
             "SHINKAI_EXECUTION_ID={}",
@@ -278,7 +332,7 @@ print("</shinkai-code-result>")
             .to_string_lossy()
             .to_string();
         let python_start_script = format!(
-            "source /app/cache/python-venv/bin/activate && python -m pipreqs.pipreqs --encoding utf-8 --force {} && pip install -r {}/requirements.txt && python {}",
+            "source /app/cache/python-venv/bin/activate && python -m pipreqs.pipreqs --encoding utf-8 --force {} && uv pip install -r {}/requirements.txt && python {}",
             code_folder.clone().as_str(),
             code_folder.clone().as_str(),
             code_entrypoint.clone().as_str(),
@@ -384,45 +438,31 @@ print("</shinkai-code-result>")
         let execution_storage = ExecutionStorage::new(code_files, self.options.context.clone());
         execution_storage.init_for_python(None)?;
 
-        let python_binary_path_at_host = path::absolute(self.options.python_binary_path.clone())
-            .unwrap()
-            .to_string_lossy()
-            .to_string();
+        let python_binary_path: String = self
+            .ensure_python_venv(execution_storage.python_cache_folder_path())
+            .await?;
+
         log::info!(
             "using python from host at path: {:?}",
-            python_binary_path_at_host.clone()
+            python_binary_path.clone()
         );
 
-        // Ensure venv exists at python cache location
-        let venv_path = execution_storage.python_cache_folder_path();
-        log::info!("Creating new venv at {:?}", venv_path);
-        let mut venv_command = tokio::process::Command::new(&python_binary_path_at_host);
-        venv_command
-            .args(["-m", "venv", venv_path.to_str().unwrap()])
-            .stdout(std::process::Stdio::piped())
-            .stderr(std::process::Stdio::piped())
-            .kill_on_drop(true);
-        let venv_output = venv_command.spawn()?.wait_with_output().await?;
-        if !venv_output.status.success() {
-            return Err(anyhow::Error::new(std::io::Error::new(
-                std::io::ErrorKind::Other,
-                String::from_utf8(venv_output.stderr)?,
-            )));
-        }
-
-        // Get pip and python paths from venv
-        let python_path_at_venv = venv_path
-            .join(if cfg!(windows) { "Scripts" } else { "bin" })
-            .join(if cfg!(windows) {
-                "python.exe"
-            } else {
-                "python"
-            });
+        let uv_binary_path = path::absolute(self.options.uv_binary_path.clone())
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .to_string();
 
-        let python_binary_path = python_path_at_venv.to_string_lossy().to_string();
-        let mut ensure_pip_command = tokio::process::Command::new(&python_binary_path);
+        let mut ensure_pip_command = tokio::process::Command::new(&uv_binary_path);
         ensure_pip_command
-            .args(["-m", "pip", "install", "pipreqs"])
+            .args(["pip", "install", "pipreqs"])
+            .env(
+                "VIRTUAL_ENV",
+                execution_storage
+                    .python_cache_folder_path()
+                    .to_str()
+                    .unwrap(),
+            )
             .stdout(std::process::Stdio::piped())
             .stderr(std::process::Stdio::piped())
             .kill_on_drop(true);
@@ -456,9 +496,16 @@ print("</shinkai-code-result>")
             )));
         }
 
-        let mut pip_install_command = tokio::process::Command::new(&python_binary_path);
+        let mut pip_install_command = tokio::process::Command::new(&uv_binary_path);
         pip_install_command
-            .args(["-m", "pip", "install", "-r", "requirements.txt"])
+            .args(["pip", "install", "-r", "requirements.txt"])
+            .env(
+                "VIRTUAL_ENV",
+                execution_storage
+                    .python_cache_folder_path()
+                    .to_str()
+                    .unwrap(),
+            )
             .current_dir(execution_storage.code_folder_path.clone())
             .stdout(std::process::Stdio::piped())
             .stderr(std::process::Stdio::piped())
@@ -510,8 +557,14 @@ print("</shinkai-code-result>")
                 .collect::<Vec<_>>()
                 .join(","),
         );
-        command.env("SHINKAI_CONTEXT_ID", self.options.context.context_id.clone());
-        command.env("SHINKAI_EXECUTION_ID", self.options.context.execution_id.clone());
+        command.env(
+            "SHINKAI_CONTEXT_ID",
+            self.options.context.context_id.clone(),
+        );
+        command.env(
+            "SHINKAI_EXECUTION_ID",
+            self.options.context.execution_id.clone(),
+        );
 
         if let Some(envs) = envs {
             command.envs(envs);

libs/shinkai-tools-runner/src/tools/python_runner_options.rs

@@ -8,7 +8,7 @@ use super::{
 #[derive(Clone)]
 pub struct PythonRunnerOptions {
     pub context: ExecutionContext,
-    pub python_binary_path: PathBuf,
+    pub uv_binary_path: PathBuf,
     pub code_runner_docker_image_name: String,
     pub force_runner_type: Option<RunnerType>,
     pub shinkai_node_location: ShinkaiNodeLocation,
@@ -18,11 +18,11 @@ impl Default for PythonRunnerOptions {
     fn default() -> Self {
         Self {
             context: ExecutionContext::default(),
-            code_runner_docker_image_name: String::from("dcspark/shinkai-code-runner:0.9.0"),
-            python_binary_path: PathBuf::from(if cfg!(windows) {
-                "C://Users/agall/AppData/Local/Microsoft/WindowsApps/python"
+            code_runner_docker_image_name: String::from("dcspark/shinkai-code-runner:0.9.1"),
+            uv_binary_path: PathBuf::from(if cfg!(windows) {
+                format!("C:\\Users\\{}\\.local\\bin\\uv.exe", whoami::username())
             } else {
-                "python"
+                format!("/Users/{}/.local/bin/uv", whoami::username())
             }),
             force_runner_type: None,
             shinkai_node_location: ShinkaiNodeLocation {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shinkai_protocol/source",
-  "version": "0.9.2",
+  "version": "0.9.3",
   "description": "This repository serves as the ecosystem to execute Shinkai tools, provided by the Shinkai team or third-party developers, in a secure environment. It provides a sandboxed space for executing these tools, ensuring that they run safely and efficiently, while also allowing for seamless integration with Rust code.",
   "main": "index.js",
   "author": "",