diff --git a/Cargo.lock b/Cargo.lock index 2f04aaf..0d50a03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1877,6 +1877,7 @@ dependencies = [ "tempfile", "tokio", "tokio-util", + "whoami", "zip", ] @@ -2232,6 +2233,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -2308,6 +2315,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "whoami" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +dependencies = [ + "redox_syscall", + "wasite", + "web-sys", +] + [[package]] name = "windows-core" version = "0.52.0" diff --git a/docker-images/Dockerfile b/docker-images/Dockerfile index 1f3640c..486cc28 100644 --- a/docker-images/Dockerfile +++ b/docker-images/Dockerfile @@ -7,10 +7,12 @@ SHELL ["/bin/bash", "-c"] RUN apt-get update && apt-get install -y chromium ENV CHROME_PATH=/usr/bin/chromium -# Add python 3 -RUN apt-get install -y python3 python3-pip python3-venv +RUN apt-get install -y curl WORKDIR /app -RUN python3 -m venv cache/python-venv && source cache/python-venv/bin/activate && pip install pipreqs +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +ENV PATH="/root/.local/bin:${PATH}" + +RUN source $HOME/.local/bin/env && uv venv --seed cache/python-venv --python 3.13 && source cache/python-venv/bin/activate && uv pip install pipreqs ENTRYPOINT ["/tini", "--"] diff --git a/libs/shinkai-tools-runner/Cargo.toml b/libs/shinkai-tools-runner/Cargo.toml index b3df013..2505c03 100644 --- a/libs/shinkai-tools-runner/Cargo.toml +++ b/libs/shinkai-tools-runner/Cargo.toml @@ -32,6 +32,7 @@ once_cell = { version = "1.20.2" } env_logger = "0.11.5" anyhow = { version = "1.0.93" } chrono = { version = "0.4.38" } +whoami = "1.5" [dev-dependencies] rstest = "0.23.0" diff --git a/libs/shinkai-tools-runner/src/tools/deno_runner_options.rs b/libs/shinkai-tools-runner/src/tools/deno_runner_options.rs index 6215657..dc74e86 100644 --- a/libs/shinkai-tools-runner/src/tools/deno_runner_options.rs +++ b/libs/shinkai-tools-runner/src/tools/deno_runner_options.rs @@ -9,7 +9,6 @@ use super::{ pub struct DenoRunnerOptions { pub context: ExecutionContext, pub deno_binary_path: PathBuf, - pub python_binary_path: PathBuf, pub code_runner_docker_image_name: String, pub force_runner_type: Option, pub shinkai_node_location: ShinkaiNodeLocation, @@ -19,17 +18,12 @@ impl Default for DenoRunnerOptions { fn default() -> Self { Self { context: ExecutionContext::default(), - code_runner_docker_image_name: String::from("dcspark/shinkai-code-runner:0.9.0"), + code_runner_docker_image_name: String::from("dcspark/shinkai-code-runner:0.9.1"), deno_binary_path: PathBuf::from(if cfg!(windows) { "./shinkai-tools-runner-resources/deno.exe" } else { "./shinkai-tools-runner-resources/deno" }), - python_binary_path: PathBuf::from(if cfg!(windows) { - "C://Users/agall/AppData/Local/Microsoft/WindowsApps/python" - } else { - "python" - }), force_runner_type: None, shinkai_node_location: ShinkaiNodeLocation { protocol: String::from("http"), diff --git a/libs/shinkai-tools-runner/src/tools/python_runner.rs b/libs/shinkai-tools-runner/src/tools/python_runner.rs index 3e8d878..fbadf9c 100644 --- a/libs/shinkai-tools-runner/src/tools/python_runner.rs +++ b/libs/shinkai-tools-runner/src/tools/python_runner.rs @@ -1,7 +1,7 @@ use serde_json::Value; use std::{ collections::HashMap, - path::{self, Path}, + path::{self, Path, PathBuf}, sync::Arc, time::Duration, }; @@ -41,16 +41,67 @@ impl PythonRunner { } } + async fn ensure_python_venv(&self, venv_path: PathBuf) -> anyhow::Result { + let uv_binary_path_at_host = path::absolute(self.options.uv_binary_path.clone()) + .unwrap() + .to_string_lossy() + .to_string(); + + log::info!("Creating new venv with uv {:?}", venv_path); + let mut venv_command = tokio::process::Command::new(&uv_binary_path_at_host); + venv_command + .args([ + "venv", + "--seed", + venv_path.to_str().unwrap(), + "--python", + "3.13", + ]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + let venv_output = venv_command.spawn()?.wait_with_output().await?; + if !venv_output.status.success() { + return Err(anyhow::Error::new(std::io::Error::new( + std::io::ErrorKind::Other, + String::from_utf8(venv_output.stderr)?, + ))); + } + + log::info!("successfully created Python venv at {:?}", venv_path); + + let python_path_at_venv = venv_path + .join(if cfg!(windows) { "Scripts" } else { "bin" }) + .join(if cfg!(windows) { + "python.exe" + } else { + "python" + }); + + let python_binary_path = python_path_at_venv.to_string_lossy().to_string(); + log::info!("python binary path: {}", python_binary_path); + Ok(python_binary_path) + } + pub async fn check(&self) -> anyhow::Result> { let execution_storage = ExecutionStorage::new(self.code.clone(), self.options.context.clone()); execution_storage.init_for_python(None)?; - let binary_path = path::absolute(self.options.python_binary_path.clone()) - .unwrap() - .to_string_lossy() - .to_string(); - let mut command = tokio::process::Command::new(binary_path); + let python_binary_path: String = self + .ensure_python_venv(execution_storage.python_cache_folder_path()) + .await?; + + log::info!( + "using python from host at path: {:?}", + python_binary_path.clone() + ); + + let mut command = tokio::process::Command::new(python_binary_path); + println!( + "compiling code in folder: {}", + execution_storage.code_folder_path.to_str().unwrap() + ); command .args([ "-m", @@ -250,7 +301,10 @@ print("") container_envs.push(String::from("-e")); container_envs.push(format!("SHINKAI_MOUNT={}", mount_env)); container_envs.push(String::from("-e")); - container_envs.push(format!("SHINKAI_CONTEXT_ID={}", self.options.context.context_id)); + container_envs.push(format!( + "SHINKAI_CONTEXT_ID={}", + self.options.context.context_id + )); container_envs.push(String::from("-e")); container_envs.push(format!( "SHINKAI_EXECUTION_ID={}", @@ -278,7 +332,7 @@ print("") .to_string_lossy() .to_string(); let python_start_script = format!( - "source /app/cache/python-venv/bin/activate && python -m pipreqs.pipreqs --encoding utf-8 --force {} && pip install -r {}/requirements.txt && python {}", + "source /app/cache/python-venv/bin/activate && python -m pipreqs.pipreqs --encoding utf-8 --force {} && uv pip install -r {}/requirements.txt && python {}", code_folder.clone().as_str(), code_folder.clone().as_str(), code_entrypoint.clone().as_str(), @@ -384,45 +438,31 @@ print("") let execution_storage = ExecutionStorage::new(code_files, self.options.context.clone()); execution_storage.init_for_python(None)?; - let python_binary_path_at_host = path::absolute(self.options.python_binary_path.clone()) - .unwrap() - .to_string_lossy() - .to_string(); + let python_binary_path: String = self + .ensure_python_venv(execution_storage.python_cache_folder_path()) + .await?; + log::info!( "using python from host at path: {:?}", - python_binary_path_at_host.clone() + python_binary_path.clone() ); - // Ensure venv exists at python cache location - let venv_path = execution_storage.python_cache_folder_path(); - log::info!("Creating new venv at {:?}", venv_path); - let mut venv_command = tokio::process::Command::new(&python_binary_path_at_host); - venv_command - .args(["-m", "venv", venv_path.to_str().unwrap()]) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .kill_on_drop(true); - let venv_output = venv_command.spawn()?.wait_with_output().await?; - if !venv_output.status.success() { - return Err(anyhow::Error::new(std::io::Error::new( - std::io::ErrorKind::Other, - String::from_utf8(venv_output.stderr)?, - ))); - } - - // Get pip and python paths from venv - let python_path_at_venv = venv_path - .join(if cfg!(windows) { "Scripts" } else { "bin" }) - .join(if cfg!(windows) { - "python.exe" - } else { - "python" - }); + let uv_binary_path = path::absolute(self.options.uv_binary_path.clone()) + .unwrap() + .to_str() + .unwrap() + .to_string(); - let python_binary_path = python_path_at_venv.to_string_lossy().to_string(); - let mut ensure_pip_command = tokio::process::Command::new(&python_binary_path); + let mut ensure_pip_command = tokio::process::Command::new(&uv_binary_path); ensure_pip_command - .args(["-m", "pip", "install", "pipreqs"]) + .args(["pip", "install", "pipreqs"]) + .env( + "VIRTUAL_ENV", + execution_storage + .python_cache_folder_path() + .to_str() + .unwrap(), + ) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .kill_on_drop(true); @@ -456,9 +496,16 @@ print("") ))); } - let mut pip_install_command = tokio::process::Command::new(&python_binary_path); + let mut pip_install_command = tokio::process::Command::new(&uv_binary_path); pip_install_command - .args(["-m", "pip", "install", "-r", "requirements.txt"]) + .args(["pip", "install", "-r", "requirements.txt"]) + .env( + "VIRTUAL_ENV", + execution_storage + .python_cache_folder_path() + .to_str() + .unwrap(), + ) .current_dir(execution_storage.code_folder_path.clone()) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) @@ -510,8 +557,14 @@ print("") .collect::>() .join(","), ); - command.env("SHINKAI_CONTEXT_ID", self.options.context.context_id.clone()); - command.env("SHINKAI_EXECUTION_ID", self.options.context.execution_id.clone()); + command.env( + "SHINKAI_CONTEXT_ID", + self.options.context.context_id.clone(), + ); + command.env( + "SHINKAI_EXECUTION_ID", + self.options.context.execution_id.clone(), + ); if let Some(envs) = envs { command.envs(envs); diff --git a/libs/shinkai-tools-runner/src/tools/python_runner_options.rs b/libs/shinkai-tools-runner/src/tools/python_runner_options.rs index b1b1600..a73026c 100644 --- a/libs/shinkai-tools-runner/src/tools/python_runner_options.rs +++ b/libs/shinkai-tools-runner/src/tools/python_runner_options.rs @@ -8,7 +8,7 @@ use super::{ #[derive(Clone)] pub struct PythonRunnerOptions { pub context: ExecutionContext, - pub python_binary_path: PathBuf, + pub uv_binary_path: PathBuf, pub code_runner_docker_image_name: String, pub force_runner_type: Option, pub shinkai_node_location: ShinkaiNodeLocation, @@ -18,11 +18,11 @@ impl Default for PythonRunnerOptions { fn default() -> Self { Self { context: ExecutionContext::default(), - code_runner_docker_image_name: String::from("dcspark/shinkai-code-runner:0.9.0"), - python_binary_path: PathBuf::from(if cfg!(windows) { - "C://Users/agall/AppData/Local/Microsoft/WindowsApps/python" + code_runner_docker_image_name: String::from("dcspark/shinkai-code-runner:0.9.1"), + uv_binary_path: PathBuf::from(if cfg!(windows) { + format!("C:\\Users\\{}\\.local\\bin\\uv.exe", whoami::username()) } else { - "python" + format!("/Users/{}/.local/bin/uv", whoami::username()) }), force_runner_type: None, shinkai_node_location: ShinkaiNodeLocation { diff --git a/package-lock.json b/package-lock.json index 8a35df3..6dea610 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@shinkai_protocol/source", - "version": "0.9.2", + "version": "0.9.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@shinkai_protocol/source", - "version": "0.9.2", + "version": "0.9.3", "license": "SEE LICENSE IN LICENSE", "dependencies": { "@coinbase/coinbase-sdk": "^0.0.16", diff --git a/package.json b/package.json index 7eaf233..0d3d620 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@shinkai_protocol/source", - "version": "0.9.2", + "version": "0.9.3", "description": "This repository serves as the ecosystem to execute Shinkai tools, provided by the Shinkai team or third-party developers, in a secure environment. It provides a sandboxed space for executing these tools, ensuring that they run safely and efficiently, while also allowing for seamless integration with Rust code.", "main": "index.js", "author": "",