Added E2B Desktop computer use implementation

.env.example

@@ -6,4 +6,6 @@ OPENAI_ORG = "org-123"
 BROWSERBASE_API_KEY="00000000-0000-0000-0000-000000000000"
 BROWSERBASE_PROJECT_ID="bb_live_00000000-00000"
 
+E2B_API_KEY="e2b_key"
+
 SCRAPYBARA_API_KEY="scrapy-123"

README.md

@@ -25,6 +25,7 @@ Other included sample [computer environments](#computer-environments):
 
 - [Docker](https://docker.com/) (containerized desktop)
 - [Browserbase](https://www.browserbase.com/) (remote browser, requires account)
+- [E2B](https://e2b.dev) (remote computer, open-source, requires account)
 - [Scrapybara](https://scrapybara.com) (remote browser or computer, requires account)
 - ...or implement your own `Computer`!
 
@@ -91,6 +92,7 @@ This sample app provides a set of implemented `Computer` examples, but feel free
 | `LocalPlaywright`   | local-playwright   | `browser` | Local browser window              | [Playwright SDK](https://playwright.dev/)                        |
 | `Docker`            | docker             | `linux`   | Docker container environment      | [Docker](https://docs.docker.com/engine/install/) running        |
 | `Browserbase`       | browserbase        | `browser` | Remote browser environment        | [Browserbase](https://www.browserbase.com/) API key in `.env`    |
+| `E2B`               | e2b                | `linux`   | Open-source desktop environment   | [E2B](https://e2b.dev) API key in `.env`                         |
 | `ScrapybaraBrowser` | scrapybara-browser | `browser` | Remote browser environment        | [Scrapybara](https://scrapybara.com/dashboard) API key in `.env` |
 | `ScrapybaraUbuntu`  | scrapybara-ubuntu  | `linux`   | Remote Ubuntu desktop environment | [Scrapybara](https://scrapybara.com/dashboard) API key in `.env` |
 

agent/agent.py

@@ -19,7 +19,7 @@ class Agent:
 
     def __init__(
         self,
-        model="computer-use-preview-2025-02-04",
+        model="computer-use-preview",
         computer: Computer = None,
         tools: list[dict] = [],
         acknowledge_safety_check_callback: Callable = lambda: False,

cli.py

@@ -2,6 +2,7 @@
 from agent.agent import Agent
 from computers import (
     BrowserbaseBrowser,
+    E2BDesktop,
     ScrapybaraBrowser,
     ScrapybaraUbuntu,
     LocalPlaywrightComputer,
@@ -26,6 +27,7 @@ def main():
             "local-playwright",
             "docker",
             "browserbase",
+            "e2b",
             "scrapybara-browser",
             "scrapybara-ubuntu",
         ],
@@ -60,6 +62,7 @@ def main():
         "local-playwright": LocalPlaywrightComputer,
         "docker": DockerComputer,
         "browserbase": BrowserbaseBrowser,
+        "e2b": E2BDesktop,
         "scrapybara-browser": ScrapybaraBrowser,
         "scrapybara-ubuntu": ScrapybaraUbuntu,
     }

computers/__init__.py

@@ -1,5 +1,6 @@
 from .computer import Computer
 from .browserbase import BrowserbaseBrowser
+from .e2b import E2BDesktop
 from .local_playwright import LocalPlaywrightComputer
 from .docker import DockerComputer
 from .scrapybara import ScrapybaraBrowser, ScrapybaraUbuntu

computers/e2b.py

@@ -0,0 +1,75 @@
+import base64
+from e2b_desktop import Sandbox
+
+class E2BDesktop:
+    """
+    E2B Desktop is an open-source desktop environment for AI Agents.
+    You can get started for free at https://e2b.dev or read our docs at https://docs.e2b.dev
+    """
+
+    def __init__(self):
+        self.environment = "linux"  # "windows", "mac", "linux", or "browser"
+        self.dimensions = (1024, 768)
+
+    def __enter__(self):
+        print("Starting E2B Desktop Sandbox")
+        self.sandbox = Sandbox(
+            resolution=self.dimensions,
+            timeout=300,
+        )
+
+        print(f"Started E2B Desktop Sandbox with id '{self.sandbox.sandbox_id}'")
+
+        print("Starting Desktop Stream")
+        self.sandbox.stream.start(require_auth=True)
+
+        stream_url = self.sandbox.stream.get_url(auth_key=self.sandbox.stream.get_auth_key())
+        print(f"Desktop Stream is running at {stream_url}")
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.sandbox.kill()
+
+    def screenshot(self) -> str:
+        screenshot = self.sandbox.screenshot()
+        base64_image = base64.b64encode(screenshot).decode("utf-8")
+        return base64_image
+
+    def click(self, x: int, y: int, button: str = "left") -> None:
+        match button:
+            case "left":
+                self.sandbox.left_click(x, y)
+            case "right":
+                self.sandbox.right_click(x, y)
+            case "middle":
+                self.sandbox.middle_click(x, y)
+
+    def double_click(self, x: int, y: int) -> None:
+        self.sandbox.double_click(x, y)
+
+    def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        self.sandbox.move_mouse(x, y)
+        self.sandbox.scroll(scroll_x, scroll_y)
+
+    def type(self, text: str) -> None:
+        self.sandbox.write(text)
+
+    def wait(self, ms: int = 1000) -> None:
+        self.sandbox.wait(ms)
+
+    def move(self, x: int, y: int) -> None:
+        self.sandbox.move_mouse(x, y)
+
+    def keypress(self, keys: list[str]) -> None:
+        self.sandbox.press(keys)
+
+    def drag(self, path: list[dict[str, int]]) -> None:
+        if not path:
+            return
+        start_x = path[0]["x"]
+        start_y = path[0]["y"]
+
+        end_x = path[-1]["x"]
+        end_y = path[-1]["y"]
+
+        self.sandbox.drag((start_x, start_y), (end_x, end_y))

requirements.txt

@@ -21,3 +21,4 @@ scrapybara>=2.3.6
 sniffio==1.3.1
 typing_extensions==4.12.2
 urllib3==2.3.0
+e2b_desktop==1.5.2

simple_cua_loop.py

@@ -72,7 +72,7 @@ def main():
 
             while True:  # keep looping until we get a final response
                 response = create_response(
-                    model="computer-use-preview-2025-02-04",
+                    model="computer-use-preview",
                     input=items,
                     tools=tools,
                     truncation="auto",