I tried 0.40. dev13 library, getting bugs as illustrated below on OPENAI ASSISTANT and Web Surfer Agents

[viswanadula8828]

What happened?

I have create a roundrobingroupchat with the below mentioned agents. I have installed playwright with debs chromium and I have received below error

from fastapi import FastAPI, HTTPException, Depends
from fastapi.responses import JSONResponse, RedirectResponse
from autogen_agentchat.teams import RoundRobinGroupChat
from autogen_agentchat.agents import CodeExecutorAgent, CodingAssistantAgent
from autogen_ext.models.openai import OpenAIChatCompletionClient
from autogen_ext.agents.web_surfer import MultimodalWebSurfer
from autogen_ext.agents.openai import OpenAIAssistantAgent
from autogen_ext.code_executors.azure import ACADynamicSessionsCodeExecutor
from autogen_ext.code_executors.docker import DockerCommandLineCodeExecutor
from typing import Optional, List
from openai import AsyncClient

import dotenv
dotenv.load_dotenv()
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())

app = FastAPI()
# Load autogen agent configurations from a JSON configuration list

def task_result_to_dict(task_result):
    return {
        "messages": [message.__dict__ for message in task_result.messages],
        "stop_reason": task_result.stop_reason,
    }

async def initialize_agents():
    # OpenAI Assistant Agent
    client = OpenAIChatCompletionClient(
        model="gpt-4o-2024-08-06",
        api_key=os.getenv("OPENAI_API_KEY")
    )
    openai_client = AsyncClient(api_key=os.getenv("OPENAI_API_KEY"))
    assistant_agent = OpenAIAssistantAgent(
        client=openai_client,
        name="assistant_agent",
        description="Helps with retreiving information from the the attached vector store.",
        model="gpt-4o",
        instructions="You are a retrieval assistant.",
        assistant_id="asst_ZUSoZeLzx24NHiB3dNGAwWRH",
        tools=["file_search"]
    )

    # Web Surfer Agent
    web_surfer_agent = MultimodalWebSurfer(
        name="web_surfer_agent",
        model_client=client
    )

    # Code Executor Agent with Docker
    code_executor = DockerCommandLineCodeExecutor()
    # code_executor = ACADynamicSessionsCodeExecutor(pool_management_endpoint=os.getenv("POOL_MANAGEMENT_ENDPOINT"))
    code_executor_agent = CodeExecutorAgent(
        name="code_executor_agent",
        code_executor=code_executor
    )

    # Coder Agent
    coder_agent = OpenAIAssistantAgent(
        name="coder_agent",
        description="Helps with Python programming",
        client=client,
        model="gpt-4o",
        instructions="You are a helpful Python programming assistant.",
        tools=["code_interpreter"],
    )

    # Group Chat with RoundRobin strategy
    agent_team = RoundRobinGroupChat(
        [assistant_agent, web_surfer_agent, code_executor_agent, coder_agent]
    )

    return agent_team

async def get_agent_team():
    return await initialize_agents()

@app.get("/")
async def root():
    return RedirectResponse("/docs")


@app.get("/chat")
async def chat(message: str, agent_team= Depends(get_agent_team)):
    try:
        # Initiating chat between CodeExecutor and CodeWriter with the provided message
        chat_result = await agent_team.run(task=message)
        print(chat_result)
        task_result_dict = task_result_to_dict(chat_result)
        result_json = json.dumps(task_result_dict, indent=2)
        print(result_json)
        # Find the last message containing 'Code output:' from assistant
        # messages = [message.content for message in chat_result.messages]
        return JSONResponse(content={"result": result_json})
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))```

# Error on OpenAI

```File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_ext\agents\openai\_openai_assistant_agent.py", line 261, in _ensure_initialized
    self._assistant = await self._client.beta.assistants.create(
                            ^^^^^^^^^^^^^^^^^
AttributeError: 'OpenAIChatCompletionClient' object has no attribute 'beta'``

# Error on WebSurfer

```Web surfing error:\n\nTraceback (most recent call last):\n  File \"c:\\Users\\lv\\Documents\\Git\\AUTOGENMAGONEREPLICA\\.venv\\Lib\\site-packages\\autogen_ext\\agents\\web_surfer\\_multimodal_web_surfer.py\", line 377, in on_messages_stream\n    content = await self._generate_reply(cancellation_token=cancellation_token)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"c:\\Users\\lv\\Documents\\Git\\AUTOGENMAGONEREPLICA\\.venv\\Lib\\site-packages\\autogen_ext\\agents\\web_surfer\\_multimodal_web_surfer.py\", line 404, in _generate_reply\n    await self._lazy_init()\n  File \"c:\\Users\\lv\\Documents\\Git\\AUTOGENMAGONEREPLICA\\.venv\\Lib\\site-packages\\autogen_ext\\agents\\web_surfer\\_multimodal_web_surfer.py\", line 256, in _lazy_init\n    self._playwright = await async_playwright().start()\n                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"c:\\Users\\lv\\Documents\\Git\\AUTOGENMAGONEREPLICA\\.venv\\Lib\\site-packages\\playwright\\async_api\\_context_manager.py\", line 51, in start\n    return await self.__aenter__()\n           ^^^^^^^^^^^^^^^^^^^^^^^\n  File \"c:\\Users\\lv\\Documents\\Git\\AUTOGENMAGONEREPLICA\\.venv\\Lib\\site-packages\\playwright\\async_api\\_context_manager.py\", line 46, in __aenter__\n    playwright = AsyncPlaywright(next(iter(done)).result())\n                                 ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"c:\\Users\\lv\\Documents\\Git\\AUTOGENMAGONEREPLICA\\.venv\\Lib\\site-packages\\playwright\\_impl\\_transport.py\", line 120, in connect\n    self._proc = await asyncio.create_subprocess_exec(\n                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\\Lib\\asyncio\\subprocess.py\", line 223, in create_subprocess_exec\n    transport, protocol = await loop.subprocess_exec(\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\\Lib\\asyncio\\base_events.py\", line 1708, in subprocess_exec\n    transport = await self._make_subprocess_transport(\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\\Lib\\asyncio\\base_events.py\", line 503, in _make_subprocess_transport\n    raise NotImplementedError\nNotImplementedError\n",
      "type": "TextMessage"
n connect\n    self._proc = await asyncio.create_subprocess_exec(\n                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\\Lib\\asyncio\\subprocess.py\", line 223, in create_subprocess_exec\n    transport, protocol = await loop.subprocess_exec(\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\\Lib\\asyncio\\base_events.py\", line 1708, in subprocess_exec\n    transport = await self._make_subprocess_transport(\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\\Lib\\asyncio\\base_events.py\", line 503, in _make_subprocess_transport\n    raise NotImplementedError\nNotImplementedError\n```

### What did you expect to happen?

With no errors

### How can we reproduce it (as minimally and precisely as possible)?

Create OpenAI agent with below code
```openai_client = AsyncClient(api_key=os.getenv("OPENAI_API_KEY"))
    assistant_agent = OpenAIAssistantAgent(
        client=openai_client,
        name="assistant_agent",
        description="Helps with retreiving information from the the attached vector store.",
        model="gpt-4o",
        instructions="You are a retrieval assistant.",
        assistant_id="asst_ZUSoZeLzx24NHiB3dNGAwWRH",
        tools=["file_search"]
    )```

Create WebSurfer Agent with below code
# Web Surfer Agent
    ```web_surfer_agent = MultimodalWebSurfer(
        name="web_surfer_agent",
        model_client=client
    )```

```agent_team = RoundRobinGroupChat(
        [assistant_agent, web_surfer_agent, code_executor_agent, coder_agent]
    )```


### AutoGen version

0.40.dev13

### Which package was this bug in

Extensions

### Model used

gpt-4o

### Python version

3.11

### Operating system

Windows

### Any additional info you think would be helpful for fixing this bug

_No response_

[ekzhu]

Looks like an openai client issue. We are looking into it.

[jackgerrits]

What is the openai client version you have?

pip freeze | grep openai

[viswanadula8828]

openai==1.59.5

[ekzhu]

Looking at this part of your code. It should be using openai_client rather than client.

# Coder Agent
    coder_agent = OpenAIAssistantAgent(
        name="coder_agent",
        description="Helps with Python programming",
        client=client,
        model="gpt-4o",
        instructions="You are a helpful Python programming assistant.",
        tools=["code_interpreter"],
    )

[viswanadula8828]

Okay, let me have a look at it and how about playwright issue for the websurfer agent?

[viswanadula8828]

@ekzhu Openai issue is resolved with your suggestion. Websurfer agent issue still exists. Few more logs for your reference

Task exception was never retrieved
future: <Task finished name='Task-53' coro=<Connection.run() done, defined at c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\playwright\_impl\_connection.py:272> exception=NotImplementedError()>
Traceback (most recent call last):
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\playwright\_impl\_connection.py", line 279, in run
    await self._transport.connect()
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\playwright\_impl\_transport.py", line 133, in connect
    raise exc
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\playwright\_impl\_transport.py", line 120, in connect
    self._proc = await asyncio.create_subprocess_exec(
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\asyncio\subprocess.py", line 223, in create_subprocess_exec
    transport, protocol = await loop.subprocess_exec(
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\asyncio\base_events.py", line 1708, in subprocess_exec
    transport = await self._make_subprocess_transport(
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\asyncio\base_events.py", line 503, in _make_subprocess_transport
    raise NotImplementedError
NotImplementedError
Task exception was never retrieved
future: <Task finished name='Task-131' coro=<Connection.run() done, defined at c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\playwright\_impl\_connection.py:272> exception=NotImplementedError()>
Traceback (most recent call last):
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\playwright\_impl\_connection.py", line 279, in run
    await self._transport.connect()
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\playwright\_impl\_transport.py", line 133, in connect
    raise exc
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\playwright\_impl\_transport.py", line 120, in connect
    self._proc = await asyncio.create_subprocess_exec(
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\asyncio\subprocess.py", line 223, in create_subprocess_exec
    transport, protocol = await loop.subprocess_exec(
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\asyncio\base_events.py", line 1708, in subprocess_exec
    transport = await self._make_subprocess_transport(
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\asyncio\base_events.py", line 503, in _make_subprocess_transport
    raise NotImplementedError
NotImplementedError
Error processing publish message
Traceback (most recent call last):
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_core\_single_threaded_agent_runtime.py", line 397, in _process_publish
    await asyncio.gather(*responses)
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_core\_single_threaded_agent_runtime.py", line 389, in _on_message
    return await agent.on_message(
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_core\_base_agent.py", line 113, in on_message
    return await self.on_message_impl(message, ctx)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_agentchat\teams\_group_chat\_sequential_routed_agent.py", line 48, in on_message_impl
    return await super().on_message_impl(message, ctx)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_core\_routed_agent.py", line 485, in on_message_impl
    return await h(self, message, ctx)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_core\_routed_agent.py", line 268, in wrapper
    return_value = await func(self, message, ctx)  # type: ignore
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_agentchat\teams\_group_chat\_chat_agent_container.py", line 53, in handle_request
    async for msg in self._agent.on_messages_stream(self._message_buffer, ctx.cancellation_token):
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_agentchat\agents\_base_chat_agent.py", line 98, in on_messages_stream
    response = await self.on_messages(messages, cancellation_token)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_agentchat\agents\_code_executor_agent.py", line 95, in on_messages
    result = await self._code_executor.execute_code_blocks(code_blocks, cancellation_token=cancellation_token)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_ext\code_executors\docker\_docker_code_executor.py", line 291, in execute_code_blocks
    return await self._execute_code_dont_check_setup(code_blocks, cancellation_token)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\lv\Documents\Git\AUTOGENMAGONEREPLICA\.venv\Lib\site-packages\autogen_ext\code_executors\docker\_docker_code_executor.py", line 228, in _execute_code_dont_check_setup
    raise ValueError("Container is not running. Must first be started with either start or a context manager.")
ValueError: Container is not running. Must first be started with either start or a context manager.

[jackgerrits]

Your docker code executor was not started - usually this is done with a with statement:

async with DockerCommandLineCodeExecutor() as code_executor:
    code_executor_agent = CodeExecutorAgent(
        name="code_executor_agent",
        code_executor=code_executor
    )