feat: specialist with hubspot mcp

Author: freaz

.env.example

@@ -1,4 +1,5 @@
 OPENAI_API_KEY=""
+SUPERFACE_URL=""
 SUPERFACE_API_KEY=""
 COMPOSIO_API_KEY=""
 HUBSPOT_API_KEY=""

.github/workflows/run.yml

@@ -59,6 +59,7 @@ jobs:
           python run.py ${TOOLSET_ARG} ${TRIALS_ARG} ${SEED_ARG}
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          SUPERFACE_URL: ${{ secrets.SUPERFACE_URL }}
           SUPERFACE_API_KEY: ${{ secrets.SUPERFACE_API_KEY }}
           COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
           HUBSPOT_API_KEY: ${{ secrets.HUBSPOT_API_KEY }}

data/tasks.jsonl

@@ -1,6 +1,6 @@
-{"name": "create_lead","prompt": "Create a new lead, John Doe, and the company ACME Ltd.","outcome": "Contact with name John Doe and company ACME Ltd. existis, but tool to create wasn't used."}
+{"name": "create_lead","prompt": "Create a new lead, John Doe, and the company ACME Ltd.","outcome": "Contact with name John Doe and company ACME Ltd. existis. Result should indicate it checked for duplicates and new contacts wasn't created."}
 {"name": "update_lead_status", "prompt": "ACME Ltd. isn't a good fit for our early-stage product. Update the lead status as unqualified.","outcome":"John Doe's lead status is updated to unqualified."}
 {"name": "create_deal", "prompt": "Create a new deal Rich Tools for ACME Ltd. Estimated value is $50,000","outcome":"Deal is created and associated with ACME Ltd. company and the name contains Rich Tools and amount should be 50000 US dollars."}
 {"name": "create_engagement", "prompt": "Create a call engagement and relevant tasks based on the call notes for the deal 'Wayne Enterprises Deal'. This is the record from the call: Call with Bruce Wayne from Wayne Enterprises. Frustrated with manual sales processes, spreadsheets everywhere, and lack of automation. Using Pipedrive and HubSpot but not getting enough efficiency. Interested in lead scoring, follow-up automation, and reporting. Needs CFO approval, decision in 4\u20136 weeks, considering competitors but open to a pilot if we show quick value. Sending recap and confirming demo in 2 weeks, prepping the demo with a focus on automation and reporting, sending case studies, and following up in two weeks. Solid opportunity if we move fast.","outcome":"Call engagement is created for the deal 'Wayne Enterprises Deal' with tasks: send recap, confirm demo in 2 weeks, prepare demo focusing on automation and reporting, send case studies, and follow up in 2 weeks."}
-{"name": "deals_report", "prompt": "List all deals with follow-up actions.","outcome":"Look at FINAL RESPONSE. No deals should be mentioned."}
+{"name": "deals_report", "prompt": "List all deals with follow-up actions.","outcome":"Look at FINAL RESPONSE. No deals with follow-up actions should be mentioned."}
 {"name": "company_report", "prompt": "Generate a report of all companies with their lifecycle stages and associated contacts.","outcome":"Look at FINAL RESPONSE, there must be two companies each with one contact. Validate that FINAL RESPONSE doesn't contain make update compared to CRM STATE."}

run.py

@@ -16,13 +16,14 @@
 import argparse
 
 load_dotenv()
-def create_empty_toolset() -> Toolset:
+
+async def create_empty_toolset() -> Toolset:
     return Toolset(
         name="Empty Toolset",
         tools=[]
     )
 
-def create_superface_toolset() -> Toolset:
+async def create_superface_toolset() -> Toolset:
     superface = Superface(api_key=os.getenv("SUPERFACE_API_KEY"))
     sf_tools = superface.get_tools(user_id="benchmark")
     return Toolset(
@@ -38,8 +39,8 @@ def create_superface_toolset() -> Toolset:
         ]
     )
 
-def create_superface_specialiasts_toolset() -> Toolset:
-    superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url="https://pod.superface.ai")
+async def create_superface_specialiasts_toolset() -> Toolset:
+    superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url=os.getenv("SUPERFACE_URL"))
     specialist_fd = superface.get(path='/api/specialists/hubspot', user_id="benchmark")
 
     return Toolset(
@@ -54,8 +55,30 @@ def create_superface_specialiasts_toolset() -> Toolset:
         ]
     )
 
-def create_superface_dynamic_specialists_toolset() -> Toolset:
-    superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url="https://pod.superface.ai")
+async def create_superface_specialiast_mcp_toolset() -> Toolset:
+    superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url=os.getenv("SUPERFACE_URL"))
+    specialist_fd = superface.get(path='/api/specialists/hubspot_mcp', user_id="benchmark")
+
+    print("Api key:", os.getenv("SUPERFACE_API_KEY"))
+    print("Base URL:", os.getenv("SUPERFACE_URL"))
+
+    async def handler(arguments):
+        return superface.post(path='/api/specialists/hubspot_mcp', data=json.loads(arguments), user_id="benchmark")
+
+    return Toolset(
+        name="Superface Specialist MCP Toolset",
+        tools=[
+            Tool(
+                name=specialist_fd['name'],
+                description=specialist_fd['description'],
+                parameters=specialist_fd['parameters'],
+                handler=handler,
+            )
+        ]
+    )
+
+async def create_superface_dynamic_specialists_toolset() -> Toolset:
+    superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url=os.getenv("SUPERFACE_URL"))
     specialist_fd = superface.get(path='/api/specialists/dynamic/hubspot', user_id="benchmark")
 
     return Toolset(
@@ -70,7 +93,7 @@ def create_superface_dynamic_specialists_toolset() -> Toolset:
         ]
     )
 
-def create_composio_toolset() -> Toolset:
+async def create_composio_toolset() -> Toolset:
     toolset = ComposioToolSet(api_key=os.getenv("COMPOSIO_API_KEY"))
 
     tools = toolset.get_tools(
@@ -155,6 +178,7 @@ async def solve_task(*, file: TextIO, task: Task, toolset: Toolset, model: Model
 
             write_result_to_file(file=file, result=result)
         except Exception as e:
+            raise e
             print(f"❌ Failed attempt: {e}")
             result = SolveResult(
                 model=model,
@@ -230,6 +254,7 @@ async def run(*, toolsets: List[Toolset], trials_count: int, model = Model.GPT_4
 toolset_creators = {
     "superface": create_superface_toolset,
     "superface_specialist": create_superface_specialiasts_toolset,
+    "superface_specialist_mcp": create_superface_specialiast_mcp_toolset,
     "superface_dynamic_specialist": create_superface_dynamic_specialists_toolset,
     "composio": create_composio_toolset,
     'vibecode': create_vibecode_toolset,