Breaking change: return metadata in page_to_text (#156)

* return metadata in page_to_text

* fix more tests, fix mypy error

* address comments

* address comments, add test

* Return full xpath (#157)

* remove early exit condition and return full xpath

* add test to make sure full xpath is returned

* reformat

* clean up nesting using ternary operator

* clean up getTagSymbol conditionals

Author: seanmcguire12

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tarsier"
-version = "0.7.2"
+version = "0.8.0"
 description = "Vision utilities for web interaction agents"
 authors = ["Rohan Pandey", "Adam Watkins", "Asim Shrestha"]
 readme = "README.md"
@@ -19,14 +19,14 @@ azure-ai-vision-imageanalysis = "^1.0.0b2"
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.10.0"
 ruff = ">=0.4.8,<0.7.0"
-bananalyzer = "^0.9.17"
+bananalyzer = "0.10.8"
 
 
 [tool.poetry.group.test.dependencies]
 pytest = ">=7.4.3,<9.0.0"
 pytest-cov = ">=4.1,<6.0"
 pytest-playwright = ">=0.4.4,<0.6.0"
-pytest-asyncio = ">=0.21.2,<0.24.0"
+pytest-asyncio = ">=0.24.0,<0.25.0"
 nest-asyncio = "^1.6.0"
 webdriver-manager = "^4.0.1"
 pytest-mock = "^3.14.0"

pyproject.toml

@@ -7,11 +7,11 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.11"
-bananalyzer = "0.9.12"
+bananalyzer = "0.10.8"
 playwright = "^1.41.1"
 python-dotenv = "^1.0.1"
 tarsier = { path = "../", develop = true }
-numpy = "^1.26.3"
+numpy = "2.1.0"
 tiktoken = "^0.6.0"
 
 

tarsier-snapshots/poetry.lock

@@ -1,20 +1,34 @@
 from asyncio import Protocol
 from pathlib import Path
-from typing import Dict, Tuple
+from typing import Tuple, Optional, TypedDict
 
 from tarsier._utils import load_js
 from tarsier.adapter import AnyDriver, BrowserAdapter, adapter_factory
 from tarsier.ocr import OCRService
 from tarsier.text_format import format_text
 
-TagToXPath = Dict[int, str]
+
+class TagMetadata(TypedDict):
+    tarsier_id: int
+    element_name: str
+    opening_tag_html: str
+    xpath: str
+    element_text: Optional[str]
+    text_node_index: Optional[int]
+    id_symbol: str
+    id_string: str
 
 
 class ITarsier(Protocol):
-    async def page_to_image(self, driver: AnyDriver) -> Tuple[bytes, Dict[int, str]]:
+    async def page_to_image(self, driver: AnyDriver) -> Tuple[bytes, list[TagMetadata]]:
         raise NotImplementedError()
 
-    async def page_to_text(self, driver: AnyDriver) -> Tuple[str, Dict[int, str]]:
+    async def page_to_text(self, driver: AnyDriver) -> Tuple[str, list[TagMetadata]]:
+        raise NotImplementedError()
+
+    async def page_to_image_and_text(
+        self, driver: AnyDriver
+    ) -> Tuple[bytes, str, list[TagMetadata]]:
         raise NotImplementedError()
 
 
@@ -31,10 +45,10 @@ async def page_to_image(
         tag_text_elements: bool = False,
         tagless: bool = False,
         keep_tags_showing: bool = False,
-    ) -> Tuple[bytes, TagToXPath]:
+    ) -> Tuple[bytes, list[TagMetadata]]:
         adapter = adapter_factory(driver)
         tag_to_xpath = (
-            await self._tag_page(adapter, tag_text_elements) if not tagless else {}
+            await self._tag_page(adapter, tag_text_elements) if not tagless else []
         )
         if tagless:
             await self._remove_tags(adapter)
@@ -44,15 +58,15 @@ async def page_to_image(
         if not keep_tags_showing:
             await self._remove_tags(adapter)
 
-        return screenshot, tag_to_xpath if not tagless else {}
+        return screenshot, tag_to_xpath if not tagless else []
 
     async def page_to_text(
         self,
         driver: AnyDriver,
         tag_text_elements: bool = False,
         tagless: bool = False,
         keep_tags_showing: bool = False,
-    ) -> Tuple[str, TagToXPath]:
+    ) -> Tuple[str, list[TagMetadata]]:
         image, tag_to_xpath = await self.page_to_image(
             driver, tag_text_elements, tagless, keep_tags_showing
         )
@@ -65,7 +79,7 @@ async def page_to_image_and_text(
         tag_text_elements: bool = False,
         tagless: bool = False,
         keep_tags_showing: bool = False,
-    ) -> Tuple[bytes, str, TagToXPath]:
+    ) -> Tuple[bytes, str, list[TagMetadata]]:
         image, tag_to_xpath = await self.page_to_image(
             driver, tag_text_elements, tagless, keep_tags_showing
         )
@@ -90,13 +104,26 @@ def _run_ocr(self, image: bytes) -> str:
 
     async def _tag_page(
         self, adapter: BrowserAdapter, tag_text_elements: bool = False
-    ) -> Dict[int, str]:
+    ) -> list[TagMetadata]:
         await self._load_tarsier_utils(adapter)
 
         script = f"return window.tagifyWebpage({str(tag_text_elements).lower()});"
-        tag_to_xpath = await adapter.run_js(script)
-
-        return {int(key): value for key, value in tag_to_xpath.items()}
+        tag_to_meta = await adapter.run_js(script)
+
+        tag_metadata_list = [
+            TagMetadata(
+                tarsier_id=meta["tarsierId"],
+                element_name=meta["elementName"],
+                opening_tag_html=meta["openingTagHTML"],
+                xpath=meta["xpath"],
+                element_text=meta.get("elementText"),
+                text_node_index=meta.get("textNodeIndex"),
+                id_symbol=meta["idSymbol"],
+                id_string=meta["idString"],
+            )
+            for meta in tag_to_meta
+        ]
+        return tag_metadata_list
 
     async def _remove_tags(self, adapter: BrowserAdapter) -> None:
         await self._load_tarsier_utils(adapter)