Skip to content

Use action coord to help Element path exact match #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Use action coord to help Element path exact match #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 21 additions & 27 deletions SEEACT/src/evaluate/step_score.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,26 @@ class ElementEvaluator(StepEvaluator):
'''Element evaluation and scoring'''

@staticmethod
async def is_same_element(page, input_element_handle, reference_element_handle):
is_same_element = await page.evaluate(
"(elements) => elements[0] === elements[1]",
[input_element_handle, reference_element_handle])
return int(is_same_element)
def is_same_element(page, input_coord, reference_element_handle):
x,y=input_coord
# Get the bounding box of the element, usually 2s is enough, but set to 5s here
bounding_box = reference_element_handle.bounding_box(timeout=5000)
if bounding_box:
element_x = bounding_box['x']
element_y = bounding_box['y']
element_width = bounding_box['width']
element_height = bounding_box['height']
# Check if the given (x, y) is within the bounding box
if (element_x <= x <= element_x + element_width and
element_y <= y <= element_y + element_height):
return True
return False


@ staticmethod
async def path_exact_match(input_answer, reference_answer, method, page, input_netloc, reference_netloc):
async def path_exact_match(input_answer, reference_answer, method, page, input_netloc, reference_netloc,input_coords=None):
# input_coords should be (x,y) in pixels, if not None
# and will be used in ElementEvaluator.path_exact_match()
score = 0
if method == "xpath":
if reference_netloc != input_netloc:
Expand Down Expand Up @@ -129,32 +141,14 @@ async def path_exact_match(input_answer, reference_answer, method, page, input_n
pass
else:
score = 0
elif method == "selector":
elif method == "selector": #modified to use coords
if reference_netloc != input_netloc:
return 0
try:
input_element = input_answer
input_element = input_coords#input element is input coord
reference_element = page.locator(reference_answer)
input_element_handle = await input_element.element_handle()
reference_element_handle = await reference_element.element_handle()
if (input_element is not None) and (reference_element is not None):
score = await ElementEvaluator().is_same_element(page, input_element_handle=input_element_handle, reference_element_handle=reference_element_handle)
try:
reference_tag = await page.evaluate("(element) => element.tagName.toLowerCase()", reference_element_handle)
if reference_tag in MapTagNameList:
trace_up_count = 0
current_element = reference_element
while trace_up_count < 3 and score == 0:
trace_up_count += 1
parent_element = current_element.locator("xpath=..")
parent_element_handle = await parent_element.element_handle()
current_element = parent_element
if parent_element:
parent_score = await ElementEvaluator().is_same_element(page, input_element_handle=input_element_handle, reference_element_handle=parent_element_handle)
score = max(score, parent_score)
except Exception as e:
print(e)
pass
score = ElementEvaluator().is_same_element(page, input_coord=input_element, reference_element_handle=reference_element)
except:
score = 0
# result_score = MatchFunction.include_match(
Expand Down
22 changes: 12 additions & 10 deletions SEEACT/src/evaluate/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,9 @@ def get_netloc(url: str) -> str:
# return evaluate_steps, match_result


async def step_evaluate(page: Page, evaluate_steps=[], input_path=None, element_value=None):
async def step_evaluate(page: Page, evaluate_steps=[], input_path=None, element_value=None,input_coords=None):\
# input_coords should be (x,y) in pixels, if not None
# and will be used in ElementEvaluator.path_exact_match()
"""Evaluate step score"""
step_score = 0
match_result = []
Expand All @@ -213,19 +215,19 @@ async def step_evaluate(page: Page, evaluate_steps=[], input_path=None, element_
method = evaluate["method"]
score = await ElementEvaluator.path_exact_match(
input_path, evaluate["reference_answer"], method, page, input_netloc,
evaluate["netloc"])
evaluate["netloc"],input_coords=input_coords)

elif match_function == "element_path_included_match":
pass

elif match_function == "element_value_exactly_match":
if input_path is not None and element_value is not None:
if (input_path is not None or input_coords is not None) and element_value is not None:
input_netloc = get_netloc(page.url)

if "path" in evaluate.keys():
path_score = await ElementEvaluator.path_exact_match(input_path, evaluate["path"], "selector",
page, input_netloc,
evaluate["netloc"])
evaluate["netloc"],input_coords=input_coords)
if path_score == 0:
score = 0
else:
Expand All @@ -238,12 +240,12 @@ async def step_evaluate(page: Page, evaluate_steps=[], input_path=None, element_
else:
score = 0
elif match_function == "element_value_included_match":
if input_path is not None and element_value is not None:
if (input_path is not None or input_coords is not None) and element_value is not None:
input_netloc = get_netloc(page.url)
if "path" in evaluate.keys():
path_score = await ElementEvaluator.path_exact_match(input_path, evaluate["path"], "selector",
page, input_netloc,
evaluate["netloc"])
evaluate["netloc"],input_coords=input_coords)
if path_score == 0:
score = 0
else:
Expand All @@ -255,14 +257,14 @@ async def step_evaluate(page: Page, evaluate_steps=[], input_path=None, element_
else:
score = 0
elif match_function == "element_value_semantic_match":
if input_path is not None and element_value is not None:
if (input_path is not None or input_coords is not None) and element_value is not None:
input_netloc = get_netloc(page.url)

if len(element_value) > 0:
if "path" in evaluate.keys():
path_score = await ElementEvaluator.path_exact_match(input_path, evaluate["path"], "selector",
page, input_netloc,
evaluate["netloc"])
evaluate["netloc"],input_coords=input_coords)
if path_score == 0:
# print("Path mismatch in value evaluation")
score = 0
Expand Down Expand Up @@ -313,14 +315,14 @@ def extract_element_value(html_content, selector):
return element_value


async def evaluate_with_webcanvas(page, selector, target_value, evaluate_steps, reference_evaluate_steps):
async def evaluate_with_webcanvas(page, selector, target_value, evaluate_steps, reference_evaluate_steps,input_coords=None):# input_coords should be (x,y) in pixels, if not None
element_value = ""
if target_value != "None":
element_value = target_value
else:
element_value = await selector.text_content()
evaluate_steps, match_result = await step_evaluate(page=page, evaluate_steps=evaluate_steps,
input_path=selector, element_value=element_value)
input_path=selector, element_value=element_value,input_coords=input_coords)#input coords is used to replace selector in path exact match
total_step_score = 0
for evaluate in evaluate_steps:
total_step_score += evaluate["score"]
Expand Down