Skip to content

Commit 8eb956b

Browse files
JessamyTnpentrel
JessamyT
authored and
npentrel
committed
Some broken changes to parse_go and update_sdk_methods
1 parent a7e2c66 commit 8eb956b

File tree

2 files changed

+121
-36
lines changed

2 files changed

+121
-36
lines changed

‎.github/workflows/parse_go.py

Lines changed: 100 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@
1515
"power_sensor": "powersensor",
1616
"generic_service": "generic",
1717
"base_remote_control": "baseremotecontrol",
18-
"data_manager": "datamanager"
18+
"data_manager": "datamanager",
19+
"data": "DataClient",
20+
"dataset": "DataClient"
1921
}
2022

2123
## Ignore these specific APIs if they error, are deprecated, etc:
@@ -85,7 +87,8 @@ def parse(self, type, viam_resources):
8587
elif type == "robot":
8688
url = f"{self.scrape_url}/go.viam.com/rdk/{type}"
8789
elif type == "app":
88-
continue
90+
url = f"{self.scrape_url}/go.viam.com/rdk/{type}"
91+
print(url)
8992

9093
self.go_methods[type][resource] = {}
9194

@@ -99,36 +102,107 @@ def parse(self, type, viam_resources):
99102
soup = make_soup(url)
100103

101104
## Get a raw dump of all go methods by interface for each resource:
102-
go_methods_raw = soup.find_all(
103-
lambda tag: tag.name == 'div'
104-
and tag.get('class') == ['Documentation-declaration']
105-
and tag.pre.text.startswith('type')
106-
and "interface {" in tag.pre.text)
105+
if type == "app":
106+
## For app resources, look for client type definitions instead of interfaces
107+
go_methods_raw = soup.find_all(
108+
lambda tag: tag.name == 'div'
109+
and tag.get('class') == ['Documentation-declaration']
110+
and tag.pre.text.startswith('func (')
111+
and any(client in tag.pre.text for client in ["DataClient", "AppClient", "BillingClient", "MLTrainingClient"]))
112+
else:
113+
go_methods_raw = soup.find_all(
114+
lambda tag: tag.name == 'div'
115+
and tag.get('class') == ['Documentation-declaration']
116+
and tag.pre.text.startswith('type')
117+
and "interface {" in tag.pre.text)
118+
119+
if type == "app":
120+
## For app resources, process function definitions directly
121+
for func_div in go_methods_raw:
122+
## Extract method name from function definition
123+
func_text = func_div.find('pre').text
124+
if 'DataClient' in func_text:
125+
method_name = func_text.split(') ')[1].split('(')[0]
126+
127+
## Create new empty dictionary for this specific method
128+
this_method_dict = {}
107129

108-
# some resources have more than one interface:
109-
for resource_interface in go_methods_raw:
130+
## Check if this method is mapped to the current resource in the CSV file
131+
method_mapped_to_resource = False
132+
with open(self.proto_map_file, 'r') as f:
133+
for row in f:
134+
if not row.startswith('#') \
135+
and row.startswith(resource + ',') \
136+
and row.split(',')[4] == method_name:
137+
method_mapped_to_resource = True
138+
break
139+
140+
## Only include methods that are mapped to this resource
141+
if method_mapped_to_resource:
142+
143+
## Exclude unwanted Go methods:
144+
check_method_name = resource + '.' + method_name
145+
if not check_method_name in go_ignore_apis:
146+
147+
## Debug: Print found methods for app resources
148+
if type == "app":
149+
print(f"Found method: {method_name} for resource: {resource}")
150+
151+
## Look up method_name in proto_map file, and return matching proto:
152+
with open(self.proto_map_file, 'r') as f:
153+
for row in f:
154+
if not row.startswith('#') \
155+
and row.startswith(resource + ',') \
156+
and row.split(',')[4] == method_name:
157+
this_method_dict["proto"] = row.split(',')[1]
158+
159+
## Extract method description
160+
method_description = ""
161+
if func_div.find('p'):
162+
method_description = func_div.find('p').text.replace("\n", " ")
163+
this_method_dict["description"] = method_description
110164

111-
## Determine the interface name, which we need for the method_link:
112-
interface_name = resource_interface.find('pre').text.splitlines()[0].removeprefix('type ').removesuffix(' interface {')
165+
## Extract method usage
166+
this_method_dict["usage"] = func_text.replace("\t", " ").lstrip().rstrip()
113167

114-
## Exclude unwanted Go interfaces:
115-
check_interface_name = 'interface.' + interface_name
116-
if not check_interface_name in go_ignore_apis:
168+
## Set method link
169+
if self.staging:
170+
this_method_dict["method_link"] = str(url + '#DataClient.' + method_name).replace(self.scrape_url, 'https://pkg.go.dev')
171+
else:
172+
this_method_dict["method_link"] = url + '#DataClient.' + method_name
117173

118-
## Loop through each method found for this interface:
119-
for tag in resource_interface.find_all('span', attrs={"data-kind" : "method"}):
174+
## Store the method
175+
self.go_methods[type][resource][method_name] = this_method_dict
120176

121-
## Create new empty dictionary for this specific method, to be appended to ongoing go_methods dictionary,
122-
## in form: go_methods[type][resource][method_name] = this_method_dict
123-
this_method_dict = {}
177+
else:
178+
# some resources have more than one interface:
179+
for resource_interface in go_methods_raw:
180+
181+
## Determine the interface name, which we need for the method_link:
182+
interface_name = resource_interface.find('pre').text.splitlines()[0].removeprefix('type ').removesuffix(' interface {')
183+
184+
## Exclude unwanted Go interfaces:
185+
check_interface_name = 'interface.' + interface_name
186+
if not check_interface_name in go_ignore_apis:
187+
188+
## Loop through each method found for this interface:
189+
for tag in resource_interface.find_all('span', attrs={"data-kind" : "method"}):
124190

125-
tag_id = tag.get('id')
126-
method_name = tag.get('id').split('.')[1]
191+
## Create new empty dictionary for this specific method, to be appended to ongoing go_methods dictionary,
192+
## in form: go_methods[type][resource][method_name] = this_method_dict
193+
this_method_dict = {}
194+
195+
tag_id = tag.get('id')
196+
method_name = tag.get('id').split('.')[1]
127197

128198
## Exclude unwanted Go methods:
129199
check_method_name = resource + '.' + method_name
130200
if not check_method_name in go_ignore_apis:
131201

202+
## Debug: Print found methods for app resources
203+
if type == "app":
204+
print(f"Found method: {method_name} for resource: {resource}")
205+
132206
## Look up method_name in proto_map file, and return matching proto:
133207
with open(self.proto_map_file, 'r') as f:
134208
for row in f:
@@ -141,7 +215,11 @@ def parse(self, type, viam_resources):
141215
## Split by method span, throwing out remainder of span tag, catching cases where
142216
## id is first attr or data-kind is first attr, and slicing to omit the first match,
143217
## which is the opening of the method span tag, not needed:
144-
this_method_raw1 = regex.split(r'id="' + tag_id + '"', str(resource_interface))[1].removeprefix('>').removeprefix(' data-kind="method">').lstrip()
218+
split_result = regex.split(r'id="' + tag_id + '"', str(resource_interface))
219+
if len(split_result) > 1:
220+
this_method_raw1 = split_result[1].removeprefix('>').removeprefix(' data-kind="method">').lstrip()
221+
else:
222+
continue
145223

146224
## Then, omit all text that begins a new method span, and additionally remove trailing
147225
## element closers for earlier tags we spliced into (pre and span):
@@ -347,9 +425,5 @@ def parse(self, type, viam_resources):
347425
if resource == "generic_service":
348426
self.go_methods[type][resource]['DoCommand']['code_sample'] = 'myGenericService, err := generic.FromRobot(machine, "my_generic_service")\n\ncommand := map[string]interface{}{"cmd": "test", "data1": 500}\nresult, err := myGenericService.DoCommand(context.Background(), command)\n'
349427

350-
# elif type == "app":
351-
# ##Go SDK has no APP API!
352-
# pass
353-
354428
return self.go_methods
355429

‎.github/workflows/update_sdk_methods.py

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,11 @@ def parse_method_usage(usage_string):
508508
## resulting in trailing ',\n)' syntax that is otherwise interpreted here as a param named ')':
509509
if param != ')':
510510

511+
## Initialize variables
512+
type_link = None
513+
type_name = ""
514+
param_type = ""
515+
511516
## HACKY: Hardcoding several params and returns. These are: the ctx, extra, and cmd params,
512517
## the error return, and several parameters that require type inference.
513518
if param.startswith('ctx'):
@@ -537,7 +542,8 @@ def parse_method_usage(usage_string):
537542
elif len(param_raw) == 2:
538543
type_name = param_raw[0]
539544
param_type = param_raw[1]
540-
type_link = regex.findall(r'href="([^"]+)">', param)[-1]
545+
links = regex.findall(r'href="([^"]+)">', param)
546+
type_link = links[-1] if links else None
541547
## Handle returns, or parameters with inferred data types:
542548
elif len(param_raw) == 1:
543549
## Hardcode for type inference for angleDeg param:
@@ -569,7 +575,8 @@ def parse_method_usage(usage_string):
569575
else:
570576
type_name = ''
571577
param_type = param_raw[0]
572-
type_link = regex.findall(r'href="([^"]+)">', param)[-1]
578+
links = regex.findall(r'href="([^"]+)">', param)
579+
type_link = links[-1] if links else None
573580

574581
if type_link:
575582
param_type_link = type_link
@@ -819,14 +826,18 @@ def write_markdown(type, names, methods):
819826
flutter_method_name = row.split(',')[5].rstrip()
820827
typescript_method_name = row.split(',')[6].rstrip()
821828

822-
if py_method_name and "python" in sdks:
823-
methods['python'][type][resource][py_method_name]["used"] = True
824-
if go_method_name and "go" in sdks:
825-
methods['go'][type][resource][go_method_name]["used"] = True
826-
if flutter_method_name and "flutter" in sdks:
827-
methods['flutter'][type][resource][flutter_method_name]["used"] = True
828-
if typescript_method_name and "typescript" in sdks:
829-
methods['typescript'][type][resource][typescript_method_name]["used"] = True
829+
if py_method_name and "python" in sdks and resource in methods['python'][type]:
830+
if py_method_name in methods['python'][type][resource]:
831+
methods['python'][type][resource][py_method_name]["used"] = True
832+
if go_method_name and "go" in sdks and resource in methods['go'][type]:
833+
if go_method_name in methods['go'][type][resource]:
834+
methods['go'][type][resource][go_method_name]["used"] = True
835+
if flutter_method_name and "flutter" in sdks and resource in methods['flutter'][type]:
836+
if flutter_method_name in methods['flutter'][type][resource]:
837+
methods['flutter'][type][resource][flutter_method_name]["used"] = True
838+
if typescript_method_name and "typescript" in sdks and resource in methods['typescript'][type]:
839+
if typescript_method_name in methods['typescript'][type][resource]:
840+
methods['typescript'][type][resource][typescript_method_name]["used"] = True
830841

831842
## Allow setting protos with 0 sdk method maps, to allow us to disable writing MD
832843
## for specific protos as needed, if needed:

0 commit comments

Comments
 (0)