Skip to content

Commit 2fc86ee

Browse files
jwalstra-keeperjwalstra-keeper
committed
DR-803 Update discovery-common wheel and job graph display.
Update the discovery-common wheel to fix problem where the jobs vertex is missing a DATA edge. Added ability to display job graph. pam action debug graph ID -t jobs -g GATEWAY --list pam action debug graph ID -t jobs -g GATEWAY --render
1 parent e7120b1 commit 2fc86ee

File tree

4 files changed

+156
-35
lines changed

4 files changed

+156
-35
lines changed

‎keepercommander/commands/discover/job_status.py‎

Lines changed: 34 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ def job_detail(self, job):
5050
pass
5151

5252
@staticmethod
53-
def print_job_table(jobs, max_gateway_name):
53+
def print_job_table(jobs, max_gateway_name, show_history=False):
5454

5555
print("")
5656
print(f"{bcolors.HEADER}{'Job ID'.ljust(14, ' ')} "
@@ -98,17 +98,17 @@ def print_job_table(jobs, max_gateway_name):
9898
f"{(job.get('duration') or 'NA').ljust(19, ' ')} "
9999
f"{bcolors.ENDC}")
100100

101-
if len(completed_jobs) > 0:
101+
if len(completed_jobs) > 0 and show_history is False:
102102
print("")
103103
if len(completed_jobs) == 1:
104104
print(f"There is one {_g('COMPLETED')} job. To process, use the following command.")
105105
else:
106106
print(f"There are {len(completed_jobs)} {_g('COMPLETED')} jobs. "
107-
"To process, use one of the the following command.")
107+
"To process, use one of the the following commands.")
108108
for job_id in completed_jobs:
109109
print(_g(f" pam action discover process -j {job_id}"))
110110

111-
if len(running_jobs) > 0:
111+
if len(running_jobs) > 0 and show_history is False:
112112
print("")
113113
if len(running_jobs) == 1:
114114
print(f"There is one {_b('RUNNING')} job. "
@@ -119,7 +119,7 @@ def print_job_table(jobs, max_gateway_name):
119119
for job_id in running_jobs:
120120
print(_b(f" pam action discover remove -j {job_id}"))
121121

122-
if len(failed_jobs) > 0:
122+
if len(failed_jobs) > 0 and show_history is False:
123123
print("")
124124
if len(failed_jobs) == 1:
125125
print(f"There is one {_f('FAILED')} job. "
@@ -180,31 +180,35 @@ def print_job_detail(params, gateway_context, jobs, job_id):
180180
job_item = job.get("job_item") # type: JobItem
181181

182182
try:
183-
infra.load(sync_point=job_item.sync_point)
183+
infra.load(sync_point=0)
184184
print("")
185-
delta = DiscoveryDelta.model_validate(job.get('delta'))
186-
print(f"{_h('Added')} - {len(delta.added)} count")
187-
for item in delta.added:
188-
vertex = infra.dag.get_vertex(item.uid)
189-
discovery_object = DiscoveryObject.get_discovery_object(vertex)
190-
print(f" * {discovery_object.description}")
191-
192-
print("")
193-
print(f"{_h('Changed')} - {len(delta.changed)} count")
194-
for item in delta.changed:
195-
vertex = infra.dag.get_vertex(item.uid)
196-
discovery_object = DiscoveryObject.get_discovery_object(vertex)
197-
print(f" * {discovery_object.description}")
198-
if item.changes is None:
199-
print(f" no changed, may be a object not added in prior discoveries.")
200-
else:
201-
for key, value in item.changes.items():
202-
print(f" - {key} = {value}")
203-
204-
print("")
205-
print(f"{_h('Deleted')} - {len(delta.deleted)} count")
206-
for item in delta.deleted:
207-
print(f" * discovery vertex {item.uid}")
185+
delta_json = job.get('delta')
186+
if delta_json is not None:
187+
delta = DiscoveryDelta.model_validate(delta_json)
188+
print(f"{_h('Added')} - {len(delta.added)} count")
189+
for item in delta.added:
190+
vertex = infra.dag.get_vertex(item.uid)
191+
discovery_object = DiscoveryObject.get_discovery_object(vertex)
192+
print(f" * {discovery_object.description}")
193+
194+
print("")
195+
print(f"{_h('Changed')} - {len(delta.changed)} count")
196+
for item in delta.changed:
197+
vertex = infra.dag.get_vertex(item.uid)
198+
discovery_object = DiscoveryObject.get_discovery_object(vertex)
199+
print(f" * {discovery_object.description}")
200+
if item.changes is None:
201+
print(f" no changed, may be a object not added in prior discoveries.")
202+
else:
203+
for key, value in item.changes.items():
204+
print(f" - {key} = {value}")
205+
206+
print("")
207+
print(f"{_h('Deleted')} - {len(delta.deleted)} count")
208+
for item in delta.deleted:
209+
print(f" * discovery vertex {item.uid}")
210+
else:
211+
print(f"{_f('There are no available delta changes for this job.')}")
208212

209213
except Exception as err:
210214
print(f"{_f('Could not load delta from infrastructure: ' + str(err))}")
@@ -300,4 +304,4 @@ def execute(self, params, **kwargs):
300304
if job_id is not None and gateway_context is not None:
301305
self.print_job_detail(params, gateway_context, all_jobs, job_id)
302306
else:
303-
self.print_job_table(all_jobs, max_gateway_name)
307+
self.print_job_table(all_jobs, max_gateway_name, show_history)

‎keepercommander/commands/pam_debug/graph.py‎

Lines changed: 121 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,12 @@
88
from discovery_common.infrastructure import Infrastructure
99
from discovery_common.record_link import RecordLink
1010
from discovery_common.user_service import UserService
11+
from discovery_common.jobs import Jobs
1112
from discovery_common.constants import (PAM_USER, PAM_DIRECTORY, PAM_MACHINE, PAM_DATABASE, VERTICES_SORT_MAP,
12-
DIS_INFRA_GRAPH_ID, RECORD_LINK_GRAPH_ID, USER_SERVICE_GRAPH_ID)
13+
DIS_INFRA_GRAPH_ID, RECORD_LINK_GRAPH_ID, USER_SERVICE_GRAPH_ID,
14+
DIS_JOBS_GRAPH_ID)
1315
from discovery_common.types import (DiscoveryObject, DiscoveryUser, DiscoveryDirectory, DiscoveryMachine,
14-
DiscoveryDatabase)
16+
DiscoveryDatabase, JobContent)
1517
from discovery_common.dag_sort import sort_infra_vertices
1618
from keeper_dag import DAG
1719
from keeper_dag.connection.commander import Connection as CommanderConnection
@@ -32,7 +34,7 @@ class PAMDebugGraphCommand(PAMGatewayActionDiscoverCommandBase):
3234
# The record to base everything on.
3335
parser.add_argument('--gateway', '-g', required=True, dest='gateway', action='store',
3436
help='Gateway name or UID.')
35-
parser.add_argument('--type', '-t', required=True, choices=['infra', 'rl', 'service'],
37+
parser.add_argument('--type', '-t', required=True, choices=['infra', 'rl', 'service', 'jobs'],
3638
dest='graph_type', action='store', help='Graph type', default='infra')
3739
parser.add_argument('--raw', required=False, dest='raw', action='store_true',
3840
help='Render raw graph. Will render corrupt graphs.')
@@ -59,7 +61,8 @@ class PAMDebugGraphCommand(PAMGatewayActionDiscoverCommandBase):
5961
graph_id_map = {
6062
"infra": DIS_INFRA_GRAPH_ID,
6163
"rl": RECORD_LINK_GRAPH_ID,
62-
"service": USER_SERVICE_GRAPH_ID
64+
"service": USER_SERVICE_GRAPH_ID,
65+
"jobs": DIS_JOBS_GRAPH_ID
6366
}
6467

6568
def get_parser(self):
@@ -283,6 +286,102 @@ def _handle(current_vertex: DAGVertex, parent_vertex: Optional[DAGVertex] = None
283286

284287
_handle(current_vertex=configuration, parent_vertex=None, indent=indent)
285288

289+
def _do_text_list_jobs(self, params: KeeperParams, gateway_context: GatewayContext, debug_level: int = 0,
290+
indent: int = 0):
291+
292+
infra = Infrastructure(record=gateway_context.configuration, params=params, logger=logging,
293+
debug_level=debug_level, fail_on_corrupt=False)
294+
infra.load(sync_point=0)
295+
296+
pad = ""
297+
if indent > 0:
298+
pad = "".ljust(2 * indent, ' ') + "* "
299+
300+
conn = get_connection(params)
301+
graph_sync = DAG(conn=conn, record=gateway_context.configuration, logger=logging, debug_level=debug_level,
302+
graph_id=DIS_JOBS_GRAPH_ID)
303+
graph_sync.load(0)
304+
configuration = graph_sync.get_root
305+
vertices = configuration.has_vertices()
306+
if len(vertices) == 0:
307+
print(self._f(f"The jobs graph has not been initialized. Only has root vertex."))
308+
return
309+
310+
vertex = vertices[0]
311+
if vertex.has_data is False:
312+
print(self._f(f"The job vertex does not contain any data"))
313+
return
314+
315+
current_json = vertex.content_as_str
316+
if current_json is None:
317+
print(self._f(f"The current job vertex content is None"))
318+
return
319+
320+
content = JobContent.model_validate_json(current_json)
321+
print(f"{pad}{self._b('Active Job ID')}: {content.active_job_id}")
322+
print("")
323+
print(f"{pad}{self._h('History')}")
324+
print("")
325+
for job in content.job_history:
326+
print(f"{pad} --------------------------------------")
327+
print(f"{pad} Job Id: {job.job_id}")
328+
print(f"{pad} Started: {job.start_ts_str}")
329+
print(f"{pad} Ended: {job.end_ts_str}")
330+
print(f"{pad} Duration: {job.duration_sec_str}")
331+
print(f"{pad} Infra Sync Point: {job.sync_point}")
332+
if job.success is True:
333+
print(f"{pad} Status: {self._gr('Success')}")
334+
else:
335+
print(f"{pad} Status: {self._f('Fail')}")
336+
if job.error is not None:
337+
print(f"{pad} Error: {self._gr(job.error)}")
338+
339+
print("")
340+
341+
if job.delta is None:
342+
print(f"{pad}{self._f('The job is missing a delta, never finished discovery.')}")
343+
else:
344+
if len(job.delta.added) > 0:
345+
print(f"{pad} {self._h('Added')}")
346+
for added in job.delta.added:
347+
vertex = infra.dag.get_vertex(added.uid)
348+
if vertex is None:
349+
print(f"{pad} * Vertex {added.uid} does not exists.")
350+
else:
351+
if vertex.active is False:
352+
print(f"{pad} * Vertex {added.uid} is inactive.")
353+
elif vertex.corrupt is True:
354+
print(f"{pad} * Vertex {added.uid} is corrupt.")
355+
else:
356+
content = DiscoveryObject.get_discovery_object(vertex)
357+
print(f"{pad} * {content.description}; Record UID: {content.record_uid}")
358+
print("")
359+
360+
if len(job.delta.changed) > 0:
361+
print(f"{pad} {self._h('Changed')}")
362+
for changed in job.delta.changed:
363+
vertex = infra.dag.get_vertex(changed.uid)
364+
if vertex is None:
365+
print(f"{pad} * Vertex {changed.uid} does not exists.")
366+
else:
367+
if vertex.active is False:
368+
print(f"{pad} * Vertex {changed.uid} is inactive.")
369+
elif vertex.corrupt is True:
370+
print(f"{pad} * Vertex {changed.uid} is corrupt.")
371+
else:
372+
content = DiscoveryObject.get_discovery_object(vertex)
373+
print(f"{pad} * {content.description}; Record UID: {content.record_uid}")
374+
if changed.changes is not None:
375+
for k, v in changed.changes.items():
376+
print(f"{pad} {k} = {v}")
377+
print("")
378+
379+
if len(job.delta.deleted) > 0:
380+
print(f"{pad} {self._h('Deleted')}")
381+
for deleted in job.delta.deleted:
382+
print(f"{pad} * Removed vertex {deleted.uid}.")
383+
print("")
384+
286385
def _do_render_infra(self, params: KeeperParams, gateway_context: GatewayContext, filepath: str, graph_format: str,
287386
debug_level: int = 0):
288387

@@ -352,6 +451,24 @@ def _do_render_service(self, params: KeeperParams, gateway_context: GatewayConte
352451
raise err
353452
print("")
354453

454+
def _do_render_jobs(self, params: KeeperParams, gateway_context: GatewayContext, filepath: str,
455+
graph_format: str, debug_level: int = 0):
456+
457+
jobs = Jobs(record=gateway_context.configuration, params=params, logger=logging, debug_level=debug_level)
458+
459+
print("")
460+
dot_instance = jobs.dag.to_dot()
461+
if graph_format == "raw":
462+
print(dot_instance)
463+
else:
464+
try:
465+
dot_instance.render(filepath)
466+
print(f"Job graph rendered to {self._gr(filepath)}")
467+
except Exception as err:
468+
print(self._f(f"Could not generate graph: {err}"))
469+
raise err
470+
print("")
471+
355472
def _do_raw_text_list(self, params: KeeperParams, gateway_context: GatewayContext, graph_id: int = 0,
356473
debug_level: int = 0):
357474

‎libs/discovery_common-1.0.18-py3-none-any.whl‎ renamed to ‎libs/discovery_common-1.0.19-py3-none-any.whl‎

39 KB
Binary file not shown.

‎requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,4 @@ pydantic>=2.6.4
2525
# pip uninstall discovery-common -y
2626
# python3 setup.py wheel --whlsrc ~/src/discovery-common --libdir $PWD/libs --reqfiles $PWD/requirements.txt
2727
# pip install $(ls libs/discovery_common-*)
28-
./libs/discovery_common-1.0.18-py3-none-any.whl
28+
./libs/discovery_common-1.0.19-py3-none-any.whl

0 commit comments

Comments
 (0)