From 8409c89cc3872a237ab5c36ae85d8425ab601c5f Mon Sep 17 00:00:00 2001 From: John Walstra Date: Mon, 6 Jan 2025 10:22:18 -0600 Subject: [PATCH] DR-857 Update the prog name for commands The project names were incorrect. Changed them to the command menu path. The discovery_common code was not update to date. Update it. --- keepercommander/commands/discover/debug.py | 268 ------------------ .../commands/discover/job_remove.py | 2 +- .../commands/discover/job_start.py | 2 +- .../commands/discover/job_status.py | 2 +- .../commands/discover/result_get.py | 56 ---- .../commands/discover/result_process.py | 2 +- keepercommander/commands/discover/rule_add.py | 2 +- .../commands/discover/rule_list.py | 2 +- .../commands/discover/rule_remove.py | 2 +- .../commands/discover/rule_update.py | 2 +- keepercommander/commands/discoveryrotation.py | 9 +- keepercommander/commands/pam_debug/acl.py | 2 +- keepercommander/commands/pam_debug/alter.py | 23 -- keepercommander/commands/pam_debug/gateway.py | 2 +- keepercommander/commands/pam_debug/graph.py | 2 +- keepercommander/commands/pam_debug/info.py | 2 +- keepercommander/commands/pam_debug/verify.py | 2 +- keepercommander/commands/pam_debug/version.py | 20 -- keepercommander/discovery_common/constants.py | 1 - .../discovery_common/infrastructure.py | 14 +- keepercommander/discovery_common/jobs.py | 5 +- keepercommander/discovery_common/process.py | 5 +- .../discovery_common/record_link.py | 5 +- keepercommander/discovery_common/types.py | 18 +- .../discovery_common/user_service.py | 8 +- keepercommander/discovery_common/utils.py | 15 +- keepercommander/keeper_dag/README.md | 9 + 27 files changed, 68 insertions(+), 414 deletions(-) delete mode 100644 keepercommander/commands/discover/debug.py delete mode 100644 keepercommander/commands/discover/result_get.py delete mode 100644 keepercommander/commands/pam_debug/alter.py delete mode 100644 keepercommander/commands/pam_debug/version.py create mode 100644 keepercommander/keeper_dag/README.md diff --git a/keepercommander/commands/discover/debug.py b/keepercommander/commands/discover/debug.py deleted file mode 100644 index d20f5efb4..000000000 --- a/keepercommander/commands/discover/debug.py +++ /dev/null @@ -1,268 +0,0 @@ -from __future__ import annotations -import argparse -import os -from . import PAMGatewayActionDiscoverCommandBase -from ...display import bcolors -from ... import vault -from keepercommander.discovery_common.infrastructure import Infrastructure -from keepercommander.discovery_common.record_link import RecordLink -from keepercommander.discovery_common.types import UserAcl, DiscoveryObject -from keepercommander.keeper_dag import EdgeType -from importlib.metadata import version -from typing import Optional, TYPE_CHECKING - -if TYPE_CHECKING: - from ...vault import TypedRecord - from ...params import KeeperParams - - -class PAMGatewayActionDiscoverDebugCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-discover-command-debug') - - # The record to base everything on. - parser.add_argument('--record-uid', '-i', required=False, dest='record_uid', action='store', - help='Keeper record UID.') - - # What to do - parser.add_argument('--info', required=False, dest='info_flag', - action='store_true', help='Display information about the record.') - parser.add_argument('--belongs-to', required=False, dest='belongs_to_flag', - action='store_true', help='Connect the record to the parent record.') - parser.add_argument('--disconnect', required=False, dest='disconnect_flag', - action='store_true', help='Disconnect the record to the parent record.') - parser.add_argument('--render', required=False, dest='render_flag', action='store_true', - help='Render graphs.') - parser.add_argument('--version', required=False, dest='version_flag', action='store_true', - help='Get module versions.') - - # For --belongs-to and --disconnect - parser.add_argument('--parent-record-uid', '-p', required=False, dest='parent_record_uid', - action='store', help='The parent record UID.') - - # For the info command - parser.add_argument('--render-all-edges', required=False, dest='render_all_edges', - action='store_false', help='Render graphs.') - parser.add_argument('--graph-dir', required=False, dest='graph_dir', action='store', - help='Directory to save graphs.') - parser.add_argument('--infra-graph-name', required=False, dest='infra_name', action='store', - default="infra_graph", help='Infrastructure graph name.') - parser.add_argument('--rl-graph-name', required=False, dest='rl_name', action='store', - default="record_linking_graph", help='Record linking graph name.') - parser.add_argument('--graph-type', '-gt', required=False, choices=['dot', 'twopi', 'patchwork'], - dest='graph_type', default="dot", action='store', help='The render graph type.') - - def get_parser(self): - return PAMGatewayActionDiscoverDebugCommand.parser - - @staticmethod - def _versions(): - print("") - print(f"{bcolors.BOLD}keeper-dag version:{bcolors.ENDC} {version('keeper-dag')}") - print(f"{bcolors.BOLD}discovery-common version:{bcolors.ENDC} {version('discovery-common')}") - print("") - - @staticmethod - def _show_info(params: KeeperParams, configuration_record: TypedRecord, record: TypedRecord): - - infra = Infrastructure(record=configuration_record, params=params) - record_link = RecordLink(record=configuration_record, params=params) - - print("") - print(f"{bcolors.BOLD}Configuration UID:{bcolors.ENDC} {configuration_record.record_uid}") - print(f"{bcolors.BOLD}Configuration Key Bytes Hex:{bcolors.ENDC} {configuration_record.record_key.hex()}") - print("") - try: - discovery_vertices = infra.dag.search_content({"record_uid": record.record_uid}) - if len(discovery_vertices) > 0: - - if len(discovery_vertices) > 1: - print(f"{bcolors.FAIL}Found multiple vertices with the record UID of " - f"{record.record_uid}{bcolors.ENDC}") - for vertex in discovery_vertices: - print(f" * Infrastructure Vertex UID: {vertex.uid}") - print("") - - discovery_vertex = discovery_vertices[0] - content = DiscoveryObject.get_discovery_object(discovery_vertex) - - print(f"{bcolors.HEADER}Discovery Object Information{bcolors.ENDC}") - print(f"Vertex UID: {content.uid}") - print(f"Record UID: {content.record_uid}") - print(f"Parent Record UID: {content.parent_record_uid}") - print(f"Shared Folder UID: {content.shared_folder_uid}") - print(f"Record Type: {content.record_type}") - print(f"Object Type: {content.object_type_value}") - print(f"Ignore Object: {content.ignore_object}") - print(f"Rule Engine Result: {content.action_rules_result}") - print(f"Discovery ID: {content.id}") - print(f"Discovery Name: {content.name}") - print(f"Discovery Title: {content.title}") - print(f"Discovery Description: {content.description}") - print(f"Discovery Notes:") - for note in content.notes: - print(f" * {note}") - if content.error is not None: - print(f"{bcolors.FAIL}Error: {content.error}{bcolors.ENDC}") - if content.stacktrace is not None: - print(f"{bcolors.FAIL}Stack Trace:{bcolors.ENDC}") - print(f"{bcolors.FAIL}{content.stacktrace}{bcolors.ENDC}") - print("") - print(f"{bcolors.HEADER}Record Type Specifics{bcolors.ENDC}") - - item_dict = content.item - for k, v in item_dict.__dict__.items(): - print(f"{k} = {v}") - - print("") - print(f"{bcolors.HEADER}Belongs To Vertices{bcolors.ENDC}") - vertices = discovery_vertex.belongs_to_vertices() - for vertex in vertices: - content = DiscoveryObject.get_discovery_object(vertex) - print(f" * {content.description} ({vertex.uid})") - for edge_type in [EdgeType.LINK, EdgeType.ACL, EdgeType.KEY, EdgeType.DELETION]: - edge = discovery_vertex.get_edge(vertex, edge_type=edge_type) - if edge is not None: - print(f" . {edge_type}, active: {edge.active}") - - if len(vertices) == 0: - print(f"{bcolors.FAIL} Does not belong to anyone{bcolors.ENDC}") - - print("") - print(f"{bcolors.HEADER}Vertices Belonging To{bcolors.ENDC}") - vertices = discovery_vertex.has_vertices() - for vertex in vertices: - content = DiscoveryObject.get_discovery_object(vertex) - print(f" * {content.description} ({vertex.uid})") - for edge_type in [EdgeType.LINK, EdgeType.ACL, EdgeType.KEY, EdgeType.DELETION]: - edge = vertex.get_edge(discovery_vertex, edge_type=edge_type) - if edge is not None: - print(f" . {edge_type}, active: {edge.active}") - if len(vertices) == 0: - print(f" Does not have any children.") - - print("") - else: - print(f"{bcolors.FAIL}Could not find infrastructure vertex.{bcolors.ENDC}") - except Exception as err: - print(f"{bcolors.FAIL}Could not get information on infrastructure: {err}{bcolors.ENDC}") - - record_vertex = record_link.dag.get_vertex(record.record_uid) - if record_vertex is not None: - print(f"{bcolors.HEADER}Record Linking{bcolors.ENDC}") - for parent_vertex in record_vertex.belongs_to_vertices(): - - description = "Unknown" - discovery_vertices = infra.dag.search_content({"record_uid": parent_vertex.uid}) - if len(discovery_vertices) > 0: - content = DiscoveryObject.get_discovery_object(discovery_vertices[0]) - description = content.description - acl_edge = record_vertex.get_edge(parent_vertex, EdgeType.ACL) - if acl_edge is not None: - acl_content = acl_edge.content_as_object(UserAcl) - print(f" * ACL to {description} ({parent_vertex.uid})") - print(f" . belongs_to = {acl_content.belongs_to}") - print(f" . is_admin = {acl_content.is_admin}") - link_edge = record_vertex.get_edge(parent_vertex, EdgeType.LINK) - if link_edge is not None: - print(f" * LINK to {description} ({parent_vertex.uid})") - else: - print(f"{bcolors.FAIL}Cannot find in record linking.{bcolors.ENDC}") - - @staticmethod - def _render(params: KeeperParams, - configuration_record: TypedRecord, - infra_name: str = "infra_name", rl_name: str = "record_link_graph", - graph_type: str = "dot", graph_dir: str = None, render_all_edges: bool = False): - - if graph_dir is None: - graph_dir = os.environ.get("HOME", os.environ.get("PROFILENAME", ".")) - - print(f"Loading graphs for controller {configuration_record.record_uid}.") - - infra = Infrastructure(record=configuration_record, params=params) - record_link = RecordLink(record=configuration_record, params=params) - - print("") - try: - filename = os.path.join(graph_dir, f"{infra_name}.dot") - infra.to_dot( - graph_type=graph_type, - show_only_active_vertices=False, - show_only_active_edges=render_all_edges - ).render(filename) - print(f"Infrastructure graph rendered to {filename}") - except Exception as err: - print(f"{bcolors.FAIL}Could not generate infrastructure graph: {err}{bcolors.ENDC}") - raise err - - try: - filename = os.path.join(graph_dir, f"{rl_name}.dot") - record_link.to_dot( - graph_type=graph_type, - show_only_active_vertices=False, - show_only_active_edges=render_all_edges - ).render(filename) - print(f"Record linking graph rendered to {filename}") - except Exception as err: - print(f"{bcolors.FAIL}Could not generate record linking graph: {err}{bcolors.ENDC}") - raise err - - filename = os.path.join(graph_dir, f"infra_raw.dot") - with open(filename, "w") as fh: - fh.write(str(infra.dag.to_dot())) - fh.close() - - filename = os.path.join(graph_dir, f"record_linking_raw.dot") - with open(filename, "w") as fh: - fh.write(str(record_link.dag.to_dot())) - fh.close() - - def execute(self, params, **kwargs): - - info_flag = kwargs.get("info_flag", False) - belongs_to_flag = kwargs.get("belongs_to_flag", False) - disconnect_flag = kwargs.get("disconnect_flag", False) - render_flag = kwargs.get("render_flag", False) - version_flag = kwargs.get("version_flag", False) - - record_uid = kwargs.get("record_uid") - configuration_record = None - if record_uid is not None: - record = vault.KeeperRecord.load(params, record_uid) # type: Optional[TypedRecord] - if record is None: - print(f"{bcolors.FAIL}Record does not exists.{bcolors.ENDC}") - return - - configuration_record = record - if record.record_type in ["pamUser", "pamMachine", "pamDatabase", "pamDirectory"]: - record_rotation = params.record_rotation_cache.get(record_uid) - if record_rotation is None: - print(f"{bcolors.FAIL}Record does not have rotation settings.{bcolors.ENDC}") - return - - controller_uid = record_rotation.get("configuration_uid") - if controller_uid is None: - print(f"{bcolors.FAIL}Record does not have the PAM Configuration set.{bcolors.ENDC}") - return - - configuration_record = vault.KeeperRecord.load(params, controller_uid) # type: Optional[TypedRecord] - - if version_flag is True: - self._versions() - if render_flag is True: - self._render( - params=params, - configuration_record=configuration_record, - infra_name=kwargs.get("infra_name"), - rl_name=kwargs.get("rl_name"), - graph_type=kwargs.get("graph_type"), - graph_dir=kwargs.get("graph_dir"), - render_all_edges=kwargs.get("render_all_edges"), - ) - if info_flag is True: - self._show_info( - params=params, - configuration_record=configuration_record, - record=record - ) - diff --git a/keepercommander/commands/discover/job_remove.py b/keepercommander/commands/discover/job_remove.py index 881562b01..aa1d5b5b0 100644 --- a/keepercommander/commands/discover/job_remove.py +++ b/keepercommander/commands/discover/job_remove.py @@ -11,7 +11,7 @@ class PAMGatewayActionDiscoverJobRemoveCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-discover-command-process') + parser = argparse.ArgumentParser(prog='pam-action-discover-remove') parser.add_argument('--job-id', '-j', required=True, dest='job_id', action='store', help='Discovery job id.') diff --git a/keepercommander/commands/discover/job_start.py b/keepercommander/commands/discover/job_start.py index 816517fbb..3549e3801 100644 --- a/keepercommander/commands/discover/job_start.py +++ b/keepercommander/commands/discover/job_start.py @@ -20,7 +20,7 @@ class PAMGatewayActionDiscoverJobStartCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-discover-start-command') + parser = argparse.ArgumentParser(prog='pam-action-discover-start') parser.add_argument('--gateway', '-g', required=True, dest='gateway', action='store', help='Gateway name of UID.') parser.add_argument('--resource', '-r', required=False, dest='resource_uid', action='store', diff --git a/keepercommander/commands/discover/job_status.py b/keepercommander/commands/discover/job_status.py index 07bd94303..35e1e551d 100644 --- a/keepercommander/commands/discover/job_status.py +++ b/keepercommander/commands/discover/job_status.py @@ -33,7 +33,7 @@ def _b(text): class PAMGatewayActionDiscoverJobStatusCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-discover-status-command') + parser = argparse.ArgumentParser(prog='pam-action-discover-status') parser.add_argument('--gateway', '-g', required=False, dest='gateway', action='store', help='Show only discovery jobs from a specific gateway.') parser.add_argument('--job-id', '-j', required=False, dest='job_id', action='store', diff --git a/keepercommander/commands/discover/result_get.py b/keepercommander/commands/discover/result_get.py deleted file mode 100644 index 4020c72e4..000000000 --- a/keepercommander/commands/discover/result_get.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations -import argparse -from . import PAMGatewayActionDiscoverCommandBase, GatewayContext -from ... import vault_extensions -from ...display import bcolors -from ..pam.router_helper import router_get_connected_gateways -from keepercommander.discovery_common.jobs import Jobs -from keepercommander.discovery_common.infrastructure import Infrastructure -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from keepercommander.discovery_common.jobs import JobItem - - -class PAMGatewayActionDiscoverResultGetCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-discover-command-process') - parser.add_argument('--job-id', '-j', required=True, dest='job_id', action='store', - help='Discovery job id.') - parser.add_argument('--file', required=True, dest='filename', action='store', - help='Save results to file.') - - def get_parser(self): - return PAMGatewayActionDiscoverResultGetCommand.parser - - def execute(self, params, **kwargs): - - job_id = kwargs.get("job_id") - - if not hasattr(params, 'pam_controllers'): - router_get_connected_gateways(params) - - configuration_records = list(vault_extensions.find_records(params, "pam.*Configuration")) - for configuration_record in configuration_records: - - gateway_context = GatewayContext.from_configuration_uid(params, configuration_record.record_uid) - if gateway_context is None: - continue - - jobs = Jobs(record=configuration_record, params=params) - job_item = jobs.get_job(job_id) # type: JobItem - if job_item is None: - continue - - if job_item.end_ts is None: - print(f'{bcolors.FAIL}Discovery job is currently running. Cannot get results.{bcolors.ENDC}') - return - if job_item.success is False: - print(f'{bcolors.FAIL}Discovery job failed. Cannot get results.{bcolors.ENDC}') - return - - # TODO - Make a way to serialize the discovery into a form - infra = Infrastructure(record=configuration_record, params=params) - - return - - print(f'{bcolors.FAIL}Discovery job not found. Cannot get results.{bcolors.ENDC}') diff --git a/keepercommander/commands/discover/result_process.py b/keepercommander/commands/discover/result_process.py index 2126a12e0..dc0f6eaff 100644 --- a/keepercommander/commands/discover/result_process.py +++ b/keepercommander/commands/discover/result_process.py @@ -54,7 +54,7 @@ class PAMGatewayActionDiscoverResultProcessCommand(PAMGatewayActionDiscoverComma Process the discovery data """ - parser = argparse.ArgumentParser(prog='dr-discover-command-process') + parser = argparse.ArgumentParser(prog='pam-action-discover-process') parser.add_argument('--job-id', '-j', required=True, dest='job_id', action='store', help='Discovery job to process.') diff --git a/keepercommander/commands/discover/rule_add.py b/keepercommander/commands/discover/rule_add.py index 8fb36ee45..1853aa5d0 100644 --- a/keepercommander/commands/discover/rule_add.py +++ b/keepercommander/commands/discover/rule_add.py @@ -15,7 +15,7 @@ class PAMGatewayActionDiscoverRuleAddCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-discover-rule-add') + parser = argparse.ArgumentParser(prog='pam-action-discover-rule-add') parser.add_argument('--gateway', '-g', required=True, dest='gateway', action='store', help='Gateway name of UID.') parser.add_argument('--action', '-a', required=True, choices=['add', 'ignore', 'prompt'], diff --git a/keepercommander/commands/discover/rule_list.py b/keepercommander/commands/discover/rule_list.py index 30819bc9d..00e0cb0ad 100644 --- a/keepercommander/commands/discover/rule_list.py +++ b/keepercommander/commands/discover/rule_list.py @@ -12,7 +12,7 @@ class PAMGatewayActionDiscoverRuleListCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-discover-rule-list') + parser = argparse.ArgumentParser(prog='pam-action-discover-rule-list') parser.add_argument('--gateway', '-g', required=True, dest='gateway', action='store', help='Gateway name of UID.') parser.add_argument('--search', '-s', required=False, dest='search', action='store', diff --git a/keepercommander/commands/discover/rule_remove.py b/keepercommander/commands/discover/rule_remove.py index c1d4785e9..5235bde61 100644 --- a/keepercommander/commands/discover/rule_remove.py +++ b/keepercommander/commands/discover/rule_remove.py @@ -7,7 +7,7 @@ class PAMGatewayActionDiscoverRuleRemoveCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-discover-rule-list') + parser = argparse.ArgumentParser(prog='pam-action-discover-rule-remove') parser.add_argument('--gateway', '-g', required=True, dest='gateway', action='store', help='Gateway name of UID') parser.add_argument('--rule-id', '-i', required=True, dest='rule_id', action='store', diff --git a/keepercommander/commands/discover/rule_update.py b/keepercommander/commands/discover/rule_update.py index 84d31b220..9e992b729 100644 --- a/keepercommander/commands/discover/rule_update.py +++ b/keepercommander/commands/discover/rule_update.py @@ -8,7 +8,7 @@ class PAMGatewayActionDiscoverRuleUpdateCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-discover-rule-add') + parser = argparse.ArgumentParser(prog='pam-action-discover-rule-update') parser.add_argument('--gateway', '-g', required=True, dest='gateway', action='store', help='Gateway name of UID.') parser.add_argument('--rule-id', '-i', required=True, dest='rule_id', action='store', diff --git a/keepercommander/commands/discoveryrotation.py b/keepercommander/commands/discoveryrotation.py index bf5742e46..965194fdc 100644 --- a/keepercommander/commands/discoveryrotation.py +++ b/keepercommander/commands/discoveryrotation.py @@ -70,11 +70,8 @@ from .discover.rule_remove import PAMGatewayActionDiscoverRuleRemoveCommand from .discover.rule_update import PAMGatewayActionDiscoverRuleUpdateCommand from .pam_debug.acl import PAMDebugACLCommand -from .pam_debug.alter import PAMDebugAlterCommand from .pam_debug.graph import PAMDebugGraphCommand from .pam_debug.info import PAMDebugInfoCommand -from .pam_debug.verify import PAMDebugVerifyCommand -from .pam_debug.version import PAMDebugVersionCommand from .pam_debug.gateway import PAMDebugGatewayCommand @@ -189,10 +186,10 @@ def __init__(self): self.register_command('info', PAMDebugInfoCommand(), 'Debug a record', 'i') self.register_command('gateway', PAMDebugGatewayCommand(), 'Debug a getway', 'g') self.register_command('graph', PAMDebugGraphCommand(), 'Render graphs', 'r') - self.register_command('verify', PAMDebugVerifyCommand(), 'Verify graphs', 'v') - self.register_command('alter', PAMDebugAlterCommand(), 'Alter graph information', 'a') + + # Disable for now. Needs more work. + # self.register_command('verify', PAMDebugVerifyCommand(), 'Verify graphs', 'v') self.register_command('acl', PAMDebugACLCommand(), 'Control ACL of PAM Users', 'c') - self.register_command('version', PAMDebugVersionCommand(), 'Version modules versions') class PAMCmdListJobs(Command): diff --git a/keepercommander/commands/pam_debug/acl.py b/keepercommander/commands/pam_debug/acl.py index 0e3170641..4d999aa21 100644 --- a/keepercommander/commands/pam_debug/acl.py +++ b/keepercommander/commands/pam_debug/acl.py @@ -14,7 +14,7 @@ class PAMDebugACLCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-pam-command-debug') + parser = argparse.ArgumentParser(prog='pam-action-debug-acl') # The record to base everything on. parser.add_argument('--gateway', '-g', required=True, dest='gateway', action='store', diff --git a/keepercommander/commands/pam_debug/alter.py b/keepercommander/commands/pam_debug/alter.py deleted file mode 100644 index d2d029cbf..000000000 --- a/keepercommander/commands/pam_debug/alter.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - -import argparse -from typing import TYPE_CHECKING - -from ..discover import PAMGatewayActionDiscoverCommandBase - -if TYPE_CHECKING: - from ...params import KeeperParams - - -class PAMDebugAlterCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-pam-command-debug') - - # The record to base everything on. - parser.add_argument('--gateway', '-g', required=False, dest='gateway', action='store', - help='Gateway name or UID.') - - def get_parser(self): - return PAMDebugAlterCommand.parser - - def execute(self, params: KeeperParams, **kwargs): - pass \ No newline at end of file diff --git a/keepercommander/commands/pam_debug/gateway.py b/keepercommander/commands/pam_debug/gateway.py index c5cb0dcc7..c43a5a006 100644 --- a/keepercommander/commands/pam_debug/gateway.py +++ b/keepercommander/commands/pam_debug/gateway.py @@ -14,7 +14,7 @@ class PAMDebugGatewayCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-pam-command-debug') + parser = argparse.ArgumentParser(prog='pam-action-debug-gateway') type_name_map = { PAM_USER: "PAM User", diff --git a/keepercommander/commands/pam_debug/graph.py b/keepercommander/commands/pam_debug/graph.py index d937b2bb6..3227e2cdd 100644 --- a/keepercommander/commands/pam_debug/graph.py +++ b/keepercommander/commands/pam_debug/graph.py @@ -29,7 +29,7 @@ class PAMDebugGraphCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-pam-command-debug') + parser = argparse.ArgumentParser(prog='pam-action-debug-graph') # The record to base everything on. parser.add_argument('--gateway', '-g', required=True, dest='gateway', action='store', diff --git a/keepercommander/commands/pam_debug/info.py b/keepercommander/commands/pam_debug/info.py index ac4ab813a..eea3e6d22 100644 --- a/keepercommander/commands/pam_debug/info.py +++ b/keepercommander/commands/pam_debug/info.py @@ -19,7 +19,7 @@ class PAMDebugInfoCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-pam-command-debug') + parser = argparse.ArgumentParser(prog='pam-action-debug-info') type_name_map = { PAM_USER: "PAM User", diff --git a/keepercommander/commands/pam_debug/verify.py b/keepercommander/commands/pam_debug/verify.py index af5957293..a2c475202 100644 --- a/keepercommander/commands/pam_debug/verify.py +++ b/keepercommander/commands/pam_debug/verify.py @@ -15,7 +15,7 @@ class PAMDebugVerifyCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-pam-command-debug') + parser = argparse.ArgumentParser(prog='pam-action-debug-verify') # The record to base everything on. parser.add_argument('--gateway', '-g', required=True, dest='gateway', action='store', diff --git a/keepercommander/commands/pam_debug/version.py b/keepercommander/commands/pam_debug/version.py deleted file mode 100644 index cfbb879da..000000000 --- a/keepercommander/commands/pam_debug/version.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import annotations -import argparse -from ..discover import PAMGatewayActionDiscoverCommandBase -from ...display import bcolors -from importlib.metadata import version -from typing import Optional, TYPE_CHECKING - -if TYPE_CHECKING: - from ...params import KeeperParams - - -class PAMDebugVersionCommand(PAMGatewayActionDiscoverCommandBase): - parser = argparse.ArgumentParser(prog='dr-pam-command-debug') - - def execute(self, params: KeeperParams, **kwargs): - - print("") - print(f"{bcolors.BOLD}keeper-dag version:{bcolors.ENDC} {version('keeper-dag')}") - print(f"{bcolors.BOLD}discovery-common version:{bcolors.ENDC} {version('discovery-common')}") - print("") \ No newline at end of file diff --git a/keepercommander/discovery_common/constants.py b/keepercommander/discovery_common/constants.py index 3811c5282..6f2ee8511 100644 --- a/keepercommander/discovery_common/constants.py +++ b/keepercommander/discovery_common/constants.py @@ -17,7 +17,6 @@ PAM_DATABASE = "pamDatabase" PAM_MACHINE = "pamMachine" PAM_USER = "pamUser" - LOCAL_USER = "local" # The record types to process. diff --git a/keepercommander/discovery_common/infrastructure.py b/keepercommander/discovery_common/infrastructure.py index 4abf16d9b..65e872c5c 100644 --- a/keepercommander/discovery_common/infrastructure.py +++ b/keepercommander/discovery_common/infrastructure.py @@ -32,9 +32,9 @@ class Infrastructure: USER_PATH = "USERS" def __init__(self, record: Any, logger: Optional[Any] = None, history_level: int = 0, - debug_level: int = 0, fail_on_corrupt: bool = True, **kwargs): - - self.conn = get_connection(**kwargs) + debug_level: int = 0, fail_on_corrupt: bool = True, log_prefix: str = "GS Infrastructure", + save_batch_count: int = 0, + **kwargs): # This will either be a KSM Record, or Commander KeeperRecord self.record = record @@ -42,23 +42,29 @@ def __init__(self, record: Any, logger: Optional[Any] = None, history_level: int if logger is None: logger = logging.getLogger() self.logger = logger + self.log_prefix = log_prefix self.history_level = history_level self.debug_level = debug_level self.fail_on_corrupt = fail_on_corrupt + self.save_batch_count = save_batch_count self.auto_save = False self.delta_graph = True self.last_sync_point = -1 + self.conn = get_connection(**kwargs) + @property def dag(self) -> DAG: if self._dag is None: self.logger.debug(f"loading the dag graph {DIS_INFRA_GRAPH_ID}") + self.logger.debug(f"setting graph save batch count to {self.save_batch_count}") self._dag = DAG(conn=self.conn, record=self.record, graph_id=DIS_INFRA_GRAPH_ID, auto_save=self.auto_save, logger=self.logger, history_level=self.history_level, debug_level=self.debug_level, - name="Discovery Infrastructure", fail_on_corrupt=self.fail_on_corrupt) + name="Discovery Infrastructure", fail_on_corrupt=self.fail_on_corrupt, + log_prefix=self.log_prefix, save_batch_count=self.save_batch_count) # Do not load the DAG here. # We don't know if we are using a sync point yet. diff --git a/keepercommander/discovery_common/jobs.py b/keepercommander/discovery_common/jobs.py index feb8eb844..f4900be7c 100644 --- a/keepercommander/discovery_common/jobs.py +++ b/keepercommander/discovery_common/jobs.py @@ -18,7 +18,7 @@ class Jobs: KEY_PATH = "jobs" def __init__(self, record: Any, logger: Optional[Any] = None, debug_level: int = 0, fail_on_corrupt: bool = True, - **kwargs): + log_prefix: str = "GS Jobs", **kwargs): self.conn = get_connection(**kwargs) @@ -28,6 +28,7 @@ def __init__(self, record: Any, logger: Optional[Any] = None, debug_level: int = if logger is None: logger = logging.getLogger() self.logger = logger + self.log_prefix = log_prefix self.debug_level = debug_level self.fail_on_corrupt = fail_on_corrupt @@ -37,7 +38,7 @@ def dag(self) -> DAG: self._dag = DAG(conn=self.conn, record=self.record, graph_id=DIS_JOBS_GRAPH_ID, auto_save=False, logger=self.logger, debug_level=self.debug_level, name="Discovery Jobs", - fail_on_corrupt=self.fail_on_corrupt) + fail_on_corrupt=self.fail_on_corrupt, log_prefix=self.log_prefix) self._dag.load() # Has the status been initialized? diff --git a/keepercommander/discovery_common/process.py b/keepercommander/discovery_common/process.py index 947c4d4d9..13b5f007d 100644 --- a/keepercommander/discovery_common/process.py +++ b/keepercommander/discovery_common/process.py @@ -1,5 +1,6 @@ from __future__ import annotations import logging +import os from .constants import PAM_DIRECTORY, PAM_USER, VERTICES_SORT_MAP, LOCAL_USER from .jobs import Jobs from .infrastructure import Infrastructure @@ -15,7 +16,6 @@ from keepercommander.keeper_dag.crypto import bytes_to_urlsafe_str import hashlib import time -import os from typing import Any, Callable, List, Optional, Union, TYPE_CHECKING @@ -324,6 +324,7 @@ def _directory_exists(self, domain: str, directory_info_func: Callable, context: self.logger.debug(f"found {len(directories)} directories in the graph") + # If we found directories, return the list of directory vertices. if len(directories) > 0: return directories @@ -812,7 +813,6 @@ def _process_auto_add_level(self, # The user is a remote user. else: self.logger.debug(" check directory for remote user") - domain = content.access_user.source if content.record_type == PAM_DIRECTORY: domain = content.name @@ -887,7 +887,6 @@ def _process_auto_add_level(self, if admin_user_record_uid is not None or admin_content is not None: if admin_content is not None: - self.logger.debug("the admin record does not exists, create it") # Create the local admin here since we need the resource record added. diff --git a/keepercommander/discovery_common/record_link.py b/keepercommander/discovery_common/record_link.py index 0cd143d8c..70a53a431 100644 --- a/keepercommander/discovery_common/record_link.py +++ b/keepercommander/discovery_common/record_link.py @@ -14,7 +14,7 @@ class RecordLink: def __init__(self, record: Any, logger: Optional[Any] = None, debug_level: int = 0, fail_on_corrupt: bool = True, - **kwargs): + log_prefix: str = "GS Record Linking", **kwargs): self.conn = get_connection(**kwargs) @@ -24,6 +24,7 @@ def __init__(self, record: Any, logger: Optional[Any] = None, debug_level: int = if logger is None: logger = logging.getLogger() self.logger = logger + self.log_prefix = log_prefix self.debug_level = debug_level # Technically, since there is no encryption in this graph, there should be no corruption. @@ -38,7 +39,7 @@ def dag(self) -> DAG: # Since we don't have transactions, we want to save the record link if everything worked. self._dag = DAG(conn=self.conn, record=self.record, graph_id=RECORD_LINK_GRAPH_ID, auto_save=False, logger=self.logger, debug_level=self.debug_level, name="Record Linking", - fail_on_corrupt=self.fail_on_corrupt) + fail_on_corrupt=self.fail_on_corrupt, log_prefix=self.log_prefix) sync_point = self._dag.load(sync_point=0) self.logger.debug(f"the record linking sync point is {sync_point or 0}") if self.dag.has_graph is False: diff --git a/keepercommander/discovery_common/types.py b/keepercommander/discovery_common/types.py index c38551255..8d5712f24 100644 --- a/keepercommander/discovery_common/types.py +++ b/keepercommander/discovery_common/types.py @@ -70,8 +70,10 @@ class Settings(BaseModel): skip_directories: bool = False skip_cloud_users: bool = False - allow_resource_deletion: bool = True - allow_user_deletion: bool = True + # For now, don't delete anything. + allow_resource_deletion: bool = False + allow_user_deletion: bool = False + resource_deletion_limit: int = 0 user_deletion_limit: int = 0 @@ -364,6 +366,10 @@ class DiscoveryUser(DiscoveryItem): password: Optional[str] = None private_key: Optional[str] = None + # Simple flag, for access user in discovery, that states could connect with creds. + # Local connection might not have passwords, so this is our flag to indicate that the user connected. + could_login: Optional[bool] = False + class FactsDirectory(BaseModel): domain: str @@ -462,6 +468,10 @@ class DiscoveryObject(BaseModel): # If the object is missing, this will show a timestamp on when it went missing. missing_since_ts: Optional[int] = None + # Should this object be deleted? This does not prevent user from deleting, but prevents automated processed from + # deleting. + allow_delete: bool = False + # This is not the official admin. # This is the user discovery used to access to the resource. # This will be used to help the user create an admin user. @@ -573,7 +583,7 @@ def find_user(self, user): def find_dn(self, user): return self._field("distinguishedName", user) - + class PromptResult(BaseModel): # "add" and "ignore" are the only action @@ -620,7 +630,7 @@ class BulkRecordAdd(BaseModel): # Record note note: Optional[str] = None - # This could be a Commander KeeperRecord, Commander RecordAdd, or KSM Record + # This could be a Commander KeeperRecord, Commander RecordAdd, NormalizedRecord, or KSM Record record: Any record_type: str diff --git a/keepercommander/discovery_common/user_service.py b/keepercommander/discovery_common/user_service.py index 65490b421..a64820fa6 100644 --- a/keepercommander/discovery_common/user_service.py +++ b/keepercommander/discovery_common/user_service.py @@ -17,7 +17,8 @@ class UserService: def __init__(self, record: Any, logger: Optional[Any] = None, history_level: int = 0, - debug_level: int = 0, fail_on_corrupt: bool = True, **kwargs): + debug_level: int = 0, fail_on_corrupt: bool = True, log_prefix: str = "GS Services/Tasks", + **kwargs): self.conn = get_connection(**kwargs) @@ -27,6 +28,7 @@ def __init__(self, record: Any, logger: Optional[Any] = None, history_level: int if logger is None: logger = logging.getLogger() self.logger = logger + self.log_prefix = log_prefix self.history_level = history_level self.debug_level = debug_level self.fail_on_corrupt = fail_on_corrupt @@ -41,7 +43,7 @@ def dag(self) -> DAG: self._dag = DAG(conn=self.conn, record=self.record, graph_id=USER_SERVICE_GRAPH_ID, auto_save=False, logger=self.logger, history_level=self.history_level, debug_level=self.debug_level, name="Discovery Service/Tasks", - fail_on_corrupt=self.fail_on_corrupt) + fail_on_corrupt=self.fail_on_corrupt, log_prefix=self.log_prefix) self._dag.load(sync_point=0) @@ -473,5 +475,3 @@ def run(self, infra: Optional[Infrastructure] = None, **kwargs): infra_resource_content.item.facts.tasks) self.save() - - diff --git a/keepercommander/discovery_common/utils.py b/keepercommander/discovery_common/utils.py index 79748b998..c381caa64 100644 --- a/keepercommander/discovery_common/utils.py +++ b/keepercommander/discovery_common/utils.py @@ -1,7 +1,7 @@ from __future__ import annotations import os -from .constants import PAM_USER -from .types import DiscoveryObject +from discovery_common.constants import PAM_USER +from discovery_common.types import DiscoveryObject from keepercommander.keeper_dag.vertex import DAGVertex from typing import List, Optional, TYPE_CHECKING @@ -48,7 +48,10 @@ def get_connection(**kwargs): return conn -def split_user_and_domain(user: str) -> (str, Optional[str]): +def split_user_and_domain(user: str) -> (Optional[str], Optional[str]): + + if user is None: + return None, None domain = None @@ -94,6 +97,7 @@ def user_in_lookup(user: str, lookup: dict, name: Optional[str] = None, source: return False + def find_user_vertex(graph: DAG, user: str, domain: Optional[str] = None) -> Optional[DAGVertex]: user_vertices = graph.search_content({"record_type": PAM_USER}) @@ -114,8 +118,3 @@ def find_user_vertex(graph: DAG, user: str, domain: Optional[str] = None) -> Opt return user_vertex return None - - - - - diff --git a/keepercommander/keeper_dag/README.md b/keepercommander/keeper_dag/README.md new file mode 100644 index 000000000..3a34d132b --- /dev/null +++ b/keepercommander/keeper_dag/README.md @@ -0,0 +1,9 @@ +# Keeper Graph (DAG) + +Do not edit this code, alone. + +Edit the repo https://github.com/Keeper-Security/keeper-dag + +If you do edit this directory, +and do not include you changed is the **keeper-dag** repo, +your changes may be overwritten.