Updated fetch entities to fetch workloads and added script to overrid…

…e and reset workload golden signals (#33)

* feat: add support for fetching workloads

Added --workload flag to fetch workload entities

* feat: Added script to override and reset workload golden signals

Golden signals can be specified in a json file saved under ./goldensignals directory

README.md

@@ -56,7 +56,7 @@ APM Settings
 
 ## Getting Started
 
-The table below lists the scripts that can be used for different migration use cases.
+The table below lists the scripts that can be used for different migration and other use cases.
 
 These scripts, in some cases, require sequential execution as they build the data necessary to migrate entities from one account to another.
 
@@ -72,7 +72,8 @@ The details for each script is provided in the next Usage section.
 | 5.  | Migrate Dashboards | migrate_dashboards.py :arrow_right: migratetags.py |
 | 6.  | Update Monitors | updatemonitors.py | 
 | 7.  | Delete Monitors | deletemonitors.py | 
-| 8.  | Migrate Tags | migratetags.py
+| 8.  | Migrate Tags | migratetags.py |
+| 9.  | Update Workload Golden Signals | wlgoldensignals.py |
 
 
 The following entities and configurations can be migrated:
@@ -490,6 +491,40 @@ output : output/<entityName>.csv file for each entityName with names of metrics
 
 received from that entity
 
+####  19) python3 wlgoldensignals.py
+Automated script for overriding and resetting golden signals for workloads. 
+####Note: By default workloads only display 4 golden signals.
+
+usage: wlgoldensignals.py --targetAccount TARGETACCOUNT --targetApiKey  TARGETAPIKEY [--targetRegion TARGETREGION]
+                          [--tagName TAGNAME] [--tagValue TAGVALUE] [--goldenSignalsJson GOLDENSIGNALSJSON]
+                          [--resetGoldenSignals] [--domain DOMAIN] [--type TYPE]
+
+Parameter      | Note
+-------------- | --------------------------------------------------
+targetAccount  | Account containing the workloads
+targetRegion   | Optional region us (default) or eu
+targetApiKey   | User API Key for targetAccount
+tagName        | Tag name to use to find matching workloads 
+tagValue       | Tag value to use to find matching workloads
+goldenSignalsJson     | File stored under ./goldensignals directory that contains list of metrics in JSON format. [./goldensignals/linuxgoldensignals.json](goldensignals/linuxgoldensignals.json)
+resetGoldenSignals | Pass this flag to reset the override golden signals for a domain/type combination
+domain | domain for which to reset the golden signals APM , BROWSER , INFRA , MOBILE , SYNTH , EXT
+type | type of entity APPLICATION , DASHBOARD , HOST , MONITOR , WORKLOAD
+
+#### example 1: override golden signals
+python3 wlgoldensignals.py --targetAccount ACCT_ID --targetApiKey USER_API_KEY --goldenSignalsJson windowsgoldensignals.json --tagName Environment --tagValue WindowsProduction
+The above will find workloads having tag Environment=WindowsProduction and then for each workload 
+override the golden signals as specified in goldensignals/windowsgoldensignals.json for entities of domain INFRA and type HOST as specified in the json file
+
+#### example 2: reset override golden signals
+python3 wlgoldensignals.py --targetAccount ACCT_ID --targetApiKey USER_API_KEY --resetGoldenSignals --tagName Environment --tagValue WindowsProduction --domain INFRA --type HOST
+The above will find workloads having tag Environment=WindowsProduction and then for each workload 
+reset the golden signals for domain INFRA and type HOST
+
+
+
+
+
 
 ### Logging
 

fetchentities.py

@@ -27,6 +27,8 @@ def configure_parser():
     parser.add_argument('--infraint', dest='infraint', required=False, action='store_true', help='Pass --infraint to list matching Infrastructure integration entities')
     parser.add_argument('--mobile', dest='mobile', required=False, action='store_true', help='Pass --mobile to list matching Mobile application entities')
     parser.add_argument('--lambda', dest='lambda_function', required=False, action='store_true', help='Pass --lambda to list matching Lambda function entities')
+    parser.add_argument('--workload', dest='workload', required=False, action='store_true',
+                        help='Pass --workloads to list matching Workload entities')
     parser.add_argument('--tagName', nargs=1, required=False, help='(Optional) Tag name to use when filtering results. Required if --tagValue is passed.')
     parser.add_argument('--tagValue', nargs=1, required=False, help='(Optional) Tag value to use when filtering results. Required if --tagName is passed.')
     return parser
@@ -63,6 +65,8 @@ def parse_entity_types(args):
         entity_types.append(ec.MOBILE_APP)
     if args.lambda_function:
         entity_types.append(ec.INFRA_LAMBDA)
+    if args.workload:
+        entity_types.append(ec.WORKLOAD)
     return entity_types
 
 

goldensignals/linuxgoldensignals.json

@@ -0,0 +1,51 @@
+{
+  "domain": "INFRA",
+  "type": "HOST",
+  "metrics": [
+    {
+      "eventId": "entityGuid",
+      "facet": "entityName",
+      "from": "SystemSample",
+      "name": "cpuUsage",
+      "select": "average(cpuPercent)",
+      "title": "CPU usage (%)",
+      "where": "operatingSystem = 'linux'"
+    },
+    {
+      "eventId": "entityGuid",
+      "facet": "entityName",
+      "from": "SystemSample",
+      "select": "average(memoryUsedPercent)",
+      "name": "memoryUsage",
+      "title": "Memory usage (%)",
+      "where": "operatingSystem = 'linux'"
+    },
+    {
+      "eventId": "entityGuid",
+      "facet": "entityName",
+      "from": "SystemSample",
+      "select": "average(loadAverageFiveMinute)",
+      "name": "loadAverageFiveMinute",
+      "title": "Load Average 5 Minutes",
+      "where": "operatingSystem = 'linux'"
+    },
+    {
+      "eventId": "entityGuid",
+      "facet": "entityName",
+      "from": "StorageSample",
+      "select": "average(diskUsedPercent)",
+      "name": "rootDiskUsedPercentage",
+      "title": "Root Disk Used (%)",
+      "where": "operatingSystem = 'linux' AND mountPoint = '/'"
+    },
+    {
+      "eventId": "entityGuid",
+      "facet": "entityName",
+      "from": "SystemSample",
+      "select": "average(swapUsedBytes/swapTotalBytes*100)",
+      "name": "swapUsedPercentage",
+      "title": "Swap Used (%)",
+      "where": "operatingSystem = 'linux'"
+    }
+  ]
+}

goldensignals/windowsgoldensignals.json

@@ -0,0 +1,51 @@
+{
+  "domain": "INFRA",
+  "type": "HOST",
+  "metrics": [
+    {
+      "eventId": "entityGuid",
+      "facet": "entityName",
+      "from": "SystemSample",
+      "name": "cpuUsage",
+      "select": "average(cpuPercent)",
+      "title": "CPU usage (%)",
+      "where": "operatingSystem = 'windows'"
+    },
+    {
+      "eventId": "entityGuid",
+      "facet": "entityName",
+      "from": "SystemSample",
+      "select": "average(memoryUsedPercent)",
+      "name": "memoryUsage",
+      "title": "Memory usage (%)",
+      "where": "operatingSystem = 'windows'"
+    },
+    {
+      "eventId": "entityGuid",
+      "facet": "entityName",
+      "from": "SystemSample",
+      "select": "average(loadAverageFiveMinute)",
+      "name": "loadAverageFiveMinute",
+      "title": "Load Average 5 Minutes",
+      "where": "operatingSystem = 'windows'"
+    },
+    {
+      "eventId": "entityGuid",
+      "facet": "entityName",
+      "from": "StorageSample",
+      "select": "average(diskUsedPercent)",
+      "name": "rootDiskUsedPercentage",
+      "title": "C: Drive Used (%)",
+      "where": "operatingSystem = 'windows' AND mountPoint= 'C:'"
+    },
+    {
+      "eventId": "entityGuid",
+      "facet": "entityName",
+      "from": "SystemSample",
+      "select": "average(swapUsedBytes/swapTotalBytes*100)",
+      "name": "swapUsedPercentage",
+      "title": "Swap Used (%)",
+      "where": "operatingSystem = 'windows'"
+    }
+  ]
+}

library/clients/entityclient.py

@@ -17,6 +17,7 @@
 INFRA_HOST = 'INFRA_HOST'
 INFRA_INT = 'INFRA_INT'
 INFRA_LAMBDA = 'INFRA_LAMBDA'
+WORKLOAD = 'WORKLOAD'
 
 # Mapping to entityType tag values
 ent_type_lookup = {}
@@ -160,6 +161,19 @@ def entity_outline(entity_type):
                     values
                 }
             } '''
+    if entity_type == WORKLOAD:
+        return ''' ... on WorkloadEntityOutline {
+                guid
+                name
+                type
+                permalink
+                entityType
+                accountId
+                tags {
+                        key
+                        values
+                } 
+              }'''
 
 
 def search_query_payload(entity_type, entity_name, acct_id = None):
@@ -339,6 +353,8 @@ def get_entities_payload(entity_type, acct_id = None, nextCursor = None, tag_nam
         gql_search_type = 'HOST'
     elif entity_type == INFRA_LAMBDA:
         gql_search_type = 'AWSLAMBDAFUNCTION'
+    elif entity_type == WORKLOAD:
+        gql_search_type = 'WORKLOAD'
 
     entity_search_query = '''query($matchingCondition: String!) { 
                                     actor { 
@@ -381,7 +397,6 @@ def gql_get_entities_by_type(api_key, entity_type, acct_id=None, tag_name=None,
 
     while not done:
         payload = get_entities_payload(entity_type, acct_id, nextCursor, tag_name, tag_value)
-
         response = requests.post(Endpoints.of(region).GRAPHQL_URL, headers=gql_headers(api_key), data=json.dumps(payload))
         if response.status_code != 200:
             done = True
@@ -637,6 +652,7 @@ def tags_diff(src_tags, tgt_tags):
             tags_arr.append(src_tag)
     return tags_arr
 
+
 def mutate_tags_payload(entity_guid, arr_tags, mutate_action):
     apply_tags_query = '''mutation($entityGuid: EntityGuid!, $tags: [TaggingTagInput!]!) 
                             {''' + mutate_action + '''(guid: $entityGuid, tags: $tags) {

library/clients/goldensignals.py

@@ -0,0 +1,54 @@
+import os
+import json
+import library.migrationlogger as logger
+import library.clients.gql as nerdgraph
+from library.clients.endpoints import Endpoints
+
+logger = logger.get_logger(os.path.basename(__file__))
+
+
+class GoldenSignals:
+
+    def __init__(self, region=Endpoints.REGION_US):
+        self.region = region
+        pass
+
+    def reset(self, user_api_key, workload_guid, domain, type):
+        payload = GoldenSignals._reset_golden_signals_payload(workload_guid, domain, type)
+        logger.debug(json.dumps(payload))
+        return nerdgraph.GraphQl.post(user_api_key, payload, self.region)
+
+
+    def override(self, user_api_key, workload_guid, domain, type, metrics):
+        payload = GoldenSignals._override_golden_signals_payload(workload_guid, domain, type, metrics)
+        logger.debug(json.dumps(payload))
+        return nerdgraph.GraphQl.post(user_api_key, payload, self.region)
+
+    @staticmethod
+    def _reset_golden_signals_payload(workload_guid, domain, type):
+        mutation_query = '''mutation($context: EntityGoldenContextInput!, $domainType: DomainTypeInput!) { 
+                                entityGoldenMetricsReset(context: $context, domainType: $domainType) { 
+                                    errors {message type} 
+                                }                                     
+                            }'''
+        return {'query': mutation_query, 'variables': {'context': {'guid':  workload_guid},
+                                                       'domainType': {'domain': domain, 'type': type}
+                                                       }
+                }
+
+    @staticmethod
+    def _override_golden_signals_payload(workload_guid, domain, type, metrics):
+
+        mutation_query = '''mutation($context: EntityGoldenContextInput!, $domainType: DomainTypeInput!,
+        $metrics: [EntityGoldenMetricInput!]!) {
+            entityGoldenMetricsOverride(context: $context, domainType: $domainType, metrics: $metrics) {
+                errors { message type }
+                metrics { metrics { name query title } }
+            }
+        }
+        '''
+        return {'query': mutation_query, 'variables': {'context': {'guid': workload_guid},
+                                                       'domainType': {'domain': domain, 'type': type},
+                                                       'metrics': metrics
+                                                       }
+                }

library/clients/gql.py

@@ -0,0 +1,34 @@
+import os
+import requests
+import json
+import library.migrationlogger as logger
+from library.clients.endpoints import Endpoints
+
+
+logger = logger.get_logger(os.path.basename(__file__))
+
+
+class GraphQl:
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def post(per_api_key, payload, region=Endpoints.REGION_US):
+        result = {}
+        response = requests.post(Endpoints.of(region).GRAPHQL_URL, headers=GraphQl.headers(per_api_key),
+                                 data=json.dumps(payload))
+        result['status'] = response.status_code
+        if response.text:
+            response_json = response.json()
+            if 'errors' in response_json:
+                logger.error('Error : ' + response.text)
+                result['error'] = response_json['errors']
+            else:
+                logger.debug('Success : ' + response.text)
+                result['response'] = response_json
+        return result
+
+    @staticmethod
+    def headers(api_key):
+        return {'api-key': api_key, 'Content-Type': 'application/json'}

library/localstore.py

@@ -166,6 +166,19 @@ def load_json_file(account_id, dir_name, json_file_name):
     return file_json
 
 
+def load_json_from_file(dir_name, json_file_name):
+    # Same as load_json_file without the hard coded DB_DIR/account_id prefixing
+    file_json = {}
+    json_dir = Path(dir_name)
+    if json_dir.exists():
+        json_file = json_dir / json_file_name
+        if json_file.exists():
+            file_json = json.loads(json_file.read_text())
+    else:
+        logger.error(dir_name + " does not exist.")
+    return file_json
+
+
 def load_monitor_labels(account_id):
     return load_json_file(account_id, LABELS_DIR, MONITOR_LABELS_FILE)
 
@@ -195,8 +208,8 @@ def create_output_file(file_name):
     logger.debug("Creating output file")
     output_dir = Path("output")
     output_dir.mkdir(mode=0o777, exist_ok=True)
-    monitor_names_file = output_dir / file_name
-    return create_file(monitor_names_file)
+    output_file = output_dir / file_name
+    return create_file(output_file)
 
 
 def sanitize(name):

library/utils.py

@@ -158,6 +158,7 @@ def error_message_and_exit(msg):
     logger.error(msg)
     sys.exit()
 
+
 def get_entity_type(app_condition):
     if app_condition['type'] in ['apm_app_metric', 'apm_jvm_metric']:
         return ec.APM_APP