feat: Added Support for region flag and EU endpoints in all remaining…

… scripts (#24)

* feat: eu endpoint and region flag support for store_policies

* feat: region flag and eu endpoint support for store_policy_entity_map

* feat: region flag and eu endpoint support for store_violations

* cleanup:  make fetchentities consistent with other scripts

Also mark replicatemonitors as do not use/unsupported

* docs: Updated README for all new region flag options

Also added note for using PEP8 Style guide in contribution section

README.md

@@ -334,8 +334,8 @@ Migrate dashboards between accounts, including modifying queries to point to the
 ####  9) python3 migratetags.py
 
 usage: migratetags.py [-h] --fromFile FROMFILE --sourceAccount
-                            SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY
-                            --targetAccount TARGETACCOUNT --targetApiKey TARGETAPIKEY
+                            SOURCEACCOUNT [--sourceRegion SOURCEREGION] --sourceApiKey SOURCEAPIKEY
+                            --targetAccount TARGETACCOUNT [--targetRegion TARGETREGION]  --targetApiKey TARGETAPIKEY
                             [--apm --browser --dashboards --infrahost --infraint --lambda --mobile --securecreds --synthetics]
 
 Migrate entity tags between entities with matching names and entity types. 
@@ -344,8 +344,10 @@ Parameter      | Note
 -------------- | --------------------------------------------------
 fromFile       | Must contain entity names one per line. The fetchentities.py script can help create this file.
 sourceAccount  | Account to search for a matching source entity
+sourceRegion   | Optional region us (default) or eu
 sourceApiKey   | User API Key for sourceAccount 
 targetAccount  | Account to search for a matching target entity
+targetRegion   | Optional region us (default) or eu
 targetApiKey   | User API Key for targetAccount for a user with admin (or add on / custom role equivalent) access to Alerts
 apm            | Pass this flag to migrate APM entity tags
 browser        | Pass this flag to migrate Browser entity tags
@@ -368,6 +370,7 @@ Parameter     | Note
 ------------- | -------------------------------------------------------------------------
 fromFile      | Specifies the file with relative path, listing the monitors to be updated. The fetchentities.py script can help generate this file.
 targetAccount | Account in which monitors need to be updated
+targetRegion   | Optional region us (default) or eu
 targetApiKey  | This should be a User API Key for targetAccount for a user with admin (or add on / custom role equivalent) access to Synthetics
 timeStamp     | must match the timeStamp generated in fetchmonitors
 renamePrefix  | Monitors are renamed with this prefix
@@ -386,7 +389,7 @@ output/targetAccount_fromFile_updated_monitors.csv
 
 ####  11) python3 fetchentities.py
 
-usage: fetchentities.py [-h] --sourceAccount SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY
+usage: fetchentities.py [-h] --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION]  --sourceApiKey SOURCEAPIKEY
                             --toFile FILENAME [--tagName TAGNAME --tagValue TAGVALUE]
                             [--apm --browser --dashboards --infrahost --infraint --lambda --mobile --securecreds --synthetics] 
 
@@ -395,6 +398,7 @@ Create a file in the output directory that contains entity names from the source
 Parameter      | Note
 -------------- | --------------------------------------------------
 sourceAccount  | Account to search for matching entities
+sourceRegion   | Optional region us (default) or eu
 sourceApiKey   | User API Key for sourceAccount 
 toFile         | File name to use to store entity names. This file will be created in the output directory.
 tagName        | Tag name to use to filter results
@@ -428,15 +432,17 @@ deleteallmonitors fetches all the monitors. Backs them up in db/accountId/monito
 
 ####  14) (optional) python3 store_policies.py
 
-usage: store_policies.py [-h] --sourceAccount SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY
+usage: store_policies.py [-h] --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION]  --sourceApiKey SOURCEAPIKEY
 
 Saves all alert polices in db/<sourceAccount>/alert_policies/alert_policies.json
 
 ####  15) (optional) python3 store_violations.py
 
-usage: store_violations.py [-h] --sourceAccount SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY --startDate STARTDATE --endDate ENDDATE [--onlyOpen]
+usage: store_violations.py [-h] --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION]  --sourceApiKey SOURCEAPIKEY --startDate STARTDATE --endDate ENDDATE [--onlyOpen]
 
  --sourceAccount SOURCEACCOUNT Source accountId
+
+ --sourceRegion' SOURCEREGION us (default) or eu
 
   --sourceApiKey SOURCEAPIKEY Source account API Key or set environment variable ENV_SOURCE_API_KEY
 
@@ -452,7 +458,7 @@ and db/<sourceAccount>/alert_violations/alert_violations.csv
 
 ####  16) (optional) python3 store_policy_entity_map.py
 
-usage: store_policy_entity_map.py [-h] --sourceAccount SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY --useLocal
+usage: store_policy_entity_map.py [-h] --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION]  --sourceApiKey SOURCEAPIKEY --useLocal
 
 Builds a mapping from APM, Browser, and Mobile applications and APM key
 transactions to and from alert policies for any policies which contain
@@ -476,6 +482,9 @@ Verification of test is also manual at the moment.
 We encourage your contributions to improve nr-account-migration! Keep in mind when you submit your pull request, you'll need to sign the CLA via the click-through using CLA-Assistant. You only have to sign the CLA one time per project.
 If you have any questions, or to execute our corporate CLA, required if your contribution is on behalf of a company,  please drop us an email at [email protected].
 
+### Style Guide
+Use PEP 8 Style Guide for Python Code https://www.python.org/dev/peps/pep-0008/
+
 **A note about vulnerabilities**
 
 As noted in our [security policy](../../security/policy), New Relic is committed to the privacy and security of our customers and their data. We believe that providing coordinated disclosure by security researchers and engaging with the security community are important means to achieve our security goals.

fetchentities.py

@@ -8,15 +8,14 @@
 import library.utils as utils
 
 logger = migrationlogger.get_logger(os.path.basename(__file__))
-args = None
 
 
-def setup_params():
+def configure_parser():
     parser = argparse.ArgumentParser(description='Migrate entity tags from one account to another')
     parser.add_argument('--sourceAccount', nargs=1, required=True, help='Source accountId')
     parser.add_argument('--sourceApiKey', nargs=1, required=False, help='Source API Key or \
     set env var ENV_SOURCE_API_KEY')
-    parser.add_argument('--region', type=str, nargs=1, required=False, help='region us(default) or eu')
+    parser.add_argument('--sourceRegion', type=str, nargs=1, required=False, help='region us(default) or eu')
     parser.add_argument('--toFile', nargs=1, required=True, help='File to populate entity names. '
                                                                  'This will be created in output directory')
     parser.add_argument('--synthetics', dest='synthetics', required=False, action='store_true', help='Pass --synthetics to list matching Synthetic monitor entities')
@@ -33,13 +32,13 @@ def setup_params():
     return parser
 
 
-def print_params(args, source_api_key, entity_types, region):
+def print_params(args, source_api_key, entity_types, src_region):
     logger.info("Using sourceAccount : " + str(args.sourceAccount[0]))
     logger.info("Using sourceApiKey : " + len(source_api_key[:-4])*"*"+source_api_key[-4:])
-    if args.region and len(args.region) > 0:
-        logger.info("region : " + args.region[0])
+    if args.sourceRegion and len(args.sourceRegion) > 0:
+        logger.info("region : " + args.sourceRegion[0])
     else:
-        logger.info("region not passed : Defaulting to " + region)
+        logger.info("region not passed : Defaulting to " + src_region)
     logger.info("Using entity types : " + str(entity_types))
     if args.tagName:
         logger.info("Using tag name " + str(args.tagName[0]) + " and tag value " + str(args.tagValue[0]))
@@ -70,10 +69,11 @@ def parse_entity_types(args):
     return entity_types
 
 
-def fetch_entities(src_account_id, src_api_key, entity_types, output_file, *, tag_name=None, tag_value=None, region='us'):
+def fetch_entities(src_account_id, src_api_key, entity_types, output_file, *,
+                   tag_name=None, tag_value=None, src_region='us'):
     entity_names = []
     for entity_type in entity_types:
-        entities = ec.gql_get_entities_by_type(src_api_key, entity_type, src_account_id, tag_name, tag_value, region)
+        entities = ec.gql_get_entities_by_type(src_api_key, entity_type, src_account_id, tag_name, tag_value, src_region)
         for entity in entities['entities']:
             entity_names.append(entity['name'])
     entity_names_file = store.create_output_file(output_file)
@@ -86,34 +86,29 @@ def fetch_entities(src_account_id, src_api_key, entity_types, output_file, *, ta
 
 
 def main():
-    parser = setup_params()
+    parser = configure_parser()
     args = parser.parse_args()
-
-    source_api_key = utils.ensure_source_api_key(args)
-    if not source_api_key:
+    src_api_key = utils.ensure_source_api_key(args)
+    if not src_api_key:
         utils.error_and_exit('source api key', 'ENV_SOURCE_API_KEY')
-
     entity_types = parse_entity_types(args)
     if len(entity_types) == 0:
         logger.error('At least one entity type must be specified. Currently supported: ' +
                             ec.SYNTH_MONITOR + ',' + ec.SYNTH_SECURE_CRED + ',' + ec.APM_APP + ',' + ec.BROWSER_APP + ',' + ec.DASHBOARDS + ',' + ec.INFRA_HOST + ',' + ec.INFRA_INT + ',' + ec.MOBILE_APP + ',' + ec.INFRA_LAMBDA)
         sys.exit()
-
     if args.tagName is not None and args.tagValue is None:
         logger.error('tagValue is required when tagName is set')
         sys.exit()
-
     if args.tagValue is not None and args.tagName is None:
         logger.error('tagName is required when tagValue is set')
         sys.exit()
-    region = utils.ensure_region(args)
-    print_params(args, source_api_key, entity_types, region)
-
+    src_region = utils.ensure_source_region(args)
+    print_params(args, src_api_key, entity_types, src_region)
     if args.tagName is None:
-        fetch_entities(args.sourceAccount[0], source_api_key, entity_types, args.toFile[0], region=region)
+        fetch_entities(args.sourceAccount[0], src_api_key, entity_types, args.toFile[0], src_region=src_region)
     else:
-        fetch_entities(args.sourceAccount[0], source_api_key, entity_types, args.toFile[0], tag_name=args.tagName[0],
-                       tag_value=args.tagValue[0], region=region)
+        fetch_entities(args.sourceAccount[0], src_api_key, entity_types, args.toFile[0], tag_name=args.tagName[0],
+                       tag_value=args.tagValue[0], src_region=src_region)
 
 
 if __name__ == '__main__':

library/clients/alertsclient.py

@@ -356,15 +356,15 @@ def get_alert_status_file_name(fromFile, fromFileEntities, src_account_id, tgt_a
     return status_file_name + str(tgt_account_id) + '_conditions.csv'
 
 
-def get_policy_entity_map(api_key, alert_policies):
+def get_policy_entity_map(api_key, alert_policies, region=Endpoints.REGION_US):
     entities_by_policy = {}
     policies_by_entity = {}
     for policy in alert_policies:
         policy_id = policy['id']
         policy_name = policy['name']
         apps = []
         logger.info('Loading app entity conditions for policy ID %d...' % policy_id)
-        conditions = get_app_conditions(api_key, policy_id)
+        conditions = get_app_conditions(api_key, policy_id, region)
         if not 'response_count' in conditions or conditions['response_count'] == 0:
             logger.info('No app entity conditions found for policy ID %d' % policy_id)
             entities_by_policy[policy_name] = []

library/clients/endpoints.py

@@ -52,6 +52,8 @@ class USEndpoints:
     INFRA_CONDITIONS_URL = 'https://infra-api.newrelic.com/v2/alerts/conditions'
     CREATE_INFRA_CONDITION_URL = 'https://infra-api.newrelic.com/v2/alerts/conditions'
     ENTITY_CONDITIONS_URL = 'https://api.newrelic.com/v2/alerts_entity_conditions'
+    ALERT_VIOLATIONS_URL = 'https://api.newrelic.com/v2/alerts_violations.json'
+
 
 class EUEndpoints:
 
@@ -88,3 +90,4 @@ class EUEndpoints:
     INFRA_CONDITIONS_URL = 'https://infra-api.eu.newrelic.com/v2/alerts/conditions'
     CREATE_INFRA_CONDITION_URL = 'https://infra-api.eu.newrelic.com/v2/alerts/conditions'
     ENTITY_CONDITIONS_URL = 'https://api.eu.newrelic.com/v2/alerts_entity_conditions'
+    ALERT_VIOLATIONS_URL = 'https://api.eu.newrelic.com/v2/alerts_violations.json'

library/clients/violationsclient.py

@@ -1,19 +1,17 @@
-import requests
 import os
-import json
 import library.migrationlogger as migrationlogger
 import library.utils as utils
+from library.clients.endpoints import Endpoints
 
 logger = migrationlogger.get_logger(os.path.basename(__file__))
 
-ALERT_VIOLATIONS_URL = 'https://api.newrelic.com/v2/alerts_violations.json'
 VIOLATIONS = 'violations'
 
 
 def setup_headers(api_key):
     return {'Api-Key': api_key, 'Content-Type': 'Application/JSON'}
 
 
-def get_all_alert_violations(api_key, start_date, end_date, only_open):
+def get_all_alert_violations(api_key, start_date, end_date, only_open=False, region=Endpoints.REGION_US):
     params = {'start_date': start_date, 'end_date': end_date, 'only_open': only_open}
-    return utils.get_paginated_entities(api_key, ALERT_VIOLATIONS_URL, VIOLATIONS, params)
+    return utils.get_paginated_entities(api_key, Endpoints.of(region).ALERT_VIOLATIONS_URL, VIOLATIONS, params)

library/localstore.py

@@ -2,7 +2,7 @@
 import json
 import os
 import csv
-import datetime
+from datetime import datetime
 import library.monitortypes as monitortypes
 import library.migrationlogger as migrationlogger
 import library.windows_names as win_names
@@ -170,9 +170,11 @@ def load_synth_conditions(account_id):
 def load_alert_policies(account_id):
     return load_json_file(account_id, ALERT_POLICIES_DIR, ALERT_POLICIES_FILE)
 
+
 def load_alert_policy_entity_map(account_id):
     return load_json_file(account_id, ALERT_POLICIES_DIR, ALERT_POLICY_ENTITY_MAP_FILE)
 
+
 def load_alert_channels(account_id):
     return load_json_file(account_id, ALERT_POLICIES_DIR, ALERT_CHANNELS_FILE)
 
@@ -272,6 +274,7 @@ def convert_timestamps_to_dates(violation):
         violation['closed_at'] = closed_at_date
     return violation
 
+
 #  db/<account_id>/alert_policies/alerts_channels.json
 def save_alert_channels(account_id, all_alert_channels):
     base_dir = Path("db")

replicatemonitors.py

@@ -7,6 +7,7 @@
 import library.monitortypes as monitortypes
 import library.clients.monitorsclient as monitorsclient
 
+# NOT SUPPORTED - DO NOT USE
 # TESTING only replicatemonitors must be used after doing a fetchmonitors
 # specify the source account, timestamp that you want to migrate
 # Also specify the targetAccount and targetApiKey to which you want to migrate the stored monitors

store_policies.py

@@ -13,29 +13,40 @@
 log = m_logger.get_logger(os.path.basename(__file__))
 
 
-def setup_params():
+def configure_parser():
+    parser = argparse.ArgumentParser(description='Store Alert Policies and channels')
     parser.add_argument('--sourceAccount', nargs=1, type=str, required=True, help='Source accountId')
     parser.add_argument('--sourceApiKey', nargs=1, type=str, required=True, help='Source account API Key or \
                                                                         set environment variable ENV_SOURCE_API_KEY')
+    parser.add_argument('--sourceRegion', type=str, nargs=1, required=False, help='sourceRegion us(default) or eu')
+    return parser
 
 
-def print_args(src_api_key, tgt_api_key):
+def print_args(args, src_api_key, src_region):
     log.info("Using sourceAccount : " + str(args.sourceAccount[0]))
-    log.info("Using sourceApiKey : " + len(src_api_key[:-4])*"*"+src_api_key[-4:])
+    log.info("Using sourceApiKey : " + len(src_api_key[:-4])*"*" + src_api_key[-4:])
+    if args.sourceRegion and len(args.sourceRegion) > 0:
+        log.info("sourceRegion : " + args.sourceRegion[0])
+    else:
+        log.info("sourceRegion not passed : Defaulting to " + src_region)
 
 
-def store_alert_policies(src_account, src_api_key):
+def store_alert_policies(src_account, src_api_key, src_region):
     log.info('Starting store alert policies.')
-    policies = ac.get_all_alert_policies(src_api_key)
+    policies = ac.get_all_alert_policies(src_api_key, src_region)
     store.save_alert_policies(src_account, policies)
     log.info('Finished store alert policies.')
 
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Store Alert Policies and channels')
-    setup_params()
+def main():
+    parser = configure_parser()
     args = parser.parse_args()
-    source_api_key = utils.ensure_source_api_key(args)
-    if not source_api_key:
+    src_api_key = utils.ensure_source_api_key(args)
+    src_region = utils.ensure_source_region(args)
+    if not src_api_key:
         utils.error_and_exit('source_api_key', 'ENV_SOURCE_API_KEY')
-    store_alert_policies(args.sourceAccount[0], source_api_key)
+    store_alert_policies(args.sourceAccount[0], src_api_key, src_region)
+
+
+if __name__ == '__main__':
+    main()