feat: Support EU Endpoints (#22)

Added support for EU endpoints for most scripts.

README.md

@@ -120,7 +120,7 @@ APM Configuration
 ####  1) python3 fetchmonitors.py 
 
 ```
-usage: fetchmonitors.py [-h] --sourceAccount SOURCEACCOUNT
+usage: fetchmonitors.py --sourceAccount SOURCEACCOUNT --region [ us (default) |eu ]
                     --sourceApiKey SOURCEAPIKEY  
                     --insightsQueryKey INSIGHTSQUERYKEY
                     --toFile TOFILE
@@ -129,6 +129,7 @@ usage: fetchmonitors.py [-h] --sourceAccount SOURCEACCOUNT
 Parameter        | Note
 ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------
 sourceAccount    | Account to fetch monitors from
+region           | Optional region us (default) or eu
 sourceApiKey     | This should be a User API Key for sourceAccount for a user with admin (or add on / custom role equivalent) access to Synthetics
 insightsQueryKey | must be supplied to fetch secure credentials from Insights for any monitors that ran in the past 7 days. Secure credentials fetching is skipped if this is not passed.
 toFile           | should only be a file name e.g. soure-monitors.csv. It will always be created in output/ directory
@@ -139,7 +140,7 @@ toFile           | should only be a file name e.g. soure-monitors.csv. It will a
 
 ####  3) python3 fetchchannels.py (optional if you want to use --useLocal option during migratepolicies)
 
-`usage: fetchalerts.py [-h] --sourceAccount SOURCEACCOUNT [--sourceApiKey SOURCEAPIKEY]`
+`usage: fetchchannels.py  --sourceAccount SOURCEACCOUNT [--sourceApiKey SOURCEAPIKEY] --region [ us (default) |eu ]`
 
 Fetches alert channels and builds a dictionary mapping channels to policy_id.
 
@@ -149,14 +150,16 @@ During migratepolicies the stored alert_channels can be used by passing --useLoc
 
 ####  4) python3 migratemonitors.py
 
-`usage: migratemonitors.py [-h] --fromFile FROMFILE --sourceAccount SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY --targetAccount TARGETACCOUNT [--targetApiKey TARGETAPIKEY] --timeStamp TIMESTAMP [--useLocal]`
+`usage: migratemonitors.py --fromFile FROMFILE --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION] --sourceApiKey SOURCEAPIKEY --targetAccount TARGETACCOUNT  [--targetRegion TARGETREGION] [--targetApiKey TARGETAPIKEY] --timeStamp TIMESTAMP [--useLocal]`
 
 Parameter     | Note
 ------------- | --------------------------------------------------------------------------------------------------------
 fromFile      | Must contain monitor names one per line. The fetchentities.py script can be used to help generate this list of monitors.
 sourceAccount | Account to fetch monitors from
+sourceRegion  | Optional region us (default) or eu
 sourceApiKey  | This should be a User API Key for sourceAccount for a user with admin (or add on / custom role equivalent) access to Synthetics
 targetAccount | Account to migrate monitors to
+targetRegion  | Optional region us (default) or eu
 targetApiKey  | This should be a User API Key for targetAccount for a user with admin (or add on / custom role equivalent) access to Synthetics
 timeStamp     | must match the timeStamp generated in fetchmonitors , used when useLocal flag is passed
 useLocal      | By default monitors are fetched from sourceAccount. A pre-fetched copy can be used by passing this flag.
@@ -179,16 +182,18 @@ A value of 0 CHECK_COUNT for scripted monitors indicates it has not run in the p
 
 ####  5) python3 migratepolicies.py
 
-`usage: migratepolicies.py [-h] --fromFile FROMFILE --sourceAccount SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY --targetAccount TARGETACCOUNT [--targetApiKey TARGETAPIKEY] [--useLocal]`
+`usage: migratepolicies.py --fromFile FROMFILE --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION] --sourceApiKey SOURCEAPIKEY --targetAccount TARGETACCOUNT [--targetRegion TARGETREGION] [--targetApiKey TARGETAPIKEY] [--useLocal]`
 
 Parameter        | Note
 ---------------- | ------------------------------------------------------------------------------------------------------
 fromFile         | must contain alert policy names one per line
 fromFileEntities | must contain APM, Browser, or Mobile application names or IDs or APM KT names or IDs (not GUIDs)
 personalApiKey   | Personal API Key used for GraphQL API Client calls
 sourceAccount    | Account to fetch monitors from
+sourceRegion     | Optional region us (default) or eu
 sourceApiKey     | User API Key for sourceAccount for a user with admin (or add on / custom role equivalent) access to Alerts
 targetAccount    | Account to migrate policies to
+targetRegion     | Optional region us (default) or eu
 targetApiKey     | User API Key for targetAccount for a user with admin (or add on / custom role equivalent) access to Alerts
 useLocal         | By alert channels are fetched from sourceAccount. A pre-fetched copy can be used by passing this flag.
 
@@ -261,16 +266,18 @@ to move will be the union of both.
 
 Any target APM , Browser, Mobile apps and Key transactions must be migrated manually.
 
-`usage: migrateconditions.py [-h] --fromFile FROMFILE --personalApiKey PERSONALAPIKEY --sourceAccount SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY --targetAccount TARGETACCOUNT [--targetApiKey TARGETAPIKEY] [--matchSourceState] [--synthetics --app_conditions --nrql_conditions --infra_conditions]`
+`usage: migrateconditions.py [-h] --fromFile FROMFILE --personalApiKey PERSONALAPIKEY --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION] --sourceApiKey SOURCEAPIKEY --targetAccount TARGETACCOUNT [--targetRegion TARGETREGION] [--targetApiKey TARGETAPIKEY] [--matchSourceState] [--synthetics --app_conditions --nrql_conditions --infra_conditions]`
 
 Parameter      | Note
 -------------- | --------------------------------------------------
 fromFile       | must contain alert policy names one per line
 fromFileEntities | must contain APM, Browser, or Mobile application names or IDs or APM KT names or IDs (not GUIDs)
 personalApiKey | Personal API Key used for GraphQL API Client calls
 sourceAccount  | Account to fetch monitors from
+sourceRegion   | Optional region us (default) or eu
 sourceApiKey   | User API Key for sourceAccount for a user with admin (or add on / custom role equivalent) access to Alerts
 targetAccount  | Account to migrate policies to
+targetRegion   | Optional region us (default) or eu
 targetApiKey   | User API Key for targetAccount for a user with admin (or add on / custom role equivalent) access to Alerts
 matchSourceState | Match alert condition enabled/disabled state from the source account in the target account. By default, all copied alert conditions are disabled in the target account.
 synthetics     | Pass this flag to migrate synthetic conditions
@@ -307,19 +314,19 @@ if `--app_conditions` is specified.
 
 Migrate APM Apdex configuration settings. **This no longer migrates labels.** Please use migratetags.py instead for tag migrations.
 
-usage: migrate_apm.py [-h] --fromFile FROMFILE --sourceAccount SOURCEACCOUNT
+usage: migrate_apm.py --fromFile FROMFILE --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION] 
                         --personalApiKey PERSONALAPIKEY --sourceApiKey
-                        SOURCEAPIKEY --targetAccount TARGETACCOUNT
+                        SOURCEAPIKEY --targetAccount TARGETACCOUNT [--targetRegion TARGETREGION] 
                         --targetApiKey TARGETAPIKEY  [--settings]
 
 ##### Note: Ensure target apps are running or were running recently so that the target ids can be picked
 
 
 ####  8) python3 migrate_dashboards.py
 
-usage: migrate_dashboards.py [-h] --fromFile FROMFILE --sourceAccount
+usage: migrate_dashboards.py [-h] --fromFile FROMFILE --sourceAccount [--sourceRegion SOURCEREGION] 
                              SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY
-                             --targetAccount TARGETACCOUNT
+                             --targetAccount TARGETACCOUNT [--targetRegion TARGETREGION] 
                              [--targetApiKey TARGETAPIKEY]
 
 Migrate dashboards between accounts, including modifying queries to point to the new target account. The fetchentities.py script can help create the file to pass with fromFile.
@@ -355,7 +362,7 @@ synthetics     | Pass this flag to migrate Synthetic monitor entity tags
 
 Potential use is for renaming/disabling migrated monitors in source account.
 
-`usage: updatemonitors.py [-h] --fromFile FROMFILE [--targetApiKey TARGETAPIKEY] --targetAccount TARGETACCOUNT --timeStamp TIMESTAMP [--renamePrefix RENAMEPREFIX] [--disable]`
+`usage: updatemonitors.py [-h] --fromFile FROMFILE [--targetApiKey TARGETAPIKEY] --targetAccount TARGETACCOUNT [--targetRegion TARGETREGION] --timeStamp TIMESTAMP [--renamePrefix RENAMEPREFIX] [--disable]`
 
 Parameter     | Note
 ------------- | -------------------------------------------------------------------------
@@ -405,15 +412,15 @@ synthetics     | Pass this flag to list Synthetic monitor entities
 
 ####  12) python3 deletemonitors.py
 
-`usage: deletemonitors.py [-h] --fromFile FROMFILE [--targetApiKey TARGETAPIKEY] --targetAccount TARGETACCOUNT --timeStamp TIMESTAMP`
+`usage: deletemonitors.py [-h] --fromFile FROMFILE [--targetApiKey TARGETAPIKEY] --targetAccount TARGETACCOUNT [--targetRegion TARGETREGION] --timeStamp TIMESTAMP`
 
 Will delete monitors listed one per line in --fromFile and stored in db/targetaccount/monitors/timeStamp. The fetchentities.py script can help generate this file.
 
 ####  13) (optional Testing purpose only) python3 deleteallmonitors.py
 
 #### Warning: All monitors in target account will be deleted
 
-`usage: deleteallmonitors.py [-h] [--targetApiKey TARGETAPIKEY] --targetAccount TARGETACCOUNT`
+`usage: deleteallmonitors.py [-h] [--targetApiKey TARGETAPIKEY] --targetAccount TARGETACCOUNT [--targetRegion TARGETREGION]`
 
 deleteallmonitors fetches all the monitors. Backs them up in db/accountId/monitors/timeStamp-bakup And deletes all the monitors
 

config.ini.example

@@ -1,7 +1,27 @@
 [default]
-personal_api_key: 123456
+
 
 [migrate.policies]
-policies_file: /tmp/policies.txt
+soure_account: 12345
+source_region: us
+source_api_key: ASFASFDASFSADF
+target_account: 98765
+target_region: us
+target_api_key: ASFASFASFASFASF
+policies_file: output/us-policies.txt
 entities_file: /tmp/entities.txt
 
+[migrate.conditions]
+source_account_id: 12345
+source_region: us
+source_api_key: ASDASFDSAFASFD
+target_account_id: 98765
+target_region: us
+target_api_key: DFDFSDFSDFDF
+policy_file: output/us-policies.txt
+all: true
+app_conditions: true
+synthetics: true
+nrql_conditions: true
+ext_svc_conditions: true
+infra_conditions: true

deleteallmonitors.py

@@ -6,6 +6,8 @@
 import library.localstore as localstore
 import library.migrationlogger as migrationlogger
 import library.clients.monitorsclient as monitorsclient
+import library.utils as utils
+from library.clients.endpoints import Endpoints
 
 
 # This script has been built for TESTING purpose only. Use at your own risk.
@@ -15,60 +17,35 @@
 # Example use same account id for source and target
 # deleteallmonitors deletes all the monitors in the targetAccount
 logger = migrationlogger.get_logger(os.path.basename(__file__))
-headers = {}
-monitors_url = 'https://synthetics.newrelic.com/synthetics/api/v3/monitors/'
-parser = argparse.ArgumentParser(description='Delete Monitors from an account')
-from_api_key = ""
 
 
-def setup_params():
+def configure_parser():
+    parser = argparse.ArgumentParser(description='Delete Monitors from an account')
     parser.add_argument('--targetApiKey', type=str, nargs=1, required=False, help='API Key for the account')
     parser.add_argument('--targetAccount', type=str, nargs=1, required=True, help='Target account')
+    parser.add_argument('--region', type=str, nargs=1, required=False, help='region us(default) or eu')
+    return parser
 
 
-def print_args():
+def print_args(args, region):
     logger.info("Using targetApiKey : " + len(args.targetApiKey[0][:-4])*"*"+args.targetApiKey[0][-4:])
     logger.info("Using targetAccount : " + str(args.targetAccount[0]))
-
-
-def setup_headers(api_key):
-    if api_key:
-        target_api_key = api_key
+    if args.region and len(args.region) > 0:
+        logger.info("region : " + args.region[0])
     else:
-        target_api_key = os.environ.get('ENV_TARGET_API_KEY')
-    headers['Api-Key'] = target_api_key
-    if not headers['Api-Key']:
-        logger.error('Error: Missing API Key. either pass as param ---targetApiKey or \
-            environment variable ENV_TARGET_API_KEY.\n \
-            e.g. export ENV_TARGET_API_KEY="NRNA7893asdfhkh"')
-        sys.exit()
+        logger.info("region not passed : Defaulting to " + region)
 
 
-def delete(monitors, target_acct):
+def delete(monitors, target_acct, tgt_api_key, region):
     success_status = {}
     failure_status = {}
     for monitor in monitors:
-        delete_monitor(monitor, target_acct, failure_status,  success_status)
+        monitorsclient.delete_monitor(monitor, target_acct, failure_status,  success_status, tgt_api_key, region)
     return {'success': success_status, 'failure': failure_status}
 
 
-def delete_monitor(monitor, target_acct, failure_status, success_status):
-    monitor_id = monitor['id']
-    monitor_name = monitor['name']
-    response = requests.delete(monitors_url + monitor_id, headers=headers)
-    if response.status_code == 204:
-        success_status[monitor_name] = {'status': response.status_code, 'responseText': response.text}
-        logger.info(target_acct + ":" + monitor_name + ":" + str(success_status[monitor_name]))
-    else:
-        failure_status[monitor_name] = {'status': response.status_code, 'responseText': response.text}
-        logger.info(target_acct + ":" + monitor_name + ":" + str(failure_status[monitor_name]))
-    # trying to stay within 3 requests per second
-    time.sleep(0.3)
-
-
-def delete_all_monitors(api_key, target_acct):
-    setup_headers(api_key)
-    all_monitors_def_json = monitorsclient.fetch_all_monitors(api_key)
+def delete_all_monitors(api_key, target_acct, region):
+    all_monitors_def_json = monitorsclient.fetch_all_monitors(api_key, region)
     timestamp = time.strftime("%Y-%m%d-%H%M%S") + "-bakup"
     storage_dir = localstore.create_storage_dirs(target_acct, timestamp)
     monitor_names_file = localstore.create_output_file("monitors-" + timestamp + ".csv")
@@ -79,14 +56,20 @@ def delete_all_monitors(api_key, target_acct):
             monitor_names_out.write(monitor_name + "\n")
             localstore.save_monitor_to_file(monitor_name, storage_dir, monitor_json)
     logger.info("Backed up %d monitors in %s before deleting", len(all_monitors_def_json), storage_dir)
-    del_response = delete(all_monitors_def_json, target_acct)
+    del_response = delete(all_monitors_def_json, target_acct, api_key, region)
     logger.debug(del_response)
 
 
-if __name__ == '__main__':
+def main():
     start_time = time.time()
-    setup_params()
+    parser = configure_parser()
     args = parser.parse_args()
-    print_args()
-    delete_all_monitors(args.targetApiKey[0], args.targetAccount[0])
+    region = utils.ensure_region(args)
+    tgt_api_key = utils.ensure_target_api_key(args)
+    print_args(args, region)
+    delete_all_monitors(tgt_api_key, args.targetAccount[0], region)
     logger.info("Time taken : " + str(time.time() - start_time) + " seconds.")
+
+
+if __name__ == '__main__':
+    main()

deletemonitors.py

@@ -5,75 +5,65 @@
 import time
 import library.localstore as store
 import library.migrationlogger as m_logger
+import library.utils as utils
+import library.clients.monitorsclient as monitorsclient
 
 
 # deletemonitors deletes a list of monitors provided in one per line in a csv file
 # monitors must have been pre-feteched using fetchmonitors and stored in db/<target_account_id>/monitors/<time_stamp>
 # The fromFile, targetAccountId, targetApiKey and timeStamp must be specified
 logger = m_logger.get_logger(os.path.basename(__file__))
 headers = {}
-monitors_url = 'https://synthetics.newrelic.com/synthetics/api/v3/monitors/'
-parser = argparse.ArgumentParser(description='Delete Monitors from an account')
 from_api_key = ""
 
 
-def setup_params():
+def configure_parser():
+    parser = argparse.ArgumentParser(description='Delete Monitors from an account')
     parser.add_argument('--fromFile', nargs=1, required=True, help='Path to file with monitor names, one per line')
     parser.add_argument('--targetApiKey', nargs=1, required=False, help='API Key for the account')
     parser.add_argument('--targetAccount', nargs=1, required=True, help='Target account')
     parser.add_argument('--timeStamp', nargs=1, required=True, help='Timestamp of the pre-fetched monitors')
+    parser.add_argument('--region', type=str, nargs=1, required=False, help='region us(default) or eu')
+    return parser
 
 
-def print_args():
+def print_args(args, region):
     logger.info("Using fromFile : " + args.fromFile[0])
     logger.info("Using targetApiKey : " + args.targetApiKey[0])
     logger.info("Using targetAccount : " + str(args.targetAccount[0]))
     logger.info("Using timeStamp : " + args.timeStamp[0])
-
-
-def setup_headers():
-    if args.targetApiKey:
-        target_api_key = args.targetApiKey[0]
+    if args.region and len(args.region) > 0:
+        logger.info("region : " + args.region[0])
     else:
-        target_api_key = os.environ.get('ENV_TARGET_API_KEY')
-    headers['Api-Key'] = target_api_key
-    if not headers['Api-Key']:
-        logger.error('Error: Missing API Key. either pass as param ---targetApiKey or \
-            environment variable ENV_TARGET_API_KEY.\n \
-            e.g. export ENV_TARGET_API_KEY="NRNA7893asdfhkh"')
-        sys.exit()
+        logger.info("region not passed : Defaulting to " + region)
 
 
-def delete(monitors):
+def delete(monitor_definitions, target_account, tgt_api_key, region):
     success_status = {}
     failure_status = {}
-    for monitor in monitors:
-        monitor_id = monitor['definition']['id']
-        monitor_name = monitor['definition']['name']
-        target_account = str(args.targetAccount[0])
-        response = requests.delete(monitors_url + monitor_id, headers=headers)
-        if response.status_code == 204:
-            success_status[monitor_name] = {'status': response.status_code, 'responseText': response.text}
-            logger.info(target_account + ":" + monitor_name + ":" + str(success_status[monitor_name]))
-        else:
-            failure_status[monitor_name] = {'status': response.status_code, 'responseText': response.text}
-            logger.info(target_account + ":" + monitor_name + ":" + str(failure_status[monitor_name]))
-        # trying to stay within 3 requests per second
-        time.sleep(0.5)
+    for monitor_definition in monitor_definitions:
+        monitorsclient.delete_monitor(monitor_definition['definition'], target_account, failure_status, success_status,
+                                      tgt_api_key, region)
     return {'success': success_status, 'failure': failure_status}
 
 
-def delete_monitors():
-    monitor_names = store.load_names(args.fromFile[0])
-    monitors = store.load_monitors(args.targetAccount[0], args.timeStamp[0], monitor_names)
-    del_response = delete(monitors)
+def delete_monitors(from_file, tgt_account, time_stamp, tgt_api_key, region):
+    monitor_names = store.load_names(from_file)
+    monitor_definitions = store.load_monitors(tgt_account, time_stamp, monitor_names)
+    del_response = delete(monitor_definitions, tgt_account, tgt_api_key, region)
     logger.debug(del_response)
 
 
-start_time = time.time()
-setup_params()
-args = parser.parse_args()
-setup_headers()
-print_args()
-delete_monitors()
-logger.info("Time taken : " + str(time.time() - start_time) + " seconds.")
+def main():
+    start_time = time.time()
+    parser = configure_parser()
+    args = parser.parse_args()
+    region = utils.ensure_region(args)
+    tgt_api_key = utils.ensure_target_api_key(args)
+    print_args(args, region)
+    delete_monitors(args.fromFile[0], args.targetAccount[0], args.timeStamp[0], tgt_api_key, region)
+    logger.info("Time taken : " + str(time.time() - start_time) + " seconds.")
+
+
+if __name__ == '__main__':
+    main()