diff --git a/README.md b/README.md index a239ab6..051e39f 100644 --- a/README.md +++ b/README.md @@ -334,8 +334,8 @@ Migrate dashboards between accounts, including modifying queries to point to the #### 9) python3 migratetags.py usage: migratetags.py [-h] --fromFile FROMFILE --sourceAccount - SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY - --targetAccount TARGETACCOUNT --targetApiKey TARGETAPIKEY + SOURCEACCOUNT [--sourceRegion SOURCEREGION] --sourceApiKey SOURCEAPIKEY + --targetAccount TARGETACCOUNT [--targetRegion TARGETREGION] --targetApiKey TARGETAPIKEY [--apm --browser --dashboards --infrahost --infraint --lambda --mobile --securecreds --synthetics] Migrate entity tags between entities with matching names and entity types. @@ -344,8 +344,10 @@ Parameter | Note -------------- | -------------------------------------------------- fromFile | Must contain entity names one per line. The fetchentities.py script can help create this file. sourceAccount | Account to search for a matching source entity +sourceRegion | Optional region us (default) or eu sourceApiKey | User API Key for sourceAccount targetAccount | Account to search for a matching target entity +targetRegion | Optional region us (default) or eu targetApiKey | User API Key for targetAccount for a user with admin (or add on / custom role equivalent) access to Alerts apm | Pass this flag to migrate APM entity tags browser | Pass this flag to migrate Browser entity tags @@ -368,6 +370,7 @@ Parameter | Note ------------- | ------------------------------------------------------------------------- fromFile | Specifies the file with relative path, listing the monitors to be updated. The fetchentities.py script can help generate this file. targetAccount | Account in which monitors need to be updated +targetRegion | Optional region us (default) or eu targetApiKey | This should be a User API Key for targetAccount for a user with admin (or add on / custom role equivalent) access to Synthetics timeStamp | must match the timeStamp generated in fetchmonitors renamePrefix | Monitors are renamed with this prefix @@ -386,7 +389,7 @@ output/targetAccount_fromFile_updated_monitors.csv #### 11) python3 fetchentities.py -usage: fetchentities.py [-h] --sourceAccount SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY +usage: fetchentities.py [-h] --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION] --sourceApiKey SOURCEAPIKEY --toFile FILENAME [--tagName TAGNAME --tagValue TAGVALUE] [--apm --browser --dashboards --infrahost --infraint --lambda --mobile --securecreds --synthetics] @@ -395,6 +398,7 @@ Create a file in the output directory that contains entity names from the source Parameter | Note -------------- | -------------------------------------------------- sourceAccount | Account to search for matching entities +sourceRegion | Optional region us (default) or eu sourceApiKey | User API Key for sourceAccount toFile | File name to use to store entity names. This file will be created in the output directory. tagName | Tag name to use to filter results @@ -428,15 +432,17 @@ deleteallmonitors fetches all the monitors. Backs them up in db/accountId/monito #### 14) (optional) python3 store_policies.py -usage: store_policies.py [-h] --sourceAccount SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY +usage: store_policies.py [-h] --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION] --sourceApiKey SOURCEAPIKEY Saves all alert polices in db//alert_policies/alert_policies.json #### 15) (optional) python3 store_violations.py -usage: store_violations.py [-h] --sourceAccount SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY --startDate STARTDATE --endDate ENDDATE [--onlyOpen] +usage: store_violations.py [-h] --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION] --sourceApiKey SOURCEAPIKEY --startDate STARTDATE --endDate ENDDATE [--onlyOpen] --sourceAccount SOURCEACCOUNT Source accountId + + --sourceRegion' SOURCEREGION us (default) or eu --sourceApiKey SOURCEAPIKEY Source account API Key or set environment variable ENV_SOURCE_API_KEY @@ -452,7 +458,7 @@ and db//alert_violations/alert_violations.csv #### 16) (optional) python3 store_policy_entity_map.py -usage: store_policy_entity_map.py [-h] --sourceAccount SOURCEACCOUNT --sourceApiKey SOURCEAPIKEY --useLocal +usage: store_policy_entity_map.py [-h] --sourceAccount SOURCEACCOUNT [--sourceRegion SOURCEREGION] --sourceApiKey SOURCEAPIKEY --useLocal Builds a mapping from APM, Browser, and Mobile applications and APM key transactions to and from alert policies for any policies which contain @@ -476,6 +482,9 @@ Verification of test is also manual at the moment. We encourage your contributions to improve nr-account-migration! Keep in mind when you submit your pull request, you'll need to sign the CLA via the click-through using CLA-Assistant. You only have to sign the CLA one time per project. If you have any questions, or to execute our corporate CLA, required if your contribution is on behalf of a company, please drop us an email at opensource@newrelic.com. +### Style Guide +Use PEP 8 Style Guide for Python Code https://www.python.org/dev/peps/pep-0008/ + **A note about vulnerabilities** As noted in our [security policy](../../security/policy), New Relic is committed to the privacy and security of our customers and their data. We believe that providing coordinated disclosure by security researchers and engaging with the security community are important means to achieve our security goals. diff --git a/fetchentities.py b/fetchentities.py index 75e1ccc..75e4854 100644 --- a/fetchentities.py +++ b/fetchentities.py @@ -8,15 +8,14 @@ import library.utils as utils logger = migrationlogger.get_logger(os.path.basename(__file__)) -args = None -def setup_params(): +def configure_parser(): parser = argparse.ArgumentParser(description='Migrate entity tags from one account to another') parser.add_argument('--sourceAccount', nargs=1, required=True, help='Source accountId') parser.add_argument('--sourceApiKey', nargs=1, required=False, help='Source API Key or \ set env var ENV_SOURCE_API_KEY') - parser.add_argument('--region', type=str, nargs=1, required=False, help='region us(default) or eu') + parser.add_argument('--sourceRegion', type=str, nargs=1, required=False, help='region us(default) or eu') parser.add_argument('--toFile', nargs=1, required=True, help='File to populate entity names. ' 'This will be created in output directory') parser.add_argument('--synthetics', dest='synthetics', required=False, action='store_true', help='Pass --synthetics to list matching Synthetic monitor entities') @@ -33,13 +32,13 @@ def setup_params(): return parser -def print_params(args, source_api_key, entity_types, region): +def print_params(args, source_api_key, entity_types, src_region): logger.info("Using sourceAccount : " + str(args.sourceAccount[0])) logger.info("Using sourceApiKey : " + len(source_api_key[:-4])*"*"+source_api_key[-4:]) - if args.region and len(args.region) > 0: - logger.info("region : " + args.region[0]) + if args.sourceRegion and len(args.sourceRegion) > 0: + logger.info("region : " + args.sourceRegion[0]) else: - logger.info("region not passed : Defaulting to " + region) + logger.info("region not passed : Defaulting to " + src_region) logger.info("Using entity types : " + str(entity_types)) if args.tagName: logger.info("Using tag name " + str(args.tagName[0]) + " and tag value " + str(args.tagValue[0])) @@ -70,10 +69,11 @@ def parse_entity_types(args): return entity_types -def fetch_entities(src_account_id, src_api_key, entity_types, output_file, *, tag_name=None, tag_value=None, region='us'): +def fetch_entities(src_account_id, src_api_key, entity_types, output_file, *, + tag_name=None, tag_value=None, src_region='us'): entity_names = [] for entity_type in entity_types: - entities = ec.gql_get_entities_by_type(src_api_key, entity_type, src_account_id, tag_name, tag_value, region) + entities = ec.gql_get_entities_by_type(src_api_key, entity_type, src_account_id, tag_name, tag_value, src_region) for entity in entities['entities']: entity_names.append(entity['name']) entity_names_file = store.create_output_file(output_file) @@ -86,34 +86,29 @@ def fetch_entities(src_account_id, src_api_key, entity_types, output_file, *, ta def main(): - parser = setup_params() + parser = configure_parser() args = parser.parse_args() - - source_api_key = utils.ensure_source_api_key(args) - if not source_api_key: + src_api_key = utils.ensure_source_api_key(args) + if not src_api_key: utils.error_and_exit('source api key', 'ENV_SOURCE_API_KEY') - entity_types = parse_entity_types(args) if len(entity_types) == 0: logger.error('At least one entity type must be specified. Currently supported: ' + ec.SYNTH_MONITOR + ',' + ec.SYNTH_SECURE_CRED + ',' + ec.APM_APP + ',' + ec.BROWSER_APP + ',' + ec.DASHBOARDS + ',' + ec.INFRA_HOST + ',' + ec.INFRA_INT + ',' + ec.MOBILE_APP + ',' + ec.INFRA_LAMBDA) sys.exit() - if args.tagName is not None and args.tagValue is None: logger.error('tagValue is required when tagName is set') sys.exit() - if args.tagValue is not None and args.tagName is None: logger.error('tagName is required when tagValue is set') sys.exit() - region = utils.ensure_region(args) - print_params(args, source_api_key, entity_types, region) - + src_region = utils.ensure_source_region(args) + print_params(args, src_api_key, entity_types, src_region) if args.tagName is None: - fetch_entities(args.sourceAccount[0], source_api_key, entity_types, args.toFile[0], region=region) + fetch_entities(args.sourceAccount[0], src_api_key, entity_types, args.toFile[0], src_region=src_region) else: - fetch_entities(args.sourceAccount[0], source_api_key, entity_types, args.toFile[0], tag_name=args.tagName[0], - tag_value=args.tagValue[0], region=region) + fetch_entities(args.sourceAccount[0], src_api_key, entity_types, args.toFile[0], tag_name=args.tagName[0], + tag_value=args.tagValue[0], src_region=src_region) if __name__ == '__main__': diff --git a/library/clients/alertsclient.py b/library/clients/alertsclient.py index 1ca0aa1..f7a5d23 100644 --- a/library/clients/alertsclient.py +++ b/library/clients/alertsclient.py @@ -356,7 +356,7 @@ def get_alert_status_file_name(fromFile, fromFileEntities, src_account_id, tgt_a return status_file_name + str(tgt_account_id) + '_conditions.csv' -def get_policy_entity_map(api_key, alert_policies): +def get_policy_entity_map(api_key, alert_policies, region=Endpoints.REGION_US): entities_by_policy = {} policies_by_entity = {} for policy in alert_policies: @@ -364,7 +364,7 @@ def get_policy_entity_map(api_key, alert_policies): policy_name = policy['name'] apps = [] logger.info('Loading app entity conditions for policy ID %d...' % policy_id) - conditions = get_app_conditions(api_key, policy_id) + conditions = get_app_conditions(api_key, policy_id, region) if not 'response_count' in conditions or conditions['response_count'] == 0: logger.info('No app entity conditions found for policy ID %d' % policy_id) entities_by_policy[policy_name] = [] diff --git a/library/clients/endpoints.py b/library/clients/endpoints.py index 28f9718..c34857b 100644 --- a/library/clients/endpoints.py +++ b/library/clients/endpoints.py @@ -52,6 +52,8 @@ class USEndpoints: INFRA_CONDITIONS_URL = 'https://infra-api.newrelic.com/v2/alerts/conditions' CREATE_INFRA_CONDITION_URL = 'https://infra-api.newrelic.com/v2/alerts/conditions' ENTITY_CONDITIONS_URL = 'https://api.newrelic.com/v2/alerts_entity_conditions' + ALERT_VIOLATIONS_URL = 'https://api.newrelic.com/v2/alerts_violations.json' + class EUEndpoints: @@ -88,3 +90,4 @@ class EUEndpoints: INFRA_CONDITIONS_URL = 'https://infra-api.eu.newrelic.com/v2/alerts/conditions' CREATE_INFRA_CONDITION_URL = 'https://infra-api.eu.newrelic.com/v2/alerts/conditions' ENTITY_CONDITIONS_URL = 'https://api.eu.newrelic.com/v2/alerts_entity_conditions' + ALERT_VIOLATIONS_URL = 'https://api.eu.newrelic.com/v2/alerts_violations.json' diff --git a/library/clients/violationsclient.py b/library/clients/violationsclient.py index 2fcd44d..b61fa7e 100644 --- a/library/clients/violationsclient.py +++ b/library/clients/violationsclient.py @@ -1,12 +1,10 @@ -import requests import os -import json import library.migrationlogger as migrationlogger import library.utils as utils +from library.clients.endpoints import Endpoints logger = migrationlogger.get_logger(os.path.basename(__file__)) -ALERT_VIOLATIONS_URL = 'https://api.newrelic.com/v2/alerts_violations.json' VIOLATIONS = 'violations' @@ -14,6 +12,6 @@ def setup_headers(api_key): return {'Api-Key': api_key, 'Content-Type': 'Application/JSON'} -def get_all_alert_violations(api_key, start_date, end_date, only_open): +def get_all_alert_violations(api_key, start_date, end_date, only_open=False, region=Endpoints.REGION_US): params = {'start_date': start_date, 'end_date': end_date, 'only_open': only_open} - return utils.get_paginated_entities(api_key, ALERT_VIOLATIONS_URL, VIOLATIONS, params) + return utils.get_paginated_entities(api_key, Endpoints.of(region).ALERT_VIOLATIONS_URL, VIOLATIONS, params) diff --git a/library/localstore.py b/library/localstore.py index 60cdc18..69d5ff9 100644 --- a/library/localstore.py +++ b/library/localstore.py @@ -2,7 +2,7 @@ import json import os import csv -import datetime +from datetime import datetime import library.monitortypes as monitortypes import library.migrationlogger as migrationlogger import library.windows_names as win_names @@ -170,9 +170,11 @@ def load_synth_conditions(account_id): def load_alert_policies(account_id): return load_json_file(account_id, ALERT_POLICIES_DIR, ALERT_POLICIES_FILE) + def load_alert_policy_entity_map(account_id): return load_json_file(account_id, ALERT_POLICIES_DIR, ALERT_POLICY_ENTITY_MAP_FILE) + def load_alert_channels(account_id): return load_json_file(account_id, ALERT_POLICIES_DIR, ALERT_CHANNELS_FILE) @@ -272,6 +274,7 @@ def convert_timestamps_to_dates(violation): violation['closed_at'] = closed_at_date return violation + # db//alert_policies/alerts_channels.json def save_alert_channels(account_id, all_alert_channels): base_dir = Path("db") diff --git a/replicatemonitors.py b/replicatemonitors.py index ce2443a..d0a49aa 100644 --- a/replicatemonitors.py +++ b/replicatemonitors.py @@ -7,6 +7,7 @@ import library.monitortypes as monitortypes import library.clients.monitorsclient as monitorsclient +# NOT SUPPORTED - DO NOT USE # TESTING only replicatemonitors must be used after doing a fetchmonitors # specify the source account, timestamp that you want to migrate # Also specify the targetAccount and targetApiKey to which you want to migrate the stored monitors diff --git a/store_policies.py b/store_policies.py index e34d6ec..1d71273 100644 --- a/store_policies.py +++ b/store_policies.py @@ -13,29 +13,40 @@ log = m_logger.get_logger(os.path.basename(__file__)) -def setup_params(): +def configure_parser(): + parser = argparse.ArgumentParser(description='Store Alert Policies and channels') parser.add_argument('--sourceAccount', nargs=1, type=str, required=True, help='Source accountId') parser.add_argument('--sourceApiKey', nargs=1, type=str, required=True, help='Source account API Key or \ set environment variable ENV_SOURCE_API_KEY') + parser.add_argument('--sourceRegion', type=str, nargs=1, required=False, help='sourceRegion us(default) or eu') + return parser -def print_args(src_api_key, tgt_api_key): +def print_args(args, src_api_key, src_region): log.info("Using sourceAccount : " + str(args.sourceAccount[0])) - log.info("Using sourceApiKey : " + len(src_api_key[:-4])*"*"+src_api_key[-4:]) + log.info("Using sourceApiKey : " + len(src_api_key[:-4])*"*" + src_api_key[-4:]) + if args.sourceRegion and len(args.sourceRegion) > 0: + log.info("sourceRegion : " + args.sourceRegion[0]) + else: + log.info("sourceRegion not passed : Defaulting to " + src_region) -def store_alert_policies(src_account, src_api_key): +def store_alert_policies(src_account, src_api_key, src_region): log.info('Starting store alert policies.') - policies = ac.get_all_alert_policies(src_api_key) + policies = ac.get_all_alert_policies(src_api_key, src_region) store.save_alert_policies(src_account, policies) log.info('Finished store alert policies.') -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Store Alert Policies and channels') - setup_params() +def main(): + parser = configure_parser() args = parser.parse_args() - source_api_key = utils.ensure_source_api_key(args) - if not source_api_key: + src_api_key = utils.ensure_source_api_key(args) + src_region = utils.ensure_source_region(args) + if not src_api_key: utils.error_and_exit('source_api_key', 'ENV_SOURCE_API_KEY') - store_alert_policies(args.sourceAccount[0], source_api_key) + store_alert_policies(args.sourceAccount[0], src_api_key, src_region) + + +if __name__ == '__main__': + main() diff --git a/store_policy_entity_map.py b/store_policy_entity_map.py index 205d0e6..b44b99c 100644 --- a/store_policy_entity_map.py +++ b/store_policy_entity_map.py @@ -8,49 +8,65 @@ logger = migrationlogger.get_logger(os.path.basename(__file__)) -def setup_params(parser): + +def configure_parser(): + parser = argparse.ArgumentParser( + description='Fetch and store a map from application entities to alert policy and ' + 'from alert policies to application entity') parser.add_argument('--sourceAccount', type=str, nargs=1, required=True, help='Source accountId to store the map') + parser.add_argument('--sourceRegion', type=str, nargs=1, required=False, help='sourceRegion us(default) or eu') parser.add_argument('--sourceApiKey', type=str, nargs=1, required=False, help='Source API Key or \ set env var ENV_SOURCE_API_KEY') parser.add_argument('--useLocal', dest='useLocal', required=False, action='store_true', help='By default policies are fetched. Pass this to use policies pre-fetched by store_policies.') + return parser + -def print_params(args): +def print_params(args, src_api_key, src_region): logger.info("Using sourceAccount : " + str(args.sourceAccount[0])) - logger.info("Using sourceApiKey : " + len(source_api_key[:-4]) * "*" + source_api_key[-4:]) + if args.sourceRegion and len(args.sourceRegion) > 0: + logger.info("sourceRegion : " + args.sourceRegion[0]) + else: + logger.info("sourceRegion not passed : Defaulting to " + src_region) + logger.info("Using sourceApiKey : " + len(src_api_key[:-4]) * "*" + src_api_key[-4:]) logger.info("Using useLocal : " + str(args.useLocal)) + def find_policy_name(policies, policy_id): for policy in policies: if policy.id == policy_id: return policy.name -def store_policy_entity_map(src_api_key, src_account_id, use_local): + +def store_policy_entity_map(src_api_key, src_account_id, src_region, use_local): if use_local: logger.info('Loading alert policies from local...') all_policies = store.load_alert_policies(src_account_id) else: logger.info('Fetching and storing alert policies...') - all_policies = ac.get_all_alert_policies(src_api_key) - - if not 'response_count' in all_policies or all_policies['response_count'] == 0: + all_policies = ac.get_all_alert_policies(src_api_key, src_region) + if 'response_count' not in all_policies or all_policies['response_count'] == 0: logger.info('No policies found for account ID %s' % str(src_account_id)) return - - logger.info('%d policies loaded. Mapping app entity conditions for account ID %s.' % (all_policies['response_count'], src_account_id)) - - policy_entity_map = ac.get_policy_entity_map(src_api_key, all_policies['policies']) + logger.info('%d policies loaded. Mapping app entity conditions for account ID %s.' % + (all_policies['response_count'], src_account_id)) + policy_entity_map = ac.get_policy_entity_map(src_api_key, all_policies['policies'], src_region) store.save_alert_policy_entity_map(src_account_id, policy_entity_map) -if __name__ == '__main__': + +def main(): print(logger) start_time = time.time() - parser = argparse.ArgumentParser(description='Fetch and store a map from application entities to alert policy and from alert policies to application entity') - setup_params(parser) + parser = configure_parser() args = parser.parse_args() - source_api_key = utils.ensure_source_api_key(args) - if not source_api_key: + src_api_key = utils.ensure_source_api_key(args) + if not src_api_key: utils.error_and_exit('source_api_key', 'ENV_SOURCE_API_KEY') - print_params(args) - store_policy_entity_map(source_api_key, args.sourceAccount[0], args.useLocal) - logger.info("Time taken : " + str(time.time() - start_time) + "seconds") \ No newline at end of file + src_region = utils.ensure_source_region(args) + print_params(args, src_api_key, src_region) + store_policy_entity_map(src_api_key, args.sourceAccount[0], src_region, args.useLocal) + logger.info("Time taken : " + str(time.time() - start_time) + "seconds") + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/store_violations.py b/store_violations.py index a24c95e..6c7ec8b 100644 --- a/store_violations.py +++ b/store_violations.py @@ -11,11 +11,12 @@ # Alert Policy and Notification Channels are created only if not present in the targetAccount log = m_logger.get_logger(os.path.basename(__file__)) -only_open = False -def setup_params(): +def configure_parser(): + parser = argparse.ArgumentParser(description='Store Alert Policies and channels') parser.add_argument('--sourceAccount', nargs=1, type=str, required=True, help='Source accountId') + parser.add_argument('--sourceRegion', type=str, nargs=1, required=False, help='sourceRegion us(default) or eu') parser.add_argument('--sourceApiKey', nargs=1, type=str, required=True, help='Source account API Key or \ set environment variable ENV_SOURCE_API_KEY') parser.add_argument('--startDate', nargs=1, type=str, required=True, @@ -24,37 +25,42 @@ def setup_params(): help='endDate format 2020-08-04T19:18:00+00:00') parser.add_argument('--onlyOpen', dest='onlyOpen', required=False, action='store_true', help='By default all violations are fetched pass --onlyOpen to fetch only open violations') + return parser -def print_args(src_api_key, src_account, start_date, end_date): - global only_open +def print_args(src_api_key, src_account, src_region, start_date, end_date, only_open): log.info("sourceAccount : " + src_account) + log.info("sourceRegion : " + src_region) log.info("sourceApiKey : " + len(src_api_key[:-4])*"*"+src_api_key[-4:]) log.info("startDate : " + start_date) log.info("endDate : " + end_date) log.info("onlyOpen flag : " + str(only_open)) -def store_alert_violations(src_api_key, src_account, start_date, end_date): - global only_open +def store_alert_violations(src_api_key, src_account, src_region, start_date, end_date, only_open=False): log.info('Starting store alert violations.') - violations = vc.get_all_alert_violations(src_api_key, start_date, end_date, only_open) + violations = vc.get_all_alert_violations(src_api_key, start_date, end_date, only_open, src_region) store.save_alert_violations(src_account, violations) store.save_alert_violations_csv(src_account, violations) log.info('Finished store alert violations.') -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Store Alert Policies and channels') - setup_params() +def main(): + parser = configure_parser() args = parser.parse_args() - source_api_key = utils.ensure_source_api_key(args) - if not source_api_key: + src_api_key = utils.ensure_source_api_key(args) + if not src_api_key: utils.error_and_exit('source_api_key', 'ENV_SOURCE_API_KEY') + src_region = utils.ensure_source_region(args) + only_open = False if args.onlyOpen: only_open = True log.info("Using onlyOpen : " + str(args.onlyOpen)) else: log.info("Using default onlyOpen :" + str(only_open)) - print_args(source_api_key, args.sourceAccount[0], args.startDate[0], args.endDate[0]) - store_alert_violations(source_api_key, args.sourceAccount[0], args.startDate[0], args.endDate[0]) + print_args(src_api_key, args.sourceAccount[0], src_region, args.startDate[0], args.endDate[0], only_open) + store_alert_violations(src_api_key, args.sourceAccount[0], src_region, args.startDate[0], args.endDate[0], only_open) + + +if __name__ == '__main__': + main()