From 5211683dfa24eb595e5bfa3c764444bedabb6b19 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Sat, 27 Jul 2019 08:31:39 +0200 Subject: [PATCH 01/12] Create Slack integration --- ecs_deploy/cli.py | 26 ++++++++- ecs_deploy/slack.py | 137 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 ecs_deploy/slack.py diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index f869256..d304c02 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -11,6 +11,7 @@ from ecs_deploy.ecs import DeployAction, ScaleAction, RunAction, EcsClient, \ TaskPlacementError, EcsError, UpdateAction, LAUNCH_TYPE_EC2, LAUNCH_TYPE_FARGATE from ecs_deploy.newrelic import Deployment, NewRelicException +from ecs_deploy.slack import SlackNotification @click.group() @@ -50,7 +51,9 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers') @click.option('--sleep-time', default=1, type=int, help='Amount of seconds to wait between each check of the service (default: 1)') -def deploy(cluster, service, tag, image, command, env, secret, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time): +@click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL') +@click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, which services should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH') +def deploy(cluster, service, tag, image, command, env, secret, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, slack_url, slack_service_match='.*'): """ Redeploy or modify a service. @@ -75,6 +78,12 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r td.set_role_arn(role) td.set_execution_role_arn(execution_role) + slack = SlackNotification( + getenv('SLACK_URL', slack_url), + getenv('SLACK_SERVICE_MATCH', slack_service_match) + ) + slack.notifiy_start(cluster, tag, td, comment, user, service=service) + click.secho('Deploying based on task definition: %s\n' % td.family_revision) if diff: @@ -97,6 +106,7 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r ) except TaskPlacementError as e: + slack.notify_failure(cluster, str(e), service=service) if rollback: click.secho('%s\n' % str(e), fg='red', err=True) rollback_task_definition(deployment, td, new_td, sleep_time=sleep_time) @@ -106,6 +116,8 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r record_deployment(tag, newrelic_apikey, newrelic_appid, comment, user) + slack.notify_success(cluster, td.revision, service=service) + except (EcsError, NewRelicException) as e: click.secho('%s\n' % str(e), fg='red', err=True) exit(1) @@ -131,7 +143,9 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r @click.option('--diff/--no-diff', default=True, help='Print what values were changed in the task definition') @click.option('--deregister/--no-deregister', default=True, help='Deregister or keep the old task definition (default: --deregister)') @click.option('--rollback/--no-rollback', default=False, help='Rollback to previous revision, if deployment failed (default: --no-rollback)') -def cron(cluster, task, rule, image, tag, command, env, role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, comment, user, profile, diff, deregister, rollback): +@click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL') +@click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, deployments of which crons should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH') +def cron(cluster, task, rule, image, tag, command, env, role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, comment, user, profile, diff, deregister, rollback, slack_url, slack_service_match): """ Update a scheduled task. @@ -152,6 +166,12 @@ def cron(cluster, task, rule, image, tag, command, env, role, region, access_key td.set_environment(env) td.set_role_arn(role) + slack = SlackNotification( + getenv('SLACK_URL', slack_url), + getenv('SLACK_SERVICE_MATCH', slack_service_match) + ) + slack.notifiy_start(cluster, tag, td, comment, user, rule=rule) + if diff: print_diff(td) @@ -165,6 +185,8 @@ def cron(cluster, task, rule, image, tag, command, env, role, region, access_key click.secho('Updating scheduled task') click.secho('Successfully updated scheduled task %s\n' % rule, fg='green') + slack.notify_success(cluster, td.revision, rule=rule) + record_deployment(tag, newrelic_apikey, newrelic_appid, comment, user) if deregister: diff --git a/ecs_deploy/slack.py b/ecs_deploy/slack.py new file mode 100644 index 0000000..ba86238 --- /dev/null +++ b/ecs_deploy/slack.py @@ -0,0 +1,137 @@ +import re +from datetime import datetime +from requests import post + + +class SlackException(Exception): + pass + + +class SlackNotification(object): + def __init__(self, url, service_match): + self.__url = url + self.__service_match_re = re.compile(service_match) + self.__timestamp_start = None + + def get_payload(self, title, messages, color=None): + fields = [] + for message in messages: + field = { + 'title': message[0], + 'value': message[1], + 'short': True + } + fields.append(field) + + payload = { + "username": "ECS Deploy", + "attachments": [ + { + "pretext": title, + "color": color, + "fields": fields + } + ] + } + + return payload + + def notifiy_start(self, cluster, tag, task_definition, comment, user, service=None, rule=None): + if not self.__url or not self.__service_match_re.search(service or rule): + return + + self.__timestamp_start = datetime.utcnow() + + messages = [ + ('Cluster', cluster), + ] + + if service: + messages.append(('Service', service)) + + if rule: + messages.append(('Scheduled Task', rule)) + + if tag: + messages.append(('Tag', tag)) + + if user: + messages.append(('User', user)) + + if comment: + messages.append(('Comment', comment)) + + for diff in task_definition.diff: + if diff.field == 'image' and diff.value.endswith(':' + tag): + continue + if diff.field == 'environment': + messages.append(('Environment', '_sensitive (therefore hidden)_')) + continue + + messages.append((diff.field, diff.value)) + + payload = self.get_payload('Deployment has started', messages) + + + response = post(self.__url, json=payload) + + response.raise_for_status() + + if response.status_code != 200: + raise SlackException('Notifying deployment failed') + + return response + + def notify_success(self, cluster, revision, service=None, rule=None): + if not self.__url or not self.__service_match_re.search(service or rule): + return + + duration = datetime.utcnow() - self.__timestamp_start + + messages = [ + ('Cluster', cluster), + ] + + if service: + messages.append(('Service', service)) + if rule: + messages.append(('Scheduled Task', rule)) + + messages.append(('Revision', revision)) + messages.append(('Duration', str(duration))) + + payload = self.get_payload('Deployment finished successfully', messages, 'good') + + response = post(self.__url, json=payload) + + if response.status_code != 200: + raise SlackException('Notifying deployment failed') + + return response + + def notify_failure(self, cluster, error, service=None, rule=None): + if not self.__url or not self.__service_match_re.search(service or rule): + return + + duration = datetime.utcnow() - self.__timestamp_start + + messages = [ + ('Cluster', cluster), + ] + + if service: + messages.append(('Service', service)) + if rule: + messages.append(('Scheduled Task', rule)) + + messages.append(('Duration', str(duration))) + messages.append(('Error', error)) + + payload = self.get_payload('Deployment failed', messages, 'danger') + + response = post(self.__url, json=payload) + + if response.status_code != 200: + raise SlackException('Notifying deployment failed') + + return response From 82ebae0aca7af20e645f1c417fabb1a221a447d0 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Sat, 27 Jul 2019 09:06:34 +0200 Subject: [PATCH 02/12] Add support for New Relic EU region --- ecs_deploy/cli.py | 33 +++++++++++++++++--------------- ecs_deploy/newrelic.py | 13 ++++++++++--- tests/test_cli.py | 10 +++++----- tests/test_newrelic.py | 43 ++++++++++++++++++++++++++++++++---------- 4 files changed, 66 insertions(+), 33 deletions(-) diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index d304c02..7bab150 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -41,8 +41,9 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--profile', required=False, help='AWS configuration profile name') @click.option('--timeout', required=False, default=300, type=int, help='Amount of seconds to wait for deployment before command fails (default: 300). To disable timeout (fire and forget) set to -1') @click.option('--ignore-warnings', is_flag=True, help='Do not fail deployment on warnings (port already in use or insufficient memory/CPU)') -@click.option('--newrelic-apikey', required=False, help='New Relic API Key for recording the deployment') -@click.option('--newrelic-appid', required=False, help='New Relic App ID for recording the deployment') +@click.option('--newrelic-apikey', required=False, help='New Relic API Key for recording the deployment. Can also be defined via environment variable NEW_RELIC_API_KEY') +@click.option('--newrelic-appid', required=False, help='New Relic App ID for recording the deployment. Can also be defined via environment variable NEW_RELIC_APP_ID') +@click.option('--newrelic-region', required=False, help='New Relic region: US or EU (default: US). Can also be defined via environment variable NEW_RELIC_REGION') @click.option('--comment', required=False, help='Description/comment for recording the deployment') @click.option('--user', required=False, help='User who executes the deployment (used for recording)') @click.option('--diff/--no-diff', default=True, help='Print which values were changed in the task definition (default: --diff)') @@ -53,7 +54,7 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--sleep-time', default=1, type=int, help='Amount of seconds to wait between each check of the service (default: 1)') @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL') @click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, which services should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH') -def deploy(cluster, service, tag, image, command, env, secret, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, slack_url, slack_service_match='.*'): +def deploy(cluster, service, tag, image, command, env, secret, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, slack_url, slack_service_match='.*'): """ Redeploy or modify a service. @@ -114,7 +115,7 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r else: raise - record_deployment(tag, newrelic_apikey, newrelic_appid, comment, user) + record_deployment(tag, newrelic_apikey, newrelic_appid, newrelic_region, comment, user) slack.notify_success(cluster, td.revision, service=service) @@ -135,8 +136,9 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r @click.option('--region', help='AWS region (e.g. eu-central-1)') @click.option('--access-key-id', help='AWS access key id') @click.option('--secret-access-key', help='AWS secret access key') -@click.option('--newrelic-apikey', required=False, help='New Relic API Key for recording the deployment') -@click.option('--newrelic-appid', required=False, help='New Relic App ID for recording the deployment') +@click.option('--newrelic-apikey', required=False, help='New Relic API Key for recording the deployment. Can also be defined via environment variable NEW_RELIC_API_KEY') +@click.option('--newrelic-appid', required=False, help='New Relic App ID for recording the deployment. Can also be defined via environment variable NEW_RELIC_APP_ID') +@click.option('--newrelic-region', required=False, help='New Relic region: US or EU (default: US). Can also be defined via environment variable NEW_RELIC_REGION') @click.option('--comment', required=False, help='Description/comment for recording the deployment') @click.option('--user', required=False, help='User who executes the deployment (used for recording)') @click.option('--profile', help='AWS configuration profile name') @@ -145,7 +147,7 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r @click.option('--rollback/--no-rollback', default=False, help='Rollback to previous revision, if deployment failed (default: --no-rollback)') @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL') @click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, deployments of which crons should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH') -def cron(cluster, task, rule, image, tag, command, env, role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, comment, user, profile, diff, deregister, rollback, slack_url, slack_service_match): +def cron(cluster, task, rule, image, tag, command, env, role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, comment, user, profile, diff, deregister, rollback, slack_url, slack_service_match): """ Update a scheduled task. @@ -177,17 +179,17 @@ def cron(cluster, task, rule, image, tag, command, env, role, region, access_key new_td = create_task_definition(action, td) - client.update_rule( - cluster=cluster, - rule=rule, - task_definition=new_td - ) + #client.update_rule( + # cluster=cluster, + # rule=rule, + # task_definition=new_td + #) click.secho('Updating scheduled task') click.secho('Successfully updated scheduled task %s\n' % rule, fg='green') slack.notify_success(cluster, td.revision, rule=rule) - record_deployment(tag, newrelic_apikey, newrelic_appid, comment, user) + record_deployment(tag, newrelic_apikey, newrelic_appid, newrelic_region, comment, user) if deregister: deregister_task_definition(action, td) @@ -463,9 +465,10 @@ def rollback_task_definition(deployment, old, new, timeout=600, sleep_time=1): ) -def record_deployment(revision, api_key, app_id, comment, user): +def record_deployment(revision, api_key, app_id, region, comment, user): api_key = getenv('NEW_RELIC_API_KEY', api_key) app_id = getenv('NEW_RELIC_APP_ID', app_id) + region = getenv('NEW_RELIC_REGION', region) if not revision or not api_key or not app_id: return False @@ -474,7 +477,7 @@ def record_deployment(revision, api_key, app_id, comment, user): click.secho('Recording deployment in New Relic', nl=False) - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, region) deployment.deploy(revision, '', comment) click.secho('\nDone\n', fg='green') diff --git a/ecs_deploy/newrelic.py b/ecs_deploy/newrelic.py index b8c531d..d3d7d34 100644 --- a/ecs_deploy/newrelic.py +++ b/ecs_deploy/newrelic.py @@ -10,16 +10,23 @@ class NewRelicDeploymentException(NewRelicException): class Deployment(object): - ENDPOINT = 'https://api.newrelic.com/v2/applications/%(app_id)s/deployments.json' + API_HOST_US = 'api.newrelic.com' + API_HOST_EU = 'api.eu.newrelic.com' + ENDPOINT = 'https://%(host)s/v2/applications/%(app_id)s/deployments.json' - def __init__(self, api_key, app_id, user): + def __init__(self, api_key, app_id, user, region): self.__api_key = api_key self.__app_id = app_id self.__user = user + self.__region = region.lower() if region else 'us' @property def endpoint(self): - return self.ENDPOINT % dict(app_id=self.__app_id) + if self.__region == 'eu': + host = self.API_HOST_EU + else: + host = self.API_HOST_US + return self.ENDPOINT % dict(host=host, app_id=self.__app_id) @property def headers(self): diff --git a/tests/test_cli.py b/tests/test_cli.py index bc07c51..a801fc7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -697,19 +697,19 @@ def test_run_task_with_invalid_cluster(get_client, runner): @patch('ecs_deploy.newrelic.Deployment') def test_record_deployment_without_revision(Deployment): - result = record_deployment(None, None, None, None, None) + result = record_deployment(None, None, None, None, None, None) assert result is False @patch('ecs_deploy.newrelic.Deployment') def test_record_deployment_without_apikey(Deployment): - result = record_deployment('1.2.3', None, None, None, None) + result = record_deployment('1.2.3', None, None, None, None, None) assert result is False @patch('ecs_deploy.newrelic.Deployment') def test_record_deployment_without_appid(Deployment): - result = record_deployment('1.2.3', 'APIKEY', None, None, None) + result = record_deployment('1.2.3', 'APIKEY',None, None, None, None) assert result is False @@ -718,9 +718,9 @@ def test_record_deployment_without_appid(Deployment): @patch.object(Deployment, '__init__') def test_record_deployment(deployment_init, deployment_deploy, secho): deployment_init.return_value = None - result = record_deployment('1.2.3', 'APIKEY', 'APPID', 'Comment', 'user') + result = record_deployment('1.2.3', 'APIKEY', 'APPID', 'EU', 'Comment', 'user') - deployment_init.assert_called_once_with('APIKEY', 'APPID', 'user') + deployment_init.assert_called_once_with('APIKEY', 'APPID', 'user', 'EU') deployment_deploy.assert_called_once_with('1.2.3', '', 'Comment') secho.assert_any_call('Recording deployment in New Relic', nl=False) secho.assert_any_call('\nDone\n', fg='green') diff --git a/tests/test_newrelic.py b/tests/test_newrelic.py index 03784ae..396d73f 100644 --- a/tests/test_newrelic.py +++ b/tests/test_newrelic.py @@ -39,6 +39,11 @@ def user(): return 'username' +@fixture +def region(): + return 'eu' + + @fixture def revision(): return '1.2.3' @@ -54,23 +59,41 @@ def description(): return 'Lorem ipsum usu amet dicat nullam ea. Nec detracto lucilius democritum in.' -def test_get_endpoint(api_key, app_id, user): +def test_get_endpoint_us(api_key, app_id, user): + endpoint = 'https://api.newrelic.com/v2/applications/%(app_id)s/deployments.json' % dict(app_id=app_id) + deployment = Deployment(api_key, app_id, user, 'us') + assert deployment.endpoint == endpoint + + +def test_get_endpoint_eu(api_key, app_id, user): + endpoint = 'https://api.eu.newrelic.com/v2/applications/%(app_id)s/deployments.json' % dict(app_id=app_id) + deployment = Deployment(api_key, app_id, user, 'eu') + assert deployment.endpoint == endpoint + + +def test_get_endpoint_unknown_region(api_key, app_id, user): + endpoint = 'https://api.newrelic.com/v2/applications/%(app_id)s/deployments.json' % dict(app_id=app_id) + deployment = Deployment(api_key, app_id, user, 'unknown') + assert deployment.endpoint == endpoint + + +def test_get_endpoint_no_region(api_key, app_id, user): endpoint = 'https://api.newrelic.com/v2/applications/%(app_id)s/deployments.json' % dict(app_id=app_id) - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, None) assert deployment.endpoint == endpoint -def test_get_headers(api_key, app_id, user): +def test_get_headers(api_key, app_id, user, region): headers = { 'X-Api-Key': api_key, 'Content-Type': 'application/json', } - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, region) assert deployment.headers == headers -def test_get_payload(api_key, app_id, user, revision, changelog, description): +def test_get_payload(api_key, app_id, user, region, revision, changelog, description): payload = { 'deployment': { 'revision': revision, @@ -79,15 +102,15 @@ def test_get_payload(api_key, app_id, user, revision, changelog, description): 'user': user, } } - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, region) assert deployment.get_payload(revision, changelog, description) == payload @patch('requests.post') -def test_deploy_sucessful(post, api_key, app_id, user, revision, changelog, description): +def test_deploy_sucessful(post, api_key, app_id, user, region, revision, changelog, description): post.return_value = DeploymentResponseSuccessfulMock() - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, region) response = deployment.deploy(revision, changelog, description) payload = deployment.get_payload(revision, changelog, description) @@ -96,8 +119,8 @@ def test_deploy_sucessful(post, api_key, app_id, user, revision, changelog, desc @patch('requests.post') -def test_deploy_unsucessful(post, api_key, app_id, user, revision, changelog, description): +def test_deploy_unsucessful(post, api_key, app_id, user, region, revision, changelog, description): with raises(NewRelicDeploymentException): post.return_value = DeploymentResponseUnsuccessfulMock() - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, region) deployment.deploy(revision, changelog, description) From c3f3acefddd5d4569eb360556eebb34dad4ea54a Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Sat, 27 Jul 2019 09:56:55 +0200 Subject: [PATCH 03/12] Add tests for Slack integration --- ecs_deploy/cli.py | 4 +- ecs_deploy/slack.py | 21 ++--- setup.py | 1 + tests/test_slack.py | 222 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 232 insertions(+), 16 deletions(-) create mode 100644 tests/test_slack.py diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index d304c02..e00c52b 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -82,7 +82,7 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r getenv('SLACK_URL', slack_url), getenv('SLACK_SERVICE_MATCH', slack_service_match) ) - slack.notifiy_start(cluster, tag, td, comment, user, service=service) + slack.notify_start(cluster, tag, td, comment, user, service=service) click.secho('Deploying based on task definition: %s\n' % td.family_revision) @@ -170,7 +170,7 @@ def cron(cluster, task, rule, image, tag, command, env, role, region, access_key getenv('SLACK_URL', slack_url), getenv('SLACK_SERVICE_MATCH', slack_service_match) ) - slack.notifiy_start(cluster, tag, td, comment, user, rule=rule) + slack.notify_start(cluster, tag, td, comment, user, rule=rule) if diff: print_diff(td) diff --git a/ecs_deploy/slack.py b/ecs_deploy/slack.py index ba86238..2e5761c 100644 --- a/ecs_deploy/slack.py +++ b/ecs_deploy/slack.py @@ -1,6 +1,6 @@ import re +import requests from datetime import datetime -from requests import post class SlackException(Exception): @@ -10,8 +10,8 @@ class SlackException(Exception): class SlackNotification(object): def __init__(self, url, service_match): self.__url = url - self.__service_match_re = re.compile(service_match) - self.__timestamp_start = None + self.__service_match_re = re.compile(service_match or '') + self.__timestamp_start = datetime.utcnow() def get_payload(self, title, messages, color=None): fields = [] @@ -36,12 +36,10 @@ def get_payload(self, title, messages, color=None): return payload - def notifiy_start(self, cluster, tag, task_definition, comment, user, service=None, rule=None): + def notify_start(self, cluster, tag, task_definition, comment, user, service=None, rule=None): if not self.__url or not self.__service_match_re.search(service or rule): return - self.__timestamp_start = datetime.utcnow() - messages = [ ('Cluster', cluster), ] @@ -72,10 +70,7 @@ def notifiy_start(self, cluster, tag, task_definition, comment, user, service=No payload = self.get_payload('Deployment has started', messages) - - response = post(self.__url, json=payload) - - response.raise_for_status() + response = requests.post(self.__url, json=payload) if response.status_code != 200: raise SlackException('Notifying deployment failed') @@ -102,13 +97,11 @@ def notify_success(self, cluster, revision, service=None, rule=None): payload = self.get_payload('Deployment finished successfully', messages, 'good') - response = post(self.__url, json=payload) + response = requests.post(self.__url, json=payload) if response.status_code != 200: raise SlackException('Notifying deployment failed') - return response - def notify_failure(self, cluster, error, service=None, rule=None): if not self.__url or not self.__service_match_re.search(service or rule): return @@ -129,7 +122,7 @@ def notify_failure(self, cluster, error, service=None, rule=None): payload = self.get_payload('Deployment failed', messages, 'danger') - response = post(self.__url, json=payload) + response = requests.post(self.__url, json=payload) if response.status_code != 200: raise SlackException('Notifying deployment failed') diff --git a/setup.py b/setup.py index 1c8393a..b45f5d6 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ def readme(): 'pytest', 'pytest-flake8', 'pytest-mock', + 'freezegun', 'coverage' ], classifiers=[ diff --git a/tests/test_slack.py b/tests/test_slack.py new file mode 100644 index 0000000..d9b0489 --- /dev/null +++ b/tests/test_slack.py @@ -0,0 +1,222 @@ +from copy import deepcopy + +from freezegun import freeze_time +from pytest import fixture, raises +from mock import patch + +from ecs_deploy.ecs import EcsTaskDefinition +from ecs_deploy.slack import SlackNotification, SlackException +from tests.test_ecs import PAYLOAD_TASK_DEFINITION_1 + + +class NotifyResponseSuccessfulMock(object): + status_code = 200 + + +class NotifyResponseUnsuccessfulMock(object): + status_code = 400 + content = {"message": "Something went wrong"} + + +@fixture +def url(): + return 'https://slack.test' + + +@fixture +def service_match(): + return '.*' + + +@fixture +def task_definition(): + return EcsTaskDefinition(**deepcopy(PAYLOAD_TASK_DEFINITION_1)) + + +def test_get_payload_without_messages(url, service_match): + slack = SlackNotification(url, service_match) + + payload = slack.get_payload('Foobar', [], 'good') + + expected = { + 'username': 'ECS Deploy', + 'attachments': [ + { + 'color': 'good', + 'pretext': 'Foobar', + 'fields': [], + } + ], + } + + assert payload == expected + + +def test_get_payload_with_messages(url, service_match): + slack = SlackNotification(url, service_match) + + messages = ( + ('foo', 'bar'), + ('lorem', 'ipsum'), + ) + + payload = slack.get_payload('Foobar', messages, 'good') + + expected = { + 'username': 'ECS Deploy', + 'attachments': [ + { + 'color': 'good', + 'pretext': 'Foobar', + 'fields': [ + {'short': True, 'title': 'foo', 'value': 'bar'}, + {'short': True, 'title': 'lorem', 'value': 'ipsum'} + ], + } + ], + } + + assert payload == expected + + +@patch('requests.post') +def test_notify_start_without_url(post_mock, url, service_match, task_definition): + slack = SlackNotification(None, None) + slack.notify_start('my-cluster', 'my-tag', task_definition, 'my-comment', 'my-user', 'my-service', 'my-rule') + + post_mock.assert_not_called() + + +@patch('requests.post') +def test_notify_start(post_mock, url, service_match, task_definition): + post_mock.return_value = NotifyResponseSuccessfulMock() + + task_definition.set_images(webserver=u'new-image:my-tag', application=u'app-image:another-tag') + task_definition.set_environment((('webserver', 'foo', 'baz'),)) + + slack = SlackNotification(url, service_match) + slack.notify_start('my-cluster', 'my-tag', task_definition, 'my-comment', 'my-user', 'my-service', 'my-rule') + + payload = { + 'username': 'ECS Deploy', + 'attachments': [ + { + 'pretext': 'Deployment has started', + 'color': None, + 'fields': [ + {'title': 'Cluster', 'value': 'my-cluster', 'short': True}, + {'title': 'Service', 'value': 'my-service', 'short': True}, + {'title': 'Scheduled Task', 'value': 'my-rule', 'short': True}, + {'title': 'Tag', 'value': 'my-tag', 'short': True}, + {'title': 'User', 'value': 'my-user', 'short': True}, + {'title': 'Comment', 'value': 'my-comment', 'short': True}, + {'title': 'image', 'value': 'app-image:another-tag', 'short': True}, + {'title': 'Environment', 'value': '_sensitive (therefore hidden)_', 'short': True} + ] + } + ] + } + + post_mock.assert_called_with(url, json=payload) + + +@patch('requests.post') +@freeze_time() +def test_notify_success(post_mock, url, service_match, task_definition): + post_mock.return_value = NotifyResponseSuccessfulMock() + + slack = SlackNotification(url, service_match) + slack.notify_success('my-cluster', 'my-tag', 'my-service', 'my-rule') + + payload = { + 'username': 'ECS Deploy', + 'attachments': [ + { + 'pretext': 'Deployment finished successfully', + 'color': 'good', + 'fields': [ + {'title': 'Cluster', 'value': 'my-cluster', 'short': True}, + {'title': 'Service', 'value': 'my-service', 'short': True}, + {'title': 'Scheduled Task', 'value': 'my-rule', 'short': True}, + {'title': 'Revision', 'value': 'my-tag', 'short': True}, + {'title': 'Duration', 'value': '0:00:00', 'short': True} + ] + } + ] + } + + post_mock.assert_called_with(url, json=payload) + + +@patch('requests.post') +@freeze_time() +def test_notify_success(post_mock, url, service_match, task_definition): + post_mock.return_value = NotifyResponseSuccessfulMock() + + slack = SlackNotification(url, service_match) + slack.notify_failure('my-cluster', 'my-error', 'my-service', 'my-rule') + + payload = { + 'username': 'ECS Deploy', + 'attachments': [ + { + 'pretext': 'Deployment failed', + 'color': 'danger', + 'fields': [ + {'title': 'Cluster', 'value': 'my-cluster', 'short': True}, + {'title': 'Service', 'value': 'my-service', 'short': True}, + {'title': 'Scheduled Task', 'value': 'my-rule', 'short': True}, + {'title': 'Duration', 'value': '0:00:00', 'short': True}, + {'title': 'Error', 'value': 'my-error', 'short': True}, + ] + } + ] + } + + post_mock.assert_called_with(url, json=payload) + + +@patch('requests.post') +def test_notify_start_without_url(post_mock, url, service_match, task_definition): + slack = SlackNotification(None, None) + slack.notify_start('my-cluster', 'my-tag', task_definition, 'my-comment', 'my-user', 'my-service', 'my-rule') + post_mock.assert_not_called() + + +@patch('requests.post') +def test_notify_success_without_url(post_mock, url, service_match, task_definition): + slack = SlackNotification(None, None) + slack.notify_success('my-cluster', 13, 'my-service', 'my-rule') + post_mock.assert_not_called() + + +@patch('requests.post') +def test_notify_failure_without_url(post_mock, url, service_match, task_definition): + slack = SlackNotification(None, None) + slack.notify_failure('my-cluster', 'my-error', 'my-service', 'my-rule') + post_mock.assert_not_called() + + + +@patch('requests.post') +def test_notify_start_failed(post, url, service_match, task_definition): + with raises(SlackException): + post.return_value = NotifyResponseUnsuccessfulMock() + slack = SlackNotification(url, service_match) + slack.notify_start('my-cluster', 'my-tag', task_definition, 'my-comment', 'my-user', 'my-service', 'my-rule') + + +@patch('requests.post') +def test_notify_success_failed(post, url, service_match, task_definition): + with raises(SlackException): + post.return_value = NotifyResponseUnsuccessfulMock() + slack = SlackNotification(url, service_match) + slack.notify_success('my-cluster', 'my-tag', 'my-service', 'my-rule') + + +@patch('requests.post') +def test_notify_failure_failed(post, url, service_match, task_definition): + with raises(SlackException): + post.return_value = NotifyResponseUnsuccessfulMock() + slack = SlackNotification(url, service_match) + slack.notify_failure('my-cluster', 'my-error', 'my-service', 'my-rule') From e73e4f6bafd25c159e0ac05012bd485365aa148d Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Sat, 27 Jul 2019 12:16:21 +0200 Subject: [PATCH 04/12] Add freezegun to tox dependencies list --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 26b8a36..8612cb3 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,7 @@ deps= mock requests boto3 + freezegun [testenv:flake8] basepython = python2.7 From ecdba553d4eacd3dd857858275aef3828282fe68 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Sat, 27 Jul 2019 13:12:48 +0200 Subject: [PATCH 05/12] Remove debug-code, do not comment out cron rule update --- ecs_deploy/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index ca9d38d..930104a 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -179,11 +179,11 @@ def cron(cluster, task, rule, image, tag, command, env, role, region, access_key new_td = create_task_definition(action, td) - #client.update_rule( - # cluster=cluster, - # rule=rule, - # task_definition=new_td - #) + client.update_rule( + cluster=cluster, + rule=rule, + task_definition=new_td + ) click.secho('Updating scheduled task') click.secho('Successfully updated scheduled task %s\n' % rule, fg='green') From 1f0512ca6d4a28739e3af6f8062ec16e3bd91106 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Sun, 28 Jul 2019 08:48:37 +0200 Subject: [PATCH 06/12] Add new diff action --- ecs_deploy/cli.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++- ecs_deploy/ecs.py | 47 +++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 3 files changed, 96 insertions(+), 2 deletions(-) diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index 930104a..976f114 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -4,11 +4,12 @@ from time import sleep import click +import json import getpass from datetime import datetime, timedelta from ecs_deploy import VERSION -from ecs_deploy.ecs import DeployAction, ScaleAction, RunAction, EcsClient, \ +from ecs_deploy.ecs import DeployAction, ScaleAction, RunAction, EcsClient, DiffAction, \ TaskPlacementError, EcsError, UpdateAction, LAUNCH_TYPE_EC2, LAUNCH_TYPE_FARGATE from ecs_deploy.newrelic import Deployment, NewRelicException from ecs_deploy.slack import SlackNotification @@ -348,6 +349,51 @@ def run(cluster, task, count, command, env, secret, launchtype, subnet, security exit(1) +@click.command() +@click.argument('task') +@click.argument('revision_a') +@click.argument('revision_b') +@click.option('--region', help='AWS region (e.g. eu-central-1)') +@click.option('--access-key-id', help='AWS access key id') +@click.option('--secret-access-key', help='AWS secret access key') +@click.option('--profile', help='AWS configuration profile name') +def diff(task, revision_a, revision_b, region, access_key_id, secret_access_key, profile): + """ + Compare two task definition revisions. + + \b + TASK is the name of your task definition (e.g. 'my-task') within ECS. + COUNT is the number of tasks your service should run. + """ + + try: + client = get_client(access_key_id, secret_access_key, region, profile) + action = DiffAction(client) + + td_a = action.get_task_definition('%s:%s' % (task, revision_a)) + td_b = action.get_task_definition('%s:%s' % (task, revision_b)) + + result = td_a.diff_raw(td_b) + for difference in result: + click.secho('%s: %s' % (difference[0], difference[1])) + + if difference[0] == 'add': + for added in difference[2]: + click.secho(' + %s: %s' % (added[0], json.dumps(added[1]))) + + if difference[0] == 'change': + click.secho(' - %s' % json.dumps(difference[2][0])) + click.secho(' + %s' % json.dumps(difference[2][1])) + + if difference[0] == 'remove': + for removed in difference[2]: + click.secho(' - %s: %s' % removed) + + except EcsError as e: + click.secho('%s\n' % str(e), fg='red', err=True) + exit(1) + + def wait_for_finish(action, timeout, title, success_message, failure_message, ignore_warnings, sleep_time=1): click.secho(title, nl=False) @@ -544,6 +590,7 @@ def inspect_errors(service, failure_message, ignore_warnings, since, timeout): ecs.add_command(run) ecs.add_command(cron) ecs.add_command(update) +ecs.add_command(diff) if __name__ == '__main__': # pragma: no cover ecs() diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index 2bfcc10..3626987 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -5,6 +5,7 @@ from boto3.session import Session from botocore.exceptions import ClientError, NoCredentialsError from dateutil.tz.tz import tzlocal +from dictdiffer import diff JSON_LIST_REGEX = re.compile(r'^\[.*\]$') @@ -237,6 +238,47 @@ def family_revision(self): def diff(self): return self._diff + def diff_raw(self, task_b): + containers_a = {c['name']: c for c in self.containers} + containers_b = {c['name']: c for c in task_b.containers} + + requirements_a = sorted([r['name'] for r in self.requires_attributes]) + requirements_b = sorted([r['name'] for r in task_b.requires_attributes]) + + for container in containers_a: + containers_a[container]['environment'] = {e['name']: e['value'] for e in containers_a[container].get('environment', {})} + + for container in containers_b: + containers_b[container]['environment'] = {e['name']: e['value'] for e in containers_b[container].get('environment', {})} + + for container in containers_a: + containers_a[container]['secrets'] = {e['name']: e['valueFrom'] for e in containers_a[container].get('secrets', {})} + + for container in containers_b: + containers_b[container]['secrets'] = {e['name']: e['valueFrom'] for e in containers_b[container].get('secrets', {})} + + composite_a = { + 'containers': containers_a, + 'volumes': self.volumes, + 'requires_attributes': requirements_a, + 'role_arn': self.role_arn, + 'execution_role_arn': self.execution_role_arn, + 'compatibilities': self.compatibilities, + 'additional_properties': self.additional_properties, + } + + composite_b = { + 'containers': containers_b, + 'volumes': task_b.volumes, + 'requires_attributes': requirements_b, + 'role_arn': task_b.role_arn, + 'execution_role_arn': task_b.execution_role_arn, + 'compatibilities': task_b.compatibilities, + 'additional_properties': task_b.additional_properties, + } + + return list(diff(composite_a, composite_b)) + def get_overrides(self): override = dict() overrides = [] @@ -670,6 +712,11 @@ def __init__(self, client): super(UpdateAction, self).__init__(client, None, None) +class DiffAction(EcsAction): + def __init__(self, client): + super(DiffAction, self).__init__(client, None, None) + + class EcsError(Exception): pass diff --git a/setup.py b/setup.py index b45f5d6..1cf5a4f 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ def readme(): return f.read() -dependencies = ['click<7.0.0', 'botocore>=1.12.0', 'boto3>=1.4.7', 'future', 'requests'] +dependencies = ['click<7.0.0', 'botocore>=1.12.0', 'boto3>=1.4.7', 'future', 'requests', 'dictdiffer==0.8.0'] setup( name='ecs-deploy', From bac41905c9654ad9986da616225e3d4ac26bcc57 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Sun, 28 Jul 2019 10:51:45 +0200 Subject: [PATCH 07/12] Add high-level functional test for cli command "diff" --- tests/test_cli.py | 46 ++++++++++++++++++++++++++++++++++++++++++++-- tests/test_ecs.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index a801fc7..7846760 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,7 +9,9 @@ from ecs_deploy.ecs import EcsClient from ecs_deploy.newrelic import Deployment, NewRelicDeploymentException from tests.test_ecs import EcsTestClient, CLUSTER_NAME, SERVICE_NAME, \ - TASK_DEFINITION_ARN_1, TASK_DEFINITION_ARN_2, TASK_DEFINITION_FAMILY_1 + TASK_DEFINITION_ARN_1, TASK_DEFINITION_ARN_2, TASK_DEFINITION_FAMILY_1, \ + TASK_DEFINITION_REVISION_2, TASK_DEFINITION_REVISION_1, \ + TASK_DEFINITION_REVISION_3 @pytest.fixture @@ -1016,5 +1018,45 @@ def test_cron(get_client, runner): assert u'Deregister task definition revision' in result.output assert u'Successfully deregistered revision: 2' in result.output - print(result.output) + + +@patch('ecs_deploy.cli.get_client') +def test_diff(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.diff, (TASK_DEFINITION_FAMILY_1, str(TASK_DEFINITION_REVISION_1), str(TASK_DEFINITION_REVISION_3))) + + expected = '''change: containers.webserver.image + - "webserver:123" + + "webserver:456" +change: containers.webserver.command + - "run" + + "execute" +change: containers.webserver.environment.foo + - "bar" + + "foobar" +change: containers.webserver.environment.lorem + - "ipsum" + + "loremipsum" +remove: containers.webserver.environment + - empty: +change: containers.webserver.secrets.baz + - "qux" + + "foobaz" +change: containers.webserver.secrets.dolor + - "sit" + + "loremdolor" +change: role_arn + - "arn:test:role:1" + + "arn:test:another-role:1" +change: execution_role_arn + - "arn:test:role:1" + + "arn:test:another-role:1" +''' + + assert result.output == expected + + #assert not result.exception + #assert result.exit_code == 0 + #assert u"Update task definition based on: test-task:1" in result.output + #assert u'Successfully created revision: 2' in result.output diff --git a/tests/test_ecs.py b/tests/test_ecs.py index 3d77b33..223a142 100644 --- a/tests/test_ecs.py +++ b/tests/test_ecs.py @@ -30,6 +30,19 @@ u'secrets': ({"name": "baz", "valueFrom": "qux"}, {"name": "dolor", "valueFrom": "sit"})}, {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()} ] + +TASK_DEFINITION_REVISION_3 = 3 +TASK_DEFINITION_ARN_3 = u'arn:aws:ecs:eu-central-1:123456789012:task-definition/%s:%s' % (TASK_DEFINITION_FAMILY_1, + TASK_DEFINITION_REVISION_3) +TASK_DEFINITION_VOLUMES_3 = [] +TASK_DEFINITION_CONTAINERS_3 = [ + {u'name': u'webserver', u'image': u'webserver:456', u'command': u'execute', + u'environment': ({"name": "foo", "value": "foobar"}, {"name": "lorem", "value": "loremipsum"}), + u'secrets': ({"name": "baz", "valueFrom": "foobaz"}, {"name": "dolor", "valueFrom": "loremdolor"})}, + {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()} +] +TASK_DEFINITION_ROLE_ARN_3 = u'arn:test:another-role:1' + TASK_DEFINITION_FAMILY_2 = u'test-task' TASK_DEFINITION_REVISION_2 = 2 TASK_DEFINITION_ARN_2 = u'arn:aws:ecs:eu-central-1:123456789012:task-definition/%s:%s' % (TASK_DEFINITION_FAMILY_2, @@ -69,6 +82,22 @@ u'compatibilities': [u'EC2'], } +PAYLOAD_TASK_DEFINITION_3 = { + u'taskDefinitionArn': TASK_DEFINITION_ARN_3, + u'family': TASK_DEFINITION_FAMILY_1, + u'revision': TASK_DEFINITION_REVISION_3, + u'taskRoleArn': TASK_DEFINITION_ROLE_ARN_3, + u'executionRoleArn': TASK_DEFINITION_ROLE_ARN_3, + u'volumes': deepcopy(TASK_DEFINITION_VOLUMES_3), + u'containerDefinitions': deepcopy(TASK_DEFINITION_CONTAINERS_3), + u'status': u'active', + u'requiresAttributes': {}, + u'networkMode': u'host', + u'placementConstraints': {}, + u'unknownProperty': u'lorem-ipsum', + u'compatibilities': [u'EC2'], +} + TASK_ARN_1 = u'arn:aws:ecs:eu-central-1:123456789012:task/12345678-1234-1234-1234-123456789011' TASK_ARN_2 = u'arn:aws:ecs:eu-central-1:123456789012:task/12345678-1234-1234-1234-123456789012' @@ -166,11 +195,17 @@ u"taskDefinition": PAYLOAD_TASK_DEFINITION_2 } +RESPONSE_TASK_DEFINITION_3 = { + u"taskDefinition": PAYLOAD_TASK_DEFINITION_3 +} + RESPONSE_TASK_DEFINITIONS = { TASK_DEFINITION_ARN_1: RESPONSE_TASK_DEFINITION, TASK_DEFINITION_ARN_2: RESPONSE_TASK_DEFINITION_2, + TASK_DEFINITION_ARN_3: RESPONSE_TASK_DEFINITION_3, u'test-task:1': RESPONSE_TASK_DEFINITION, u'test-task:2': RESPONSE_TASK_DEFINITION_2, + u'test-task:3': RESPONSE_TASK_DEFINITION_3, u'test-task': RESPONSE_TASK_DEFINITION_2, } From 8ac9777f390dcf19c3a4d1b6dd53f09ac6ab0bfe Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Sun, 28 Jul 2019 10:59:13 +0200 Subject: [PATCH 08/12] Refactor diff cli test to avoid sorting issues --- tests/test_cli.py | 65 +++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 7846760..4f6aa70 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1018,45 +1018,38 @@ def test_cron(get_client, runner): assert u'Deregister task definition revision' in result.output assert u'Successfully deregistered revision: 2' in result.output - print(result.output) - @patch('ecs_deploy.cli.get_client') def test_diff(get_client, runner): get_client.return_value = EcsTestClient('acces_key', 'secret_key') result = runner.invoke(cli.diff, (TASK_DEFINITION_FAMILY_1, str(TASK_DEFINITION_REVISION_1), str(TASK_DEFINITION_REVISION_3))) - expected = '''change: containers.webserver.image - - "webserver:123" - + "webserver:456" -change: containers.webserver.command - - "run" - + "execute" -change: containers.webserver.environment.foo - - "bar" - + "foobar" -change: containers.webserver.environment.lorem - - "ipsum" - + "loremipsum" -remove: containers.webserver.environment - - empty: -change: containers.webserver.secrets.baz - - "qux" - + "foobaz" -change: containers.webserver.secrets.dolor - - "sit" - + "loremdolor" -change: role_arn - - "arn:test:role:1" - + "arn:test:another-role:1" -change: execution_role_arn - - "arn:test:role:1" - + "arn:test:another-role:1" -''' - - assert result.output == expected - - #assert not result.exception - #assert result.exit_code == 0 - #assert u"Update task definition based on: test-task:1" in result.output - #assert u'Successfully created revision: 2' in result.output + assert not result.exception + assert result.exit_code == 0 + + assert 'change: containers.webserver.image' in result.output + assert '- "webserver:123"' in result.output + assert '+ "webserver:456"' in result.output + assert 'change: containers.webserver.command' in result.output + assert '- "run"' in result.output + assert '+ "execute"' in result.output + assert 'change: containers.webserver.environment.foo' in result.output + assert '- "bar"' in result.output + assert '+ "foobar"' in result.output + assert 'change: containers.webserver.environment.lorem' in result.output + assert '- "ipsum"' in result.output + assert '+ "loremipsum"' in result.output + assert 'remove: containers.webserver.environment' in result.output + assert '- empty: ' in result.output + assert 'change: containers.webserver.secrets.baz' in result.output + assert '- "qux"' in result.output + assert '+ "foobaz"' in result.output + assert 'change: containers.webserver.secrets.dolor' in result.output + assert '- "sit"' in result.output + assert '+ "loremdolor"' in result.output + assert 'change: role_arn' in result.output + assert '- "arn:test:role:1"' in result.output + assert '+ "arn:test:another-role:1"' in result.output + assert 'change: execution_role_arn' in result.output + assert '- "arn:test:role:1"' in result.output + assert '+ "arn:test:another-role:1"' in result.output From 7a36cc446265cedf8c47d46c3957fca7fb89e842 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Sun, 28 Jul 2019 11:12:41 +0200 Subject: [PATCH 09/12] Test credentials error in diff command --- tests/test_cli.py | 14 +++++++++++--- tests/test_ecs.py | 26 ++++++++++++++------------ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4f6aa70..534cae1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1036,9 +1036,6 @@ def test_diff(get_client, runner): assert 'change: containers.webserver.environment.foo' in result.output assert '- "bar"' in result.output assert '+ "foobar"' in result.output - assert 'change: containers.webserver.environment.lorem' in result.output - assert '- "ipsum"' in result.output - assert '+ "loremipsum"' in result.output assert 'remove: containers.webserver.environment' in result.output assert '- empty: ' in result.output assert 'change: containers.webserver.secrets.baz' in result.output @@ -1053,3 +1050,14 @@ def test_diff(get_client, runner): assert 'change: execution_role_arn' in result.output assert '- "arn:test:role:1"' in result.output assert '+ "arn:test:another-role:1"' in result.output + assert 'add: containers.webserver.environment' in result.output + assert '+ newvar: "new value"' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_diff_without_credentials(get_client, runner): + get_client.return_value = EcsTestClient() + result = runner.invoke(cli.diff, (TASK_DEFINITION_FAMILY_1, str(TASK_DEFINITION_REVISION_1), str(TASK_DEFINITION_REVISION_3))) + + assert result.exit_code == 1 + assert u'Unable to locate credentials. Configure credentials by running "aws configure".\n\n' in result.output diff --git a/tests/test_ecs.py b/tests/test_ecs.py index 223a142..d91bdde 100644 --- a/tests/test_ecs.py +++ b/tests/test_ecs.py @@ -31,18 +31,6 @@ {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()} ] -TASK_DEFINITION_REVISION_3 = 3 -TASK_DEFINITION_ARN_3 = u'arn:aws:ecs:eu-central-1:123456789012:task-definition/%s:%s' % (TASK_DEFINITION_FAMILY_1, - TASK_DEFINITION_REVISION_3) -TASK_DEFINITION_VOLUMES_3 = [] -TASK_DEFINITION_CONTAINERS_3 = [ - {u'name': u'webserver', u'image': u'webserver:456', u'command': u'execute', - u'environment': ({"name": "foo", "value": "foobar"}, {"name": "lorem", "value": "loremipsum"}), - u'secrets': ({"name": "baz", "valueFrom": "foobaz"}, {"name": "dolor", "valueFrom": "loremdolor"})}, - {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()} -] -TASK_DEFINITION_ROLE_ARN_3 = u'arn:test:another-role:1' - TASK_DEFINITION_FAMILY_2 = u'test-task' TASK_DEFINITION_REVISION_2 = 2 TASK_DEFINITION_ARN_2 = u'arn:aws:ecs:eu-central-1:123456789012:task-definition/%s:%s' % (TASK_DEFINITION_FAMILY_2, @@ -55,6 +43,18 @@ {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()} ] +TASK_DEFINITION_REVISION_3 = 3 +TASK_DEFINITION_ARN_3 = u'arn:aws:ecs:eu-central-1:123456789012:task-definition/%s:%s' % (TASK_DEFINITION_FAMILY_1, + TASK_DEFINITION_REVISION_3) +TASK_DEFINITION_VOLUMES_3 = [] +TASK_DEFINITION_CONTAINERS_3 = [ + {u'name': u'webserver', u'image': u'webserver:456', u'command': u'execute', + u'environment': ({"name": "foo", "value": "foobar"}, {"name": "newvar", "value": "new value"}), + u'secrets': ({"name": "baz", "valueFrom": "foobaz"}, {"name": "dolor", "valueFrom": "loremdolor"})}, + {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()} +] +TASK_DEFINITION_ROLE_ARN_3 = u'arn:test:another-role:1' + PAYLOAD_TASK_DEFINITION_1 = { u'taskDefinitionArn': TASK_DEFINITION_ARN_1, u'family': TASK_DEFINITION_FAMILY_1, @@ -927,6 +927,8 @@ def describe_services(self, cluster_name, service_name): } def describe_task_definition(self, task_definition_arn): + if not self.access_key_id or not self.secret_access_key: + raise EcsConnectionError(u'Unable to locate credentials. Configure credentials by running "aws configure".') if task_definition_arn in RESPONSE_TASK_DEFINITIONS: return deepcopy(RESPONSE_TASK_DEFINITIONS[task_definition_arn]) raise UnknownTaskDefinitionError('Unknown task definition arn: %s' % task_definition_arn) From b763bfc98267c9202885afd4fe3cb077e2ed4343 Mon Sep 17 00:00:00 2001 From: Emma Hulme Date: Mon, 26 Aug 2019 21:42:20 +0100 Subject: [PATCH 10/12] allow extra EcsParameters to not be lost when updating rule --- ecs_deploy/ecs.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index 3626987..6e99d91 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -125,18 +125,9 @@ def run_task(self, cluster, task_definition, count, started_by, overrides, def update_rule(self, cluster, rule, task_definition): target = self.events.list_targets_by_rule(Rule=rule)['Targets'][0] - self.events.put_targets( - Rule=rule, - Targets=[{ - 'Arn': task_definition.arn.partition('task-definition')[0] + 'cluster/' + cluster, - 'Id': target['Id'], - 'RoleArn': target['RoleArn'], - 'EcsParameters': { - 'TaskDefinitionArn': task_definition.arn, - 'TaskCount': 1 - } - }] - ) + target['Arn'] = task_definition.arn.partition('task-definition')[0] + 'cluster/' + cluster + target['EcsParameters']['TaskDefinitionArn'] = task_definition.arn + self.events.put_targets(Rule=rule, Targets=[target]) return target['Id'] From 69683d38a3e34a1cbed958d8914b9b47e7f29395 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Sun, 29 Sep 2019 11:27:57 +0200 Subject: [PATCH 11/12] Add colors to diff output --- ecs_deploy/cli.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index 976f114..800c019 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -375,19 +375,20 @@ def diff(task, revision_a, revision_b, region, access_key_id, secret_access_key, result = td_a.diff_raw(td_b) for difference in result: - click.secho('%s: %s' % (difference[0], difference[1])) - if difference[0] == 'add': + click.secho('%s: %s' % (difference[0], difference[1]), fg='green') for added in difference[2]: - click.secho(' + %s: %s' % (added[0], json.dumps(added[1]))) + click.secho(' + %s: %s' % (added[0], json.dumps(added[1])), fg='green') if difference[0] == 'change': - click.secho(' - %s' % json.dumps(difference[2][0])) - click.secho(' + %s' % json.dumps(difference[2][1])) + click.secho('%s: %s' % (difference[0], difference[1]), fg='yellow') + click.secho(' - %s' % json.dumps(difference[2][0]), fg='red') + click.secho(' + %s' % json.dumps(difference[2][1]), fg='green') if difference[0] == 'remove': + click.secho('%s: %s' % (difference[0], difference[1]), fg='red') for removed in difference[2]: - click.secho(' - %s: %s' % removed) + click.secho(' - %s: %s' % removed, fg='red') except EcsError as e: click.secho('%s\n' % str(e), fg='red', err=True) From c679ae81c318d984af56474877552796c49ed71c Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Sun, 29 Sep 2019 11:28:23 +0200 Subject: [PATCH 12/12] Bump version to 1.10.0 --- ecs_deploy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs_deploy/__init__.py b/ecs_deploy/__init__.py index a05e3e3..57a21c5 100644 --- a/ecs_deploy/__init__.py +++ b/ecs_deploy/__init__.py @@ -1 +1 @@ -VERSION = '1.9.0' +VERSION = '1.10.0'