diff --git a/ecs_deploy/__init__.py b/ecs_deploy/__init__.py index a05e3e3..57a21c5 100644 --- a/ecs_deploy/__init__.py +++ b/ecs_deploy/__init__.py @@ -1 +1 @@ -VERSION = '1.9.0' +VERSION = '1.10.0' diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index f869256..800c019 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -4,13 +4,15 @@ from time import sleep import click +import json import getpass from datetime import datetime, timedelta from ecs_deploy import VERSION -from ecs_deploy.ecs import DeployAction, ScaleAction, RunAction, EcsClient, \ +from ecs_deploy.ecs import DeployAction, ScaleAction, RunAction, EcsClient, DiffAction, \ TaskPlacementError, EcsError, UpdateAction, LAUNCH_TYPE_EC2, LAUNCH_TYPE_FARGATE from ecs_deploy.newrelic import Deployment, NewRelicException +from ecs_deploy.slack import SlackNotification @click.group() @@ -40,8 +42,9 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--profile', required=False, help='AWS configuration profile name') @click.option('--timeout', required=False, default=300, type=int, help='Amount of seconds to wait for deployment before command fails (default: 300). To disable timeout (fire and forget) set to -1') @click.option('--ignore-warnings', is_flag=True, help='Do not fail deployment on warnings (port already in use or insufficient memory/CPU)') -@click.option('--newrelic-apikey', required=False, help='New Relic API Key for recording the deployment') -@click.option('--newrelic-appid', required=False, help='New Relic App ID for recording the deployment') +@click.option('--newrelic-apikey', required=False, help='New Relic API Key for recording the deployment. Can also be defined via environment variable NEW_RELIC_API_KEY') +@click.option('--newrelic-appid', required=False, help='New Relic App ID for recording the deployment. Can also be defined via environment variable NEW_RELIC_APP_ID') +@click.option('--newrelic-region', required=False, help='New Relic region: US or EU (default: US). Can also be defined via environment variable NEW_RELIC_REGION') @click.option('--comment', required=False, help='Description/comment for recording the deployment') @click.option('--user', required=False, help='User who executes the deployment (used for recording)') @click.option('--diff/--no-diff', default=True, help='Print which values were changed in the task definition (default: --diff)') @@ -50,7 +53,9 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers') @click.option('--sleep-time', default=1, type=int, help='Amount of seconds to wait between each check of the service (default: 1)') -def deploy(cluster, service, tag, image, command, env, secret, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time): +@click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL') +@click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, which services should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH') +def deploy(cluster, service, tag, image, command, env, secret, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, slack_url, slack_service_match='.*'): """ Redeploy or modify a service. @@ -75,6 +80,12 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r td.set_role_arn(role) td.set_execution_role_arn(execution_role) + slack = SlackNotification( + getenv('SLACK_URL', slack_url), + getenv('SLACK_SERVICE_MATCH', slack_service_match) + ) + slack.notify_start(cluster, tag, td, comment, user, service=service) + click.secho('Deploying based on task definition: %s\n' % td.family_revision) if diff: @@ -97,6 +108,7 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r ) except TaskPlacementError as e: + slack.notify_failure(cluster, str(e), service=service) if rollback: click.secho('%s\n' % str(e), fg='red', err=True) rollback_task_definition(deployment, td, new_td, sleep_time=sleep_time) @@ -104,7 +116,9 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r else: raise - record_deployment(tag, newrelic_apikey, newrelic_appid, comment, user) + record_deployment(tag, newrelic_apikey, newrelic_appid, newrelic_region, comment, user) + + slack.notify_success(cluster, td.revision, service=service) except (EcsError, NewRelicException) as e: click.secho('%s\n' % str(e), fg='red', err=True) @@ -123,15 +137,18 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r @click.option('--region', help='AWS region (e.g. eu-central-1)') @click.option('--access-key-id', help='AWS access key id') @click.option('--secret-access-key', help='AWS secret access key') -@click.option('--newrelic-apikey', required=False, help='New Relic API Key for recording the deployment') -@click.option('--newrelic-appid', required=False, help='New Relic App ID for recording the deployment') +@click.option('--newrelic-apikey', required=False, help='New Relic API Key for recording the deployment. Can also be defined via environment variable NEW_RELIC_API_KEY') +@click.option('--newrelic-appid', required=False, help='New Relic App ID for recording the deployment. Can also be defined via environment variable NEW_RELIC_APP_ID') +@click.option('--newrelic-region', required=False, help='New Relic region: US or EU (default: US). Can also be defined via environment variable NEW_RELIC_REGION') @click.option('--comment', required=False, help='Description/comment for recording the deployment') @click.option('--user', required=False, help='User who executes the deployment (used for recording)') @click.option('--profile', help='AWS configuration profile name') @click.option('--diff/--no-diff', default=True, help='Print what values were changed in the task definition') @click.option('--deregister/--no-deregister', default=True, help='Deregister or keep the old task definition (default: --deregister)') @click.option('--rollback/--no-rollback', default=False, help='Rollback to previous revision, if deployment failed (default: --no-rollback)') -def cron(cluster, task, rule, image, tag, command, env, role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, comment, user, profile, diff, deregister, rollback): +@click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL') +@click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, deployments of which crons should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH') +def cron(cluster, task, rule, image, tag, command, env, role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, comment, user, profile, diff, deregister, rollback, slack_url, slack_service_match): """ Update a scheduled task. @@ -152,6 +169,12 @@ def cron(cluster, task, rule, image, tag, command, env, role, region, access_key td.set_environment(env) td.set_role_arn(role) + slack = SlackNotification( + getenv('SLACK_URL', slack_url), + getenv('SLACK_SERVICE_MATCH', slack_service_match) + ) + slack.notify_start(cluster, tag, td, comment, user, rule=rule) + if diff: print_diff(td) @@ -165,7 +188,9 @@ def cron(cluster, task, rule, image, tag, command, env, role, region, access_key click.secho('Updating scheduled task') click.secho('Successfully updated scheduled task %s\n' % rule, fg='green') - record_deployment(tag, newrelic_apikey, newrelic_appid, comment, user) + slack.notify_success(cluster, td.revision, rule=rule) + + record_deployment(tag, newrelic_apikey, newrelic_appid, newrelic_region, comment, user) if deregister: deregister_task_definition(action, td) @@ -324,6 +349,52 @@ def run(cluster, task, count, command, env, secret, launchtype, subnet, security exit(1) +@click.command() +@click.argument('task') +@click.argument('revision_a') +@click.argument('revision_b') +@click.option('--region', help='AWS region (e.g. eu-central-1)') +@click.option('--access-key-id', help='AWS access key id') +@click.option('--secret-access-key', help='AWS secret access key') +@click.option('--profile', help='AWS configuration profile name') +def diff(task, revision_a, revision_b, region, access_key_id, secret_access_key, profile): + """ + Compare two task definition revisions. + + \b + TASK is the name of your task definition (e.g. 'my-task') within ECS. + COUNT is the number of tasks your service should run. + """ + + try: + client = get_client(access_key_id, secret_access_key, region, profile) + action = DiffAction(client) + + td_a = action.get_task_definition('%s:%s' % (task, revision_a)) + td_b = action.get_task_definition('%s:%s' % (task, revision_b)) + + result = td_a.diff_raw(td_b) + for difference in result: + if difference[0] == 'add': + click.secho('%s: %s' % (difference[0], difference[1]), fg='green') + for added in difference[2]: + click.secho(' + %s: %s' % (added[0], json.dumps(added[1])), fg='green') + + if difference[0] == 'change': + click.secho('%s: %s' % (difference[0], difference[1]), fg='yellow') + click.secho(' - %s' % json.dumps(difference[2][0]), fg='red') + click.secho(' + %s' % json.dumps(difference[2][1]), fg='green') + + if difference[0] == 'remove': + click.secho('%s: %s' % (difference[0], difference[1]), fg='red') + for removed in difference[2]: + click.secho(' - %s: %s' % removed, fg='red') + + except EcsError as e: + click.secho('%s\n' % str(e), fg='red', err=True) + exit(1) + + def wait_for_finish(action, timeout, title, success_message, failure_message, ignore_warnings, sleep_time=1): click.secho(title, nl=False) @@ -441,9 +512,10 @@ def rollback_task_definition(deployment, old, new, timeout=600, sleep_time=1): ) -def record_deployment(revision, api_key, app_id, comment, user): +def record_deployment(revision, api_key, app_id, region, comment, user): api_key = getenv('NEW_RELIC_API_KEY', api_key) app_id = getenv('NEW_RELIC_APP_ID', app_id) + region = getenv('NEW_RELIC_REGION', region) if not revision or not api_key or not app_id: return False @@ -452,7 +524,7 @@ def record_deployment(revision, api_key, app_id, comment, user): click.secho('Recording deployment in New Relic', nl=False) - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, region) deployment.deploy(revision, '', comment) click.secho('\nDone\n', fg='green') @@ -519,6 +591,7 @@ def inspect_errors(service, failure_message, ignore_warnings, since, timeout): ecs.add_command(run) ecs.add_command(cron) ecs.add_command(update) +ecs.add_command(diff) if __name__ == '__main__': # pragma: no cover ecs() diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index 2bfcc10..6e99d91 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -5,6 +5,7 @@ from boto3.session import Session from botocore.exceptions import ClientError, NoCredentialsError from dateutil.tz.tz import tzlocal +from dictdiffer import diff JSON_LIST_REGEX = re.compile(r'^\[.*\]$') @@ -124,18 +125,9 @@ def run_task(self, cluster, task_definition, count, started_by, overrides, def update_rule(self, cluster, rule, task_definition): target = self.events.list_targets_by_rule(Rule=rule)['Targets'][0] - self.events.put_targets( - Rule=rule, - Targets=[{ - 'Arn': task_definition.arn.partition('task-definition')[0] + 'cluster/' + cluster, - 'Id': target['Id'], - 'RoleArn': target['RoleArn'], - 'EcsParameters': { - 'TaskDefinitionArn': task_definition.arn, - 'TaskCount': 1 - } - }] - ) + target['Arn'] = task_definition.arn.partition('task-definition')[0] + 'cluster/' + cluster + target['EcsParameters']['TaskDefinitionArn'] = task_definition.arn + self.events.put_targets(Rule=rule, Targets=[target]) return target['Id'] @@ -237,6 +229,47 @@ def family_revision(self): def diff(self): return self._diff + def diff_raw(self, task_b): + containers_a = {c['name']: c for c in self.containers} + containers_b = {c['name']: c for c in task_b.containers} + + requirements_a = sorted([r['name'] for r in self.requires_attributes]) + requirements_b = sorted([r['name'] for r in task_b.requires_attributes]) + + for container in containers_a: + containers_a[container]['environment'] = {e['name']: e['value'] for e in containers_a[container].get('environment', {})} + + for container in containers_b: + containers_b[container]['environment'] = {e['name']: e['value'] for e in containers_b[container].get('environment', {})} + + for container in containers_a: + containers_a[container]['secrets'] = {e['name']: e['valueFrom'] for e in containers_a[container].get('secrets', {})} + + for container in containers_b: + containers_b[container]['secrets'] = {e['name']: e['valueFrom'] for e in containers_b[container].get('secrets', {})} + + composite_a = { + 'containers': containers_a, + 'volumes': self.volumes, + 'requires_attributes': requirements_a, + 'role_arn': self.role_arn, + 'execution_role_arn': self.execution_role_arn, + 'compatibilities': self.compatibilities, + 'additional_properties': self.additional_properties, + } + + composite_b = { + 'containers': containers_b, + 'volumes': task_b.volumes, + 'requires_attributes': requirements_b, + 'role_arn': task_b.role_arn, + 'execution_role_arn': task_b.execution_role_arn, + 'compatibilities': task_b.compatibilities, + 'additional_properties': task_b.additional_properties, + } + + return list(diff(composite_a, composite_b)) + def get_overrides(self): override = dict() overrides = [] @@ -670,6 +703,11 @@ def __init__(self, client): super(UpdateAction, self).__init__(client, None, None) +class DiffAction(EcsAction): + def __init__(self, client): + super(DiffAction, self).__init__(client, None, None) + + class EcsError(Exception): pass diff --git a/ecs_deploy/newrelic.py b/ecs_deploy/newrelic.py index b8c531d..d3d7d34 100644 --- a/ecs_deploy/newrelic.py +++ b/ecs_deploy/newrelic.py @@ -10,16 +10,23 @@ class NewRelicDeploymentException(NewRelicException): class Deployment(object): - ENDPOINT = 'https://api.newrelic.com/v2/applications/%(app_id)s/deployments.json' + API_HOST_US = 'api.newrelic.com' + API_HOST_EU = 'api.eu.newrelic.com' + ENDPOINT = 'https://%(host)s/v2/applications/%(app_id)s/deployments.json' - def __init__(self, api_key, app_id, user): + def __init__(self, api_key, app_id, user, region): self.__api_key = api_key self.__app_id = app_id self.__user = user + self.__region = region.lower() if region else 'us' @property def endpoint(self): - return self.ENDPOINT % dict(app_id=self.__app_id) + if self.__region == 'eu': + host = self.API_HOST_EU + else: + host = self.API_HOST_US + return self.ENDPOINT % dict(host=host, app_id=self.__app_id) @property def headers(self): diff --git a/ecs_deploy/slack.py b/ecs_deploy/slack.py new file mode 100644 index 0000000..2e5761c --- /dev/null +++ b/ecs_deploy/slack.py @@ -0,0 +1,130 @@ +import re +import requests +from datetime import datetime + + +class SlackException(Exception): + pass + + +class SlackNotification(object): + def __init__(self, url, service_match): + self.__url = url + self.__service_match_re = re.compile(service_match or '') + self.__timestamp_start = datetime.utcnow() + + def get_payload(self, title, messages, color=None): + fields = [] + for message in messages: + field = { + 'title': message[0], + 'value': message[1], + 'short': True + } + fields.append(field) + + payload = { + "username": "ECS Deploy", + "attachments": [ + { + "pretext": title, + "color": color, + "fields": fields + } + ] + } + + return payload + + def notify_start(self, cluster, tag, task_definition, comment, user, service=None, rule=None): + if not self.__url or not self.__service_match_re.search(service or rule): + return + + messages = [ + ('Cluster', cluster), + ] + + if service: + messages.append(('Service', service)) + + if rule: + messages.append(('Scheduled Task', rule)) + + if tag: + messages.append(('Tag', tag)) + + if user: + messages.append(('User', user)) + + if comment: + messages.append(('Comment', comment)) + + for diff in task_definition.diff: + if diff.field == 'image' and diff.value.endswith(':' + tag): + continue + if diff.field == 'environment': + messages.append(('Environment', '_sensitive (therefore hidden)_')) + continue + + messages.append((diff.field, diff.value)) + + payload = self.get_payload('Deployment has started', messages) + + response = requests.post(self.__url, json=payload) + + if response.status_code != 200: + raise SlackException('Notifying deployment failed') + + return response + + def notify_success(self, cluster, revision, service=None, rule=None): + if not self.__url or not self.__service_match_re.search(service or rule): + return + + duration = datetime.utcnow() - self.__timestamp_start + + messages = [ + ('Cluster', cluster), + ] + + if service: + messages.append(('Service', service)) + if rule: + messages.append(('Scheduled Task', rule)) + + messages.append(('Revision', revision)) + messages.append(('Duration', str(duration))) + + payload = self.get_payload('Deployment finished successfully', messages, 'good') + + response = requests.post(self.__url, json=payload) + + if response.status_code != 200: + raise SlackException('Notifying deployment failed') + + def notify_failure(self, cluster, error, service=None, rule=None): + if not self.__url or not self.__service_match_re.search(service or rule): + return + + duration = datetime.utcnow() - self.__timestamp_start + + messages = [ + ('Cluster', cluster), + ] + + if service: + messages.append(('Service', service)) + if rule: + messages.append(('Scheduled Task', rule)) + + messages.append(('Duration', str(duration))) + messages.append(('Error', error)) + + payload = self.get_payload('Deployment failed', messages, 'danger') + + response = requests.post(self.__url, json=payload) + + if response.status_code != 200: + raise SlackException('Notifying deployment failed') + + return response diff --git a/setup.py b/setup.py index 1c8393a..1cf5a4f 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ def readme(): return f.read() -dependencies = ['click<7.0.0', 'botocore>=1.12.0', 'boto3>=1.4.7', 'future', 'requests'] +dependencies = ['click<7.0.0', 'botocore>=1.12.0', 'boto3>=1.4.7', 'future', 'requests', 'dictdiffer==0.8.0'] setup( name='ecs-deploy', @@ -39,6 +39,7 @@ def readme(): 'pytest', 'pytest-flake8', 'pytest-mock', + 'freezegun', 'coverage' ], classifiers=[ diff --git a/tests/test_cli.py b/tests/test_cli.py index bc07c51..534cae1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,7 +9,9 @@ from ecs_deploy.ecs import EcsClient from ecs_deploy.newrelic import Deployment, NewRelicDeploymentException from tests.test_ecs import EcsTestClient, CLUSTER_NAME, SERVICE_NAME, \ - TASK_DEFINITION_ARN_1, TASK_DEFINITION_ARN_2, TASK_DEFINITION_FAMILY_1 + TASK_DEFINITION_ARN_1, TASK_DEFINITION_ARN_2, TASK_DEFINITION_FAMILY_1, \ + TASK_DEFINITION_REVISION_2, TASK_DEFINITION_REVISION_1, \ + TASK_DEFINITION_REVISION_3 @pytest.fixture @@ -697,19 +699,19 @@ def test_run_task_with_invalid_cluster(get_client, runner): @patch('ecs_deploy.newrelic.Deployment') def test_record_deployment_without_revision(Deployment): - result = record_deployment(None, None, None, None, None) + result = record_deployment(None, None, None, None, None, None) assert result is False @patch('ecs_deploy.newrelic.Deployment') def test_record_deployment_without_apikey(Deployment): - result = record_deployment('1.2.3', None, None, None, None) + result = record_deployment('1.2.3', None, None, None, None, None) assert result is False @patch('ecs_deploy.newrelic.Deployment') def test_record_deployment_without_appid(Deployment): - result = record_deployment('1.2.3', 'APIKEY', None, None, None) + result = record_deployment('1.2.3', 'APIKEY',None, None, None, None) assert result is False @@ -718,9 +720,9 @@ def test_record_deployment_without_appid(Deployment): @patch.object(Deployment, '__init__') def test_record_deployment(deployment_init, deployment_deploy, secho): deployment_init.return_value = None - result = record_deployment('1.2.3', 'APIKEY', 'APPID', 'Comment', 'user') + result = record_deployment('1.2.3', 'APIKEY', 'APPID', 'EU', 'Comment', 'user') - deployment_init.assert_called_once_with('APIKEY', 'APPID', 'user') + deployment_init.assert_called_once_with('APIKEY', 'APPID', 'user', 'EU') deployment_deploy.assert_called_once_with('1.2.3', '', 'Comment') secho.assert_any_call('Recording deployment in New Relic', nl=False) secho.assert_any_call('\nDone\n', fg='green') @@ -1017,4 +1019,45 @@ def test_cron(get_client, runner): assert u'Successfully deregistered revision: 2' in result.output - print(result.output) +@patch('ecs_deploy.cli.get_client') +def test_diff(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.diff, (TASK_DEFINITION_FAMILY_1, str(TASK_DEFINITION_REVISION_1), str(TASK_DEFINITION_REVISION_3))) + + assert not result.exception + assert result.exit_code == 0 + + assert 'change: containers.webserver.image' in result.output + assert '- "webserver:123"' in result.output + assert '+ "webserver:456"' in result.output + assert 'change: containers.webserver.command' in result.output + assert '- "run"' in result.output + assert '+ "execute"' in result.output + assert 'change: containers.webserver.environment.foo' in result.output + assert '- "bar"' in result.output + assert '+ "foobar"' in result.output + assert 'remove: containers.webserver.environment' in result.output + assert '- empty: ' in result.output + assert 'change: containers.webserver.secrets.baz' in result.output + assert '- "qux"' in result.output + assert '+ "foobaz"' in result.output + assert 'change: containers.webserver.secrets.dolor' in result.output + assert '- "sit"' in result.output + assert '+ "loremdolor"' in result.output + assert 'change: role_arn' in result.output + assert '- "arn:test:role:1"' in result.output + assert '+ "arn:test:another-role:1"' in result.output + assert 'change: execution_role_arn' in result.output + assert '- "arn:test:role:1"' in result.output + assert '+ "arn:test:another-role:1"' in result.output + assert 'add: containers.webserver.environment' in result.output + assert '+ newvar: "new value"' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_diff_without_credentials(get_client, runner): + get_client.return_value = EcsTestClient() + result = runner.invoke(cli.diff, (TASK_DEFINITION_FAMILY_1, str(TASK_DEFINITION_REVISION_1), str(TASK_DEFINITION_REVISION_3))) + + assert result.exit_code == 1 + assert u'Unable to locate credentials. Configure credentials by running "aws configure".\n\n' in result.output diff --git a/tests/test_ecs.py b/tests/test_ecs.py index 3d77b33..d91bdde 100644 --- a/tests/test_ecs.py +++ b/tests/test_ecs.py @@ -30,6 +30,7 @@ u'secrets': ({"name": "baz", "valueFrom": "qux"}, {"name": "dolor", "valueFrom": "sit"})}, {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()} ] + TASK_DEFINITION_FAMILY_2 = u'test-task' TASK_DEFINITION_REVISION_2 = 2 TASK_DEFINITION_ARN_2 = u'arn:aws:ecs:eu-central-1:123456789012:task-definition/%s:%s' % (TASK_DEFINITION_FAMILY_2, @@ -42,6 +43,18 @@ {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()} ] +TASK_DEFINITION_REVISION_3 = 3 +TASK_DEFINITION_ARN_3 = u'arn:aws:ecs:eu-central-1:123456789012:task-definition/%s:%s' % (TASK_DEFINITION_FAMILY_1, + TASK_DEFINITION_REVISION_3) +TASK_DEFINITION_VOLUMES_3 = [] +TASK_DEFINITION_CONTAINERS_3 = [ + {u'name': u'webserver', u'image': u'webserver:456', u'command': u'execute', + u'environment': ({"name": "foo", "value": "foobar"}, {"name": "newvar", "value": "new value"}), + u'secrets': ({"name": "baz", "valueFrom": "foobaz"}, {"name": "dolor", "valueFrom": "loremdolor"})}, + {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()} +] +TASK_DEFINITION_ROLE_ARN_3 = u'arn:test:another-role:1' + PAYLOAD_TASK_DEFINITION_1 = { u'taskDefinitionArn': TASK_DEFINITION_ARN_1, u'family': TASK_DEFINITION_FAMILY_1, @@ -69,6 +82,22 @@ u'compatibilities': [u'EC2'], } +PAYLOAD_TASK_DEFINITION_3 = { + u'taskDefinitionArn': TASK_DEFINITION_ARN_3, + u'family': TASK_DEFINITION_FAMILY_1, + u'revision': TASK_DEFINITION_REVISION_3, + u'taskRoleArn': TASK_DEFINITION_ROLE_ARN_3, + u'executionRoleArn': TASK_DEFINITION_ROLE_ARN_3, + u'volumes': deepcopy(TASK_DEFINITION_VOLUMES_3), + u'containerDefinitions': deepcopy(TASK_DEFINITION_CONTAINERS_3), + u'status': u'active', + u'requiresAttributes': {}, + u'networkMode': u'host', + u'placementConstraints': {}, + u'unknownProperty': u'lorem-ipsum', + u'compatibilities': [u'EC2'], +} + TASK_ARN_1 = u'arn:aws:ecs:eu-central-1:123456789012:task/12345678-1234-1234-1234-123456789011' TASK_ARN_2 = u'arn:aws:ecs:eu-central-1:123456789012:task/12345678-1234-1234-1234-123456789012' @@ -166,11 +195,17 @@ u"taskDefinition": PAYLOAD_TASK_DEFINITION_2 } +RESPONSE_TASK_DEFINITION_3 = { + u"taskDefinition": PAYLOAD_TASK_DEFINITION_3 +} + RESPONSE_TASK_DEFINITIONS = { TASK_DEFINITION_ARN_1: RESPONSE_TASK_DEFINITION, TASK_DEFINITION_ARN_2: RESPONSE_TASK_DEFINITION_2, + TASK_DEFINITION_ARN_3: RESPONSE_TASK_DEFINITION_3, u'test-task:1': RESPONSE_TASK_DEFINITION, u'test-task:2': RESPONSE_TASK_DEFINITION_2, + u'test-task:3': RESPONSE_TASK_DEFINITION_3, u'test-task': RESPONSE_TASK_DEFINITION_2, } @@ -892,6 +927,8 @@ def describe_services(self, cluster_name, service_name): } def describe_task_definition(self, task_definition_arn): + if not self.access_key_id or not self.secret_access_key: + raise EcsConnectionError(u'Unable to locate credentials. Configure credentials by running "aws configure".') if task_definition_arn in RESPONSE_TASK_DEFINITIONS: return deepcopy(RESPONSE_TASK_DEFINITIONS[task_definition_arn]) raise UnknownTaskDefinitionError('Unknown task definition arn: %s' % task_definition_arn) diff --git a/tests/test_newrelic.py b/tests/test_newrelic.py index 03784ae..396d73f 100644 --- a/tests/test_newrelic.py +++ b/tests/test_newrelic.py @@ -39,6 +39,11 @@ def user(): return 'username' +@fixture +def region(): + return 'eu' + + @fixture def revision(): return '1.2.3' @@ -54,23 +59,41 @@ def description(): return 'Lorem ipsum usu amet dicat nullam ea. Nec detracto lucilius democritum in.' -def test_get_endpoint(api_key, app_id, user): +def test_get_endpoint_us(api_key, app_id, user): + endpoint = 'https://api.newrelic.com/v2/applications/%(app_id)s/deployments.json' % dict(app_id=app_id) + deployment = Deployment(api_key, app_id, user, 'us') + assert deployment.endpoint == endpoint + + +def test_get_endpoint_eu(api_key, app_id, user): + endpoint = 'https://api.eu.newrelic.com/v2/applications/%(app_id)s/deployments.json' % dict(app_id=app_id) + deployment = Deployment(api_key, app_id, user, 'eu') + assert deployment.endpoint == endpoint + + +def test_get_endpoint_unknown_region(api_key, app_id, user): + endpoint = 'https://api.newrelic.com/v2/applications/%(app_id)s/deployments.json' % dict(app_id=app_id) + deployment = Deployment(api_key, app_id, user, 'unknown') + assert deployment.endpoint == endpoint + + +def test_get_endpoint_no_region(api_key, app_id, user): endpoint = 'https://api.newrelic.com/v2/applications/%(app_id)s/deployments.json' % dict(app_id=app_id) - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, None) assert deployment.endpoint == endpoint -def test_get_headers(api_key, app_id, user): +def test_get_headers(api_key, app_id, user, region): headers = { 'X-Api-Key': api_key, 'Content-Type': 'application/json', } - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, region) assert deployment.headers == headers -def test_get_payload(api_key, app_id, user, revision, changelog, description): +def test_get_payload(api_key, app_id, user, region, revision, changelog, description): payload = { 'deployment': { 'revision': revision, @@ -79,15 +102,15 @@ def test_get_payload(api_key, app_id, user, revision, changelog, description): 'user': user, } } - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, region) assert deployment.get_payload(revision, changelog, description) == payload @patch('requests.post') -def test_deploy_sucessful(post, api_key, app_id, user, revision, changelog, description): +def test_deploy_sucessful(post, api_key, app_id, user, region, revision, changelog, description): post.return_value = DeploymentResponseSuccessfulMock() - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, region) response = deployment.deploy(revision, changelog, description) payload = deployment.get_payload(revision, changelog, description) @@ -96,8 +119,8 @@ def test_deploy_sucessful(post, api_key, app_id, user, revision, changelog, desc @patch('requests.post') -def test_deploy_unsucessful(post, api_key, app_id, user, revision, changelog, description): +def test_deploy_unsucessful(post, api_key, app_id, user, region, revision, changelog, description): with raises(NewRelicDeploymentException): post.return_value = DeploymentResponseUnsuccessfulMock() - deployment = Deployment(api_key, app_id, user) + deployment = Deployment(api_key, app_id, user, region) deployment.deploy(revision, changelog, description) diff --git a/tests/test_slack.py b/tests/test_slack.py new file mode 100644 index 0000000..d9b0489 --- /dev/null +++ b/tests/test_slack.py @@ -0,0 +1,222 @@ +from copy import deepcopy + +from freezegun import freeze_time +from pytest import fixture, raises +from mock import patch + +from ecs_deploy.ecs import EcsTaskDefinition +from ecs_deploy.slack import SlackNotification, SlackException +from tests.test_ecs import PAYLOAD_TASK_DEFINITION_1 + + +class NotifyResponseSuccessfulMock(object): + status_code = 200 + + +class NotifyResponseUnsuccessfulMock(object): + status_code = 400 + content = {"message": "Something went wrong"} + + +@fixture +def url(): + return 'https://slack.test' + + +@fixture +def service_match(): + return '.*' + + +@fixture +def task_definition(): + return EcsTaskDefinition(**deepcopy(PAYLOAD_TASK_DEFINITION_1)) + + +def test_get_payload_without_messages(url, service_match): + slack = SlackNotification(url, service_match) + + payload = slack.get_payload('Foobar', [], 'good') + + expected = { + 'username': 'ECS Deploy', + 'attachments': [ + { + 'color': 'good', + 'pretext': 'Foobar', + 'fields': [], + } + ], + } + + assert payload == expected + + +def test_get_payload_with_messages(url, service_match): + slack = SlackNotification(url, service_match) + + messages = ( + ('foo', 'bar'), + ('lorem', 'ipsum'), + ) + + payload = slack.get_payload('Foobar', messages, 'good') + + expected = { + 'username': 'ECS Deploy', + 'attachments': [ + { + 'color': 'good', + 'pretext': 'Foobar', + 'fields': [ + {'short': True, 'title': 'foo', 'value': 'bar'}, + {'short': True, 'title': 'lorem', 'value': 'ipsum'} + ], + } + ], + } + + assert payload == expected + + +@patch('requests.post') +def test_notify_start_without_url(post_mock, url, service_match, task_definition): + slack = SlackNotification(None, None) + slack.notify_start('my-cluster', 'my-tag', task_definition, 'my-comment', 'my-user', 'my-service', 'my-rule') + + post_mock.assert_not_called() + + +@patch('requests.post') +def test_notify_start(post_mock, url, service_match, task_definition): + post_mock.return_value = NotifyResponseSuccessfulMock() + + task_definition.set_images(webserver=u'new-image:my-tag', application=u'app-image:another-tag') + task_definition.set_environment((('webserver', 'foo', 'baz'),)) + + slack = SlackNotification(url, service_match) + slack.notify_start('my-cluster', 'my-tag', task_definition, 'my-comment', 'my-user', 'my-service', 'my-rule') + + payload = { + 'username': 'ECS Deploy', + 'attachments': [ + { + 'pretext': 'Deployment has started', + 'color': None, + 'fields': [ + {'title': 'Cluster', 'value': 'my-cluster', 'short': True}, + {'title': 'Service', 'value': 'my-service', 'short': True}, + {'title': 'Scheduled Task', 'value': 'my-rule', 'short': True}, + {'title': 'Tag', 'value': 'my-tag', 'short': True}, + {'title': 'User', 'value': 'my-user', 'short': True}, + {'title': 'Comment', 'value': 'my-comment', 'short': True}, + {'title': 'image', 'value': 'app-image:another-tag', 'short': True}, + {'title': 'Environment', 'value': '_sensitive (therefore hidden)_', 'short': True} + ] + } + ] + } + + post_mock.assert_called_with(url, json=payload) + + +@patch('requests.post') +@freeze_time() +def test_notify_success(post_mock, url, service_match, task_definition): + post_mock.return_value = NotifyResponseSuccessfulMock() + + slack = SlackNotification(url, service_match) + slack.notify_success('my-cluster', 'my-tag', 'my-service', 'my-rule') + + payload = { + 'username': 'ECS Deploy', + 'attachments': [ + { + 'pretext': 'Deployment finished successfully', + 'color': 'good', + 'fields': [ + {'title': 'Cluster', 'value': 'my-cluster', 'short': True}, + {'title': 'Service', 'value': 'my-service', 'short': True}, + {'title': 'Scheduled Task', 'value': 'my-rule', 'short': True}, + {'title': 'Revision', 'value': 'my-tag', 'short': True}, + {'title': 'Duration', 'value': '0:00:00', 'short': True} + ] + } + ] + } + + post_mock.assert_called_with(url, json=payload) + + +@patch('requests.post') +@freeze_time() +def test_notify_success(post_mock, url, service_match, task_definition): + post_mock.return_value = NotifyResponseSuccessfulMock() + + slack = SlackNotification(url, service_match) + slack.notify_failure('my-cluster', 'my-error', 'my-service', 'my-rule') + + payload = { + 'username': 'ECS Deploy', + 'attachments': [ + { + 'pretext': 'Deployment failed', + 'color': 'danger', + 'fields': [ + {'title': 'Cluster', 'value': 'my-cluster', 'short': True}, + {'title': 'Service', 'value': 'my-service', 'short': True}, + {'title': 'Scheduled Task', 'value': 'my-rule', 'short': True}, + {'title': 'Duration', 'value': '0:00:00', 'short': True}, + {'title': 'Error', 'value': 'my-error', 'short': True}, + ] + } + ] + } + + post_mock.assert_called_with(url, json=payload) + + +@patch('requests.post') +def test_notify_start_without_url(post_mock, url, service_match, task_definition): + slack = SlackNotification(None, None) + slack.notify_start('my-cluster', 'my-tag', task_definition, 'my-comment', 'my-user', 'my-service', 'my-rule') + post_mock.assert_not_called() + + +@patch('requests.post') +def test_notify_success_without_url(post_mock, url, service_match, task_definition): + slack = SlackNotification(None, None) + slack.notify_success('my-cluster', 13, 'my-service', 'my-rule') + post_mock.assert_not_called() + + +@patch('requests.post') +def test_notify_failure_without_url(post_mock, url, service_match, task_definition): + slack = SlackNotification(None, None) + slack.notify_failure('my-cluster', 'my-error', 'my-service', 'my-rule') + post_mock.assert_not_called() + + + +@patch('requests.post') +def test_notify_start_failed(post, url, service_match, task_definition): + with raises(SlackException): + post.return_value = NotifyResponseUnsuccessfulMock() + slack = SlackNotification(url, service_match) + slack.notify_start('my-cluster', 'my-tag', task_definition, 'my-comment', 'my-user', 'my-service', 'my-rule') + + +@patch('requests.post') +def test_notify_success_failed(post, url, service_match, task_definition): + with raises(SlackException): + post.return_value = NotifyResponseUnsuccessfulMock() + slack = SlackNotification(url, service_match) + slack.notify_success('my-cluster', 'my-tag', 'my-service', 'my-rule') + + +@patch('requests.post') +def test_notify_failure_failed(post, url, service_match, task_definition): + with raises(SlackException): + post.return_value = NotifyResponseUnsuccessfulMock() + slack = SlackNotification(url, service_match) + slack.notify_failure('my-cluster', 'my-error', 'my-service', 'my-rule') diff --git a/tox.ini b/tox.ini index 26b8a36..8612cb3 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,7 @@ deps= mock requests boto3 + freezegun [testenv:flake8] basepython = python2.7