From 7320999016bcbb3bf00b11a137e947f4d7b94a83 Mon Sep 17 00:00:00 2001 From: Mark Heiges Date: Tue, 7 Mar 2023 10:21:30 -0500 Subject: [PATCH 1/2] add support for managing tags on task definitions closes #206 --- ecs_deploy/cli.py | 15 ++++++++-- ecs_deploy/ecs.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 35 ++++++++++++++++++++++ tests/test_ecs.py | 3 +- 4 files changed, 124 insertions(+), 4 deletions(-) diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index 20465eb..6a3aa54 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -45,6 +45,7 @@ def get_client(access_key_id, secret_access_key, region, profile, assume_account @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): ') @click.option('--secrets-env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load secrets from .env-file: ') @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') +@click.option('--task-tag', type=(str, str), multiple=True, help='Adds or changes a task definition tag: ') @click.option('-u', '--ulimit', type=(str, str, int, int), multiple=True, help='Adds or changes a ulimit variable in the container description (Not available for Fargate): ') @click.option('--system-control', type=(str, str, str), multiple=True, help='Adds or changes a system control variable in the container description (Not available for Fargate): ') @click.option('-p', '--port', type=(str, int, int), multiple=True, help='Adds or changes a port mappings in the container description (Not available for Fargate): ') @@ -74,6 +75,7 @@ def get_client(access_key_id, secret_access_key, region, profile, assume_account @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers') @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') +@click.option('--exclusive-task-tags', is_flag=True, default=False, help='Set the given task definition tags exclusively and remove all other pre-existing tags from the task definition') @click.option('--exclusive-s3-env-file', is_flag=True, default=False, help='Set the given s3 env files exclusively and remove all other pre-existing s3 env files from all containers') @click.option('--sleep-time', default=1, type=int, help='Amount of seconds to wait between each check of the service (default: 1)') @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL') @@ -85,7 +87,7 @@ def get_client(access_key_id, secret_access_key, region, profile, assume_account @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.') @click.option('--add-container', type=str, multiple=True, required=False, help='Add a placeholder container in the task definition.') @click.option('--remove-container', type=str, multiple=True, required=False, help='Remove a container from the task definition.') -def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, task_cpu, task_memory, privileged, essential, env, env_file, s3_env_file, secret, secrets_env_file, ulimit, system_control, port, mount, log, role, execution_role, runtime_platform, task, region, access_key_id, secret_access_key, profile, account, assume_role, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, exclusive_s3_env_file, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, docker_label, exclusive_docker_labels, slack_service_match='.*'): +def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, task_cpu, task_memory, privileged, essential, env, env_file, s3_env_file, secret, secrets_env_file, ulimit, system_control, port, mount, log, role, execution_role, runtime_platform, task, region, access_key_id, secret_access_key, profile, account, assume_role, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, exclusive_s3_env_file, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, docker_label, exclusive_docker_labels, task_tag, exclusive_task_tags, slack_service_match='.*'): """ Redeploy or modify a service. @@ -128,6 +130,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem td.set_execution_role_arn(execution_role) td.set_runtime_platform(runtime_platform) td.set_volumes(volume) + td.set_task_tags(task_tag, exclusive_task_tags) slack = SlackNotification( getenv('SLACK_URL', slack_url), @@ -191,6 +194,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): ') @click.option('--secrets-env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load secrets from .env-file: ') @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') +@click.option('--task-tag', type=(str, str), multiple=True, help='Adds or changes a task definition tag: ') @click.option('-u', '--ulimit', type=(str, str, int, int), multiple=True, help='Adds or changes a ulimit variable in the container description (Not available for Fargate): ') @click.option('--system-control', type=(str, str, str), multiple=True, help='Adds or changes a system control variable in the container description (Not available for Fargate): ') @click.option('-p', '--port', type=(str, int, int), multiple=True, help='Adds or changes a port mappings in the container description (Not available for Fargate): ') @@ -218,6 +222,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers') @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') +@click.option('--exclusive-task-tags', is_flag=True, default=False, help='Set the given task definition tags exclusively and remove all other pre-existing tags from the task definition') @click.option('--exclusive-s3-env-file', is_flag=True, default=False, help='Set the given s3 env files exclusively and remove all other pre-existing s3 env files from all containers') @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL') @click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, deployments of which crons should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH') @@ -226,7 +231,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem @click.option('--exclusive-ports', is_flag=True, default=False, help='Set the given port mappings exclusively and remove all other pre-existing port mappings from all containers') @click.option('--exclusive-mounts', is_flag=True, default=False, help='Set the given mount points exclusively and remove all other pre-existing mount points from all containers') @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.') -def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservation, task_cpu, task_memory, privileged, env, env_file, s3_env_file, secret, secrets_env_file, ulimit, system_control, port, mount, log, role, execution_role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, profile, account, assume_role, diff, deregister, rollback, exclusive_env, exclusive_secrets, exclusive_s3_env_file, slack_url, slack_service_match, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, docker_label, exclusive_docker_labels): +def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservation, task_cpu, task_memory, privileged, env, env_file, s3_env_file, secret, secrets_env_file, ulimit, system_control, port, mount, log, role, execution_role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, profile, account, assume_role, diff, deregister, rollback, exclusive_env, exclusive_secrets, exclusive_s3_env_file, slack_url, slack_service_match, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, docker_label, exclusive_docker_labels, task_tag, exclusive_task_tags): """ Update a scheduled task. @@ -262,6 +267,7 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio td.set_role_arn(role) td.set_execution_role_arn(execution_role) td.set_volumes(volume) + td.set_task_tags(task_tag, exclusive_task_tags) slack = SlackNotification( getenv('SLACK_URL', slack_url), @@ -305,6 +311,7 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): ') @click.option('--secrets-env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load secrets from .env-file: ') @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') +@click.option('--task-tag', type=(str, str), multiple=True, help='Adds or changes a task definition tag: ') @click.option('-r', '--role', type=str, help='Sets the task\'s role ARN: ') @click.option('--runtime-platform', type=str, nargs=2, help='Overwrites runtimePlatform: ') @click.option('--region', help='AWS region (e.g. eu-central-1)') @@ -317,9 +324,10 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers') @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') +@click.option('--exclusive-task-tags', is_flag=True, default=False, help='Set the given task definition tags exclusively and remove all other pre-existing tags from the task definition') @click.option('--exclusive-s3-env-file', is_flag=True, default=False, help='Set the given s3 env files exclusively and remove all other pre-existing s3 env files from all containers') @click.option('--deregister/--no-deregister', default=True, help='Deregister or keep the old task definition (default: --deregister)') -def update(task, image, tag, command, env, env_file, s3_env_file, secret, secrets_env_file, role, region, access_key_id, secret_access_key, profile, account, assume_role, diff, exclusive_env, exclusive_s3_env_file, exclusive_secrets, runtime_platform, deregister, docker_label, exclusive_docker_labels): +def update(task, image, tag, command, env, env_file, s3_env_file, secret, secrets_env_file, role, region, access_key_id, secret_access_key, profile, account, assume_role, diff, exclusive_env, exclusive_s3_env_file, exclusive_secrets, runtime_platform, deregister, docker_label, exclusive_docker_labels, task_tag, exclusive_task_tags): """ Update a task definition. @@ -341,6 +349,7 @@ def update(task, image, tag, command, env, env_file, s3_env_file, secret, secret td.set_s3_env_file(s3_env_file, exclusive_s3_env_file) td.set_role_arn(role) td.set_runtime_platform(runtime_platform) + td.set_task_tags(task_tag, exclusive_task_tags) if diff: print_diff(td) diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index 2fb4824..5843edf 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -525,6 +525,44 @@ def set_runtime_platform(self, runtime_platform): self.runtime_platform = new_runtime_platform self._diff.append(diff) + def set_task_tags(self, task_tag_list, exclusive=False): + new_tags_dict = {} + for tag in task_tag_list: + new_tags_dict[tag[0]] = tag[1] + + old_tags_dict = {} + if self.tags: + for tag in self.tags: + old_tags_dict[tag["key"]] = tag["value"] + + if exclusive is True: + merged = new_tags_dict + else: + merged = old_tags_dict.copy() + merged.update(new_tags_dict) + + if old_tags_dict == merged: + return + + new_tags = [] + for key, value in merged.items(): + mapping = {} + mapping["key"] = key + mapping["value"] = value + new_tags.append(mapping) + + self.apply_task_tags(new_tags) + + def apply_task_tags(self, new_tags): + diff = EcsTaskDefinitionDiff( + container=None, + field=u'tags', + value=new_tags, + old_value=self.tags + ) + self._diff.append(diff) + self.tags = new_tags + def set_cpu(self, **cpu): self.validate_container_options(**cpu) for container in self.containers: @@ -1234,6 +1272,11 @@ def __repr__(self): self.value, self.old_value, )) + elif self.field == u'tags': + return '\n'.join(self._get_task_tag_diffs( + self.value, + self.old_value, + )) elif self.container: return u'Changed %s of container "%s" to: "%s" (was: "%s")' % ( self.field, @@ -1280,6 +1323,38 @@ def _get_docker_label_diffs(container, dockerlabels, old_dockerlabels): diffs.append(message) return diffs + @staticmethod + def _get_task_tag_diffs(task_tag_list, old_task_tag_list): + msg = u'Changed task tag "%s" value to: "%s"' + msg_added = u'Added task tag "%s" with: "%s"' + msg_removed = u'Removed task tag "%s"' + tags_dict = __class__._tag_list_to_dict(task_tag_list) + old_tags_dict = __class__._tag_list_to_dict(old_task_tag_list) + diffs = [] + for key, value in tags_dict.items(): + old_value = old_tags_dict.get(key) + if value and not old_value: + message = msg_added % (key, value) + diffs.append(message) + elif value != old_value: + message = msg % (key, value) + diffs.append(message) + for old_key in old_tags_dict.keys(): + if old_key not in tags_dict.keys(): + message = msg_removed % (old_key) + diffs.append(message) + return diffs + + @staticmethod + def _tag_list_to_dict(tag_list): + tag_dict = {} + if tag_list: + for tag in tag_list: + key = tag["key"] + value = tag["value"] + tag_dict[key] = value + return tag_dict + @staticmethod def _get_secrets_diffs(container, secrets, old_secrets): msg = u'Changed secret "%s" of container "%s" to: "%s"' diff --git a/tests/test_cli.py b/tests/test_cli.py index 3fdf914..24074c3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -516,6 +516,41 @@ def test_deploy_exclusive_docker_label(get_client, runner): assert u'Successfully changed task definition to: test-task:2' in result.output assert u'Deployment successful' in result.output +@patch('ecs_deploy.cli.get_client') +def test_deploy_task_task_tag_merge(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '--task-tag', 'key1', 'value1', '--task-tag', 'key2', 'value2')) + assert result.exit_code == 0 + assert not result.exception + assert u"Deploying based on task definition: test-task:1" in result.output + assert u"Updating task definition" in result.output + assert u'Added task tag "key1" with: "value1"' in result.output + assert u'Added task tag "key2" with: "value2"' in result.output + assert u'Successfully created revision: 2' in result.output + +@patch('ecs_deploy.cli.get_client') +def test_update_task_task_tag_merge(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.update, (TASK_DEFINITION_ARN_2, '--task-tag', 'old-key', 'replace-value', '--task-tag', 'new-key', 'new-value')) + assert result.exit_code == 0 + assert not result.exception + assert u"Update task definition based on: test-task:2" in result.output + assert u"Updating task definition" in result.output + assert u'Changed task tag "old-key" value to: "replace-value"' in result.output + assert u'Added task tag "new-key" with: "new-value"' in result.output + assert u'Successfully created revision: 2' in result.output + +@patch('ecs_deploy.cli.get_client') +def test_update_task_exclusive_task_tag(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.update, (TASK_DEFINITION_ARN_2, '--task-tag', 'new-key', 'new-value', '--exclusive-task-tags')) + assert result.exit_code == 0 + assert not result.exception + assert u"Update task definition based on: test-task:2" in result.output + assert u"Updating task definition" in result.output + assert u'Removed task tag "old-key"' in result.output + assert u'Added task tag "new-key" with: "new-value"' in result.output + assert u'Successfully created revision: 2' in result.output @patch('ecs_deploy.cli.get_client') def test_deploy_exclusive_secret(get_client, runner): diff --git a/tests/test_ecs.py b/tests/test_ecs.py index ac75650..549f1ef 100644 --- a/tests/test_ecs.py +++ b/tests/test_ecs.py @@ -379,7 +379,8 @@ } RESPONSE_TASK_DEFINITION_2 = { - u"taskDefinition": PAYLOAD_TASK_DEFINITION_2 + u"taskDefinition": PAYLOAD_TASK_DEFINITION_2, + u'tags': [{'key': 'old-key', 'value': 'old-value'}] } RESPONSE_TASK_DEFINITION_3 = { From 1b77eacf6abafb4b687c61665c87c05bd2e6e971 Mon Sep 17 00:00:00 2001 From: Mark Heiges Date: Fri, 14 Apr 2023 08:38:31 -0400 Subject: [PATCH 2/2] syntax fix for python 2.7 --- ecs_deploy/ecs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index 5843edf..71044b2 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -1328,8 +1328,8 @@ def _get_task_tag_diffs(task_tag_list, old_task_tag_list): msg = u'Changed task tag "%s" value to: "%s"' msg_added = u'Added task tag "%s" with: "%s"' msg_removed = u'Removed task tag "%s"' - tags_dict = __class__._tag_list_to_dict(task_tag_list) - old_tags_dict = __class__._tag_list_to_dict(old_task_tag_list) + tags_dict = EcsTaskDefinitionDiff._tag_list_to_dict(task_tag_list) + old_tags_dict = EcsTaskDefinitionDiff._tag_list_to_dict(old_task_tag_list) diffs = [] for key, value in tags_dict.items(): old_value = old_tags_dict.get(key)