add support for managing tags on task definitions

closes fabfuel#206

ecs_deploy/cli.py

@@ -45,6 +45,7 @@ def get_client(access_key_id, secret_access_key, region, profile, assume_account
 @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): <container> <name> <parameter name>')
 @click.option('--secrets-env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load secrets from .env-file: <container> <env file path>')
 @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: <container> <name> <value>')
+@click.option('--task-tag', type=(str, str), multiple=True, help='Adds or changes a task definition tag: <key> <value>')
 @click.option('-u', '--ulimit', type=(str, str, int, int), multiple=True, help='Adds or changes a ulimit variable in the container description (Not available for Fargate): <container> <ulimit name> <softlimit value> <hardlimit value>')
 @click.option('--system-control', type=(str, str, str), multiple=True, help='Adds or changes a system control variable in the container description (Not available for Fargate): <container> <namespace> <value>')
 @click.option('-p', '--port', type=(str, int, int), multiple=True, help='Adds or changes a port mappings in the container description (Not available for Fargate): <container> <container port value> <host port value>')
@@ -74,6 +75,7 @@ def get_client(access_key_id, secret_access_key, region, profile, assume_account
 @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers')
 @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers')
 @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers')
+@click.option('--exclusive-task-tags', is_flag=True, default=False, help='Set the given task definition tags exclusively and remove all other pre-existing tags from the task definition')
 @click.option('--exclusive-s3-env-file', is_flag=True, default=False, help='Set the given s3 env files exclusively and remove all other pre-existing s3 env files from all containers')
 @click.option('--sleep-time', default=1, type=int, help='Amount of seconds to wait between each check of the service (default: 1)')
 @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL')
@@ -85,7 +87,7 @@ def get_client(access_key_id, secret_access_key, region, profile, assume_account
 @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.')
 @click.option('--add-container', type=str, multiple=True, required=False, help='Add a placeholder container in the task definition.')
 @click.option('--remove-container', type=str, multiple=True, required=False, help='Remove a container from the task definition.')
-def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, task_cpu, task_memory, privileged, essential, env, env_file, s3_env_file, secret, secrets_env_file, ulimit, system_control, port, mount, log, role, execution_role, runtime_platform, task, region, access_key_id, secret_access_key, profile, account, assume_role, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, exclusive_s3_env_file, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, docker_label, exclusive_docker_labels, slack_service_match='.*'):
+def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, task_cpu, task_memory, privileged, essential, env, env_file, s3_env_file, secret, secrets_env_file, ulimit, system_control, port, mount, log, role, execution_role, runtime_platform, task, region, access_key_id, secret_access_key, profile, account, assume_role, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, exclusive_s3_env_file, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, docker_label, exclusive_docker_labels, task_tag, exclusive_task_tags, slack_service_match='.*'):
     """
     Redeploy or modify a service.
 
@@ -128,6 +130,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem
         td.set_execution_role_arn(execution_role)
         td.set_runtime_platform(runtime_platform)
         td.set_volumes(volume)
+        td.set_task_tags(task_tag, exclusive_task_tags)
 
         slack = SlackNotification(
             getenv('SLACK_URL', slack_url),
@@ -191,6 +194,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem
 @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): <container> <name> <parameter name>')
 @click.option('--secrets-env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load secrets from .env-file: <container> <env file path>')
 @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: <container> <name> <value>')
+@click.option('--task-tag', type=(str, str), multiple=True, help='Adds or changes a task definition tag: <key> <value>')
 @click.option('-u', '--ulimit', type=(str, str, int, int), multiple=True, help='Adds or changes a ulimit variable in the container description (Not available for Fargate): <container> <ulimit name> <softlimit value> <hardlimit value>')
 @click.option('--system-control', type=(str, str, str), multiple=True, help='Adds or changes a system control variable in the container description (Not available for Fargate): <container> <namespace> <value>')
 @click.option('-p', '--port', type=(str, int, int), multiple=True, help='Adds or changes a port mappings in the container description (Not available for Fargate): <container> <container port value> <host port value>')
@@ -218,6 +222,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem
 @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers')
 @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers')
 @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers')
+@click.option('--exclusive-task-tags', is_flag=True, default=False, help='Set the given task definition tags exclusively and remove all other pre-existing tags from the task definition')
 @click.option('--exclusive-s3-env-file', is_flag=True, default=False, help='Set the given s3 env files exclusively and remove all other pre-existing s3 env files from all containers')
 @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL')
 @click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, deployments of which crons should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH')
@@ -226,7 +231,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem
 @click.option('--exclusive-ports', is_flag=True, default=False, help='Set the given port mappings exclusively and remove all other pre-existing port mappings from all containers')
 @click.option('--exclusive-mounts', is_flag=True, default=False, help='Set the given mount points exclusively and remove all other pre-existing mount points from all containers')
 @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.')
-def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservation, task_cpu, task_memory, privileged, env, env_file, s3_env_file, secret, secrets_env_file, ulimit, system_control, port, mount, log, role, execution_role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, profile, account, assume_role, diff, deregister, rollback, exclusive_env, exclusive_secrets, exclusive_s3_env_file, slack_url, slack_service_match, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, docker_label, exclusive_docker_labels):
+def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservation, task_cpu, task_memory, privileged, env, env_file, s3_env_file, secret, secrets_env_file, ulimit, system_control, port, mount, log, role, execution_role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, profile, account, assume_role, diff, deregister, rollback, exclusive_env, exclusive_secrets, exclusive_s3_env_file, slack_url, slack_service_match, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, docker_label, exclusive_docker_labels, task_tag, exclusive_task_tags):
     """
     Update a scheduled task.
 
@@ -262,6 +267,7 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio
         td.set_role_arn(role)
         td.set_execution_role_arn(execution_role)
         td.set_volumes(volume)
+        td.set_task_tags(task_tag, exclusive_task_tags)
 
         slack = SlackNotification(
             getenv('SLACK_URL', slack_url),
@@ -305,6 +311,7 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio
 @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): <container> <name> <parameter name>')
 @click.option('--secrets-env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load secrets from .env-file: <container> <env file path>')
 @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: <container> <name> <value>')
+@click.option('--task-tag', type=(str, str), multiple=True, help='Adds or changes a task definition tag: <key> <value>')
 @click.option('-r', '--role', type=str, help='Sets the task\'s role ARN: <task role ARN>')
 @click.option('--runtime-platform', type=str, nargs=2, help='Overwrites runtimePlatform: <cpuArchitecture> <operatingSystemFamily>')
 @click.option('--region', help='AWS region (e.g. eu-central-1)')
@@ -317,9 +324,10 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio
 @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers')
 @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers')
 @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers')
+@click.option('--exclusive-task-tags', is_flag=True, default=False, help='Set the given task definition tags exclusively and remove all other pre-existing tags from the task definition')
 @click.option('--exclusive-s3-env-file', is_flag=True, default=False, help='Set the given s3 env files exclusively and remove all other pre-existing s3 env files from all containers')
 @click.option('--deregister/--no-deregister', default=True, help='Deregister or keep the old task definition (default: --deregister)')
-def update(task, image, tag, command, env, env_file, s3_env_file, secret, secrets_env_file, role, region, access_key_id, secret_access_key, profile, account, assume_role, diff, exclusive_env, exclusive_s3_env_file, exclusive_secrets, runtime_platform, deregister, docker_label, exclusive_docker_labels):
+def update(task, image, tag, command, env, env_file, s3_env_file, secret, secrets_env_file, role, region, access_key_id, secret_access_key, profile, account, assume_role, diff, exclusive_env, exclusive_s3_env_file, exclusive_secrets, runtime_platform, deregister, docker_label, exclusive_docker_labels, task_tag, exclusive_task_tags):
     """
     Update a task definition.
 
@@ -341,6 +349,7 @@ def update(task, image, tag, command, env, env_file, s3_env_file, secret, secret
         td.set_s3_env_file(s3_env_file, exclusive_s3_env_file)
         td.set_role_arn(role)
         td.set_runtime_platform(runtime_platform)
+        td.set_task_tags(task_tag, exclusive_task_tags)
 
         if diff:
             print_diff(td)

ecs_deploy/ecs.py

@@ -525,6 +525,44 @@ def set_runtime_platform(self, runtime_platform):
             self.runtime_platform = new_runtime_platform
             self._diff.append(diff)
 
+    def set_task_tags(self, task_tag_list, exclusive=False):
+        new_tags_dict = {}
+        for tag in task_tag_list:
+            new_tags_dict[tag[0]] = tag[1]
+
+        old_tags_dict = {}
+        if self.tags:
+          for tag in self.tags:
+              old_tags_dict[tag["key"]] = tag["value"]
+
+        if exclusive is True:
+            merged = new_tags_dict
+        else:
+            merged = old_tags_dict.copy()
+            merged.update(new_tags_dict)
+
+        if old_tags_dict == merged:
+            return
+
+        new_tags = []
+        for key, value in merged.items():
+            mapping = {}
+            mapping["key"] = key
+            mapping["value"] = value
+            new_tags.append(mapping)
+
+        self.apply_task_tags(new_tags)
+
+    def apply_task_tags(self, new_tags):
+        diff = EcsTaskDefinitionDiff(
+            container=None,
+            field=u'tags',
+            value=new_tags,
+            old_value=self.tags
+        )
+        self._diff.append(diff)
+        self.tags = new_tags
+
     def set_cpu(self, **cpu):
         self.validate_container_options(**cpu)
         for container in self.containers:
@@ -1234,6 +1272,11 @@ def __repr__(self):
                 self.value,
                 self.old_value,
             ))
+        elif self.field == u'tags':
+            return '\n'.join(self._get_task_tag_diffs(
+                self.value,
+                self.old_value,
+            ))
         elif self.container:
             return u'Changed %s of container "%s" to: "%s" (was: "%s")' % (
                 self.field,
@@ -1280,6 +1323,38 @@ def _get_docker_label_diffs(container, dockerlabels, old_dockerlabels):
                 diffs.append(message)
         return diffs
 
+    @staticmethod
+    def _get_task_tag_diffs(task_tag_list, old_task_tag_list):
+        msg = u'Changed task tag "%s" value to: "%s"'
+        msg_added = u'Added task tag "%s" with: "%s"'
+        msg_removed = u'Removed task tag "%s"'
+        tags_dict = __class__._tag_list_to_dict(task_tag_list)
+        old_tags_dict = __class__._tag_list_to_dict(old_task_tag_list)
+        diffs = []
+        for key, value in tags_dict.items():
+            old_value = old_tags_dict.get(key)
+            if value and not old_value:
+                message = msg_added % (key, value)
+                diffs.append(message)
+            elif value != old_value:
+                message = msg % (key, value)
+                diffs.append(message)
+        for old_key in old_tags_dict.keys():
+            if old_key not in tags_dict.keys():
+                message = msg_removed % (old_key)
+                diffs.append(message)
+        return diffs
+
+    @staticmethod
+    def _tag_list_to_dict(tag_list):
+        tag_dict = {}
+        if tag_list:
+          for tag in tag_list:
+              key = tag["key"]
+              value = tag["value"]
+              tag_dict[key] = value
+        return tag_dict
+
     @staticmethod
     def _get_secrets_diffs(container, secrets, old_secrets):
         msg = u'Changed secret "%s" of container "%s" to: "%s"'

tests/test_cli.py

@@ -516,6 +516,41 @@ def test_deploy_exclusive_docker_label(get_client, runner):
     assert u'Successfully changed task definition to: test-task:2' in result.output
     assert u'Deployment successful' in result.output
 
+@patch('ecs_deploy.cli.get_client')
+def test_deploy_task_task_tag_merge(get_client, runner):
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key')
+    result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '--task-tag', 'key1', 'value1', '--task-tag', 'key2', 'value2'))
+    assert result.exit_code == 0
+    assert not result.exception
+    assert u"Deploying based on task definition: test-task:1" in result.output
+    assert u"Updating task definition" in result.output
+    assert u'Added task tag "key1" with: "value1"' in result.output
+    assert u'Added task tag "key2" with: "value2"' in result.output
+    assert u'Successfully created revision: 2' in result.output
+
+@patch('ecs_deploy.cli.get_client')
+def test_update_task_task_tag_merge(get_client, runner):
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key')
+    result = runner.invoke(cli.update, (TASK_DEFINITION_ARN_2, '--task-tag', 'old-key', 'replace-value', '--task-tag', 'new-key', 'new-value'))
+    assert result.exit_code == 0
+    assert not result.exception
+    assert u"Update task definition based on: test-task:2" in result.output
+    assert u"Updating task definition" in result.output
+    assert u'Changed task tag "old-key" value to: "replace-value"' in result.output
+    assert u'Added task tag "new-key" with: "new-value"' in result.output
+    assert u'Successfully created revision: 2' in result.output
+
+@patch('ecs_deploy.cli.get_client')
+def test_update_task_exclusive_task_tag(get_client, runner):
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key')
+    result = runner.invoke(cli.update, (TASK_DEFINITION_ARN_2, '--task-tag', 'new-key', 'new-value', '--exclusive-task-tags'))
+    assert result.exit_code == 0
+    assert not result.exception
+    assert u"Update task definition based on: test-task:2" in result.output
+    assert u"Updating task definition" in result.output
+    assert u'Removed task tag "old-key"' in result.output
+    assert u'Added task tag "new-key" with: "new-value"' in result.output
+    assert u'Successfully created revision: 2' in result.output
 
 @patch('ecs_deploy.cli.get_client')
 def test_deploy_exclusive_secret(get_client, runner):

tests/test_ecs.py

@@ -379,7 +379,8 @@
 }
 
 RESPONSE_TASK_DEFINITION_2 = {
-    u"taskDefinition": PAYLOAD_TASK_DEFINITION_2
+    u"taskDefinition": PAYLOAD_TASK_DEFINITION_2,
+    u'tags': [{'key': 'old-key', 'value': 'old-value'}]
 }
 
 RESPONSE_TASK_DEFINITION_3 = {