From 4de8ebde1e5e327416b2d10ed826ece469415450 Mon Sep 17 00:00:00 2001 From: "mark.otting" Date: Mon, 18 Mar 2024 12:47:32 +0100 Subject: [PATCH 01/10] Add cluster(less) query to ecs_service_info --- plugins/modules/ecs_service_info.py | 67 ++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/plugins/modules/ecs_service_info.py b/plugins/modules/ecs_service_info.py index 02a6abff207..e0d8bade899 100644 --- a/plugins/modules/ecs_service_info.py +++ b/plugins/modules/ecs_service_info.py @@ -29,12 +29,13 @@ type: bool cluster: description: - - The cluster ARNS in which to list the services. + - The cluster ARNS in which to list the services. If not provided, all clusters are listed. required: false - type: str + type: list + elements: str service: description: - - One or more services to get details for + - One or more services to get details for. If not provided, all services are listed. required: false type: list elements: str @@ -55,10 +56,17 @@ details: true register: output -# Basic listing example +# Basic listing example for all services in all clusters - community.aws.ecs_service_info: - cluster: test-cluster + details: true register: output + +# Basic listing example for the list of services in two specific clusters +- community.aws.ecs_service_info: + cluster: + - test-cluster + - prod-cluster + register: output """ RETURN = r""" @@ -161,6 +169,17 @@ def list_services_with_backoff(self, **kwargs): def describe_services_with_backoff(self, **kwargs): return self.ecs.describe_services(**kwargs) + def list_clusters(self): + try: + paginator = self.ecs.get_paginator("list_clusters") + response = paginator.paginate().build_full_result() + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + self.module.fail_json_aws(e, msg="Couldn't list ECS clusters") + clusters = list(response["clusterArns"]) + if not clusters: + self.module.fail_json_aws(e, msg="Account does not have any ECS clusters") + return clusters + def list_services(self, cluster): fn_args = dict() if cluster and cluster is not None: @@ -203,6 +222,22 @@ def extract_service_from(self, service): e["createdAt"] = str(e["createdAt"]) return service + def get_cluster_services(self): + if self.module.params["cluster"]: + clusters = self.module.params["cluster"] + else: + clusters = self.list_clusters() + + cluster_services = {} + for cluster in clusters: + services = self.list_services(cluster)["services"] + if self.module.params["service"]: + services_found = [service for service in self.module.params["service"] if service in services] + cluster_services[cluster] = services_found + else: + cluster_services[cluster] = services + return cluster_services + def chunks(l, n): """Yield successive n-sized chunks from l.""" @@ -215,7 +250,7 @@ def main(): argument_spec = dict( details=dict(type="bool", default=False), events=dict(type="bool", default=True), - cluster=dict(), + cluster=dict(type="list", elements="str"), service=dict(type="list", elements="str", aliases=["name"]), ) @@ -224,18 +259,20 @@ def main(): show_details = module.params.get("details") task_mgr = EcsServiceManager(module) + cluster_services = task_mgr.get_cluster_services() + if show_details: - if module.params["service"]: - services = module.params["service"] - else: - services = task_mgr.list_services(module.params["cluster"])["services"] ecs_info = dict(services=[], services_not_running=[]) - for chunk in chunks(services, 10): - running_services, services_not_running = task_mgr.describe_services(module.params["cluster"], chunk) - ecs_info["services"].extend(running_services) - ecs_info["services_not_running"].extend(services_not_running) + for cluster, services in cluster_services.items(): + for chunk in chunks(services, 10): + running_services, services_not_running = task_mgr.describe_services(cluster, chunk) + ecs_info["services"].extend(running_services) + ecs_info["services_not_running"].extend(services_not_running) else: - ecs_info = task_mgr.list_services(module.params["cluster"]) + service_arns = [] + for service_list in cluster_services.values(): + service_arns.extend(service_list) + ecs_info = dict(services=service_arns) module.exit_json(changed=False, **ecs_info) From eb16fcf7d5e402e9a70b4d85150edfa6c8652b7c Mon Sep 17 00:00:00 2001 From: "mark.otting" Date: Mon, 18 Mar 2024 14:15:19 +0100 Subject: [PATCH 02/10] Added test to validate commits with no cluster --- .../ecs_cluster/tasks/20_ecs_service.yml | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml b/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml index 3c4bbcb28c7..b18c050d89f 100644 --- a/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml +++ b/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml @@ -400,7 +400,6 @@ - name: obtain facts for existing service in the cluster ecs_service_info: - cluster: "{{ ecs_cluster_name }}" service: "{{ ecs_service_name }}" details: true events: false @@ -437,6 +436,19 @@ that: - "'networkConfiguration' in ecs_service_info.services[0]" +- name: obtain facts for all clusters + ecs_service_info: + details: true + events: false + register: ecs_service_info + +- name: assert that service info is available + assert: + that: + - "ecs_service_info.services|length == 1" + - "ecs_service_info.services_not_running|length == 0" + - "ecs_service_info.services[0].clusterArn == ecs_cluster_name" + - name: attempt to get facts from missing task definition ecs_taskdefinition_info: task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_task_definition.taskdefinition.revision + 1}}" @@ -547,7 +559,7 @@ >> "rolloutStateReason": "ECS deployment ecs-svc/5156684577543126023 in progress.", constraints and placement strategies are only changeable if the rollout state is "COMPLETED" - + a) ecs_service has currently no waiter function. so this is a DIY waiter b) the state reached never "COMPLETED" because something if wrong with the ECS EC2 Instances or the network setup. The EC2 instance never arrived as an active instance in the cluster. @@ -555,9 +567,9 @@ >> no container instance met all of its requirements. Reason: No Container Instances were found in your cluster. >> For more information, see the Troubleshooting section of the Amazon ECS Developer Guide. >> ec2_instance networking does not work correctly, no instance available for the cluster - + Because all of this, all following tasks, that test the change of a constraint or placement stragegy are - using `force_new_deployment: true`. That ignores a) and b). + using `force_new_deployment: true`. That ignores a) and b). ignore_errors: true ecs_service_info: name: "{{ ecs_service_name }}-constraint" From d381118328ac9f4d717d7a59b7c011655cf8fe95 Mon Sep 17 00:00:00 2001 From: "mark.otting" Date: Mon, 18 Mar 2024 16:04:26 +0100 Subject: [PATCH 03/10] Fixed not being able to deal with service names --- plugins/modules/ecs_service_info.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/modules/ecs_service_info.py b/plugins/modules/ecs_service_info.py index e0d8bade899..ee0dd3cccd2 100644 --- a/plugins/modules/ecs_service_info.py +++ b/plugins/modules/ecs_service_info.py @@ -230,12 +230,16 @@ def get_cluster_services(self): cluster_services = {} for cluster in clusters: - services = self.list_services(cluster)["services"] + service_arns = self.list_services(cluster)["services"] if self.module.params["service"]: - services_found = [service for service in self.module.params["service"] if service in services] + service_names = [service.split("/")[-1] for service in service_arns] + services_found = [] + for service in self.module.params["service"]: + if service in service_names or service in service_arns: + services_found.append(service) cluster_services[cluster] = services_found else: - cluster_services[cluster] = services + cluster_services[cluster] = service_arns return cluster_services From c1028cfa06bf4b7af35213a1ab73df2e64e73999 Mon Sep 17 00:00:00 2001 From: "mark.otting" Date: Mon, 18 Mar 2024 17:33:51 +0100 Subject: [PATCH 04/10] Fixed using the wrong service name in tests --- tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml b/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml index b18c050d89f..9475f39e03f 100644 --- a/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml +++ b/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml @@ -400,7 +400,7 @@ - name: obtain facts for existing service in the cluster ecs_service_info: - service: "{{ ecs_service_name }}" + service: "{{ ecs_service_name }}2" details: true events: false register: ecs_service_info From 0d1ebeef171bb9b25784d35256518b99a4582273 Mon Sep 17 00:00:00 2001 From: "mark.otting" Date: Mon, 18 Mar 2024 23:53:56 +0100 Subject: [PATCH 05/10] Modified integration test. Given that we can now pass multiple clusters, returning a non-existent service in a cluster result does not make sense. --- tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml b/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml index 9475f39e03f..16a065fab23 100644 --- a/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml +++ b/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml @@ -422,7 +422,7 @@ - name: assert that non-existent service is missing assert: that: - - "ecs_service_info.services_not_running[0].reason == 'MISSING'" + - "ecs_service_info.services|length == 0" - name: obtain specific ECS service facts ecs_service_info: From 253a097043939ced4a364815b449177ca6d3f779 Mon Sep 17 00:00:00 2001 From: "mark.otting" Date: Tue, 19 Mar 2024 00:49:42 +0100 Subject: [PATCH 06/10] Last fix to integration tests --- .../integration/targets/ecs_cluster/tasks/20_ecs_service.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml b/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml index 16a065fab23..59c454762e0 100644 --- a/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml +++ b/tests/integration/targets/ecs_cluster/tasks/20_ecs_service.yml @@ -445,9 +445,9 @@ - name: assert that service info is available assert: that: - - "ecs_service_info.services|length == 1" + - "ecs_service_info.services|length == 3" - "ecs_service_info.services_not_running|length == 0" - - "ecs_service_info.services[0].clusterArn == ecs_cluster_name" + - "ecs_cluster_name in ecs_service_info.services[0].clusterArn" - name: attempt to get facts from missing task definition ecs_taskdefinition_info: From eed692dfe666b678712cca944c6a8925d73e8d27 Mon Sep 17 00:00:00 2001 From: Mark Otting Date: Tue, 19 Mar 2024 17:52:26 +0100 Subject: [PATCH 07/10] Update plugins/modules/ecs_service_info.py Co-authored-by: Markus Bergholz --- plugins/modules/ecs_service_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/ecs_service_info.py b/plugins/modules/ecs_service_info.py index ee0dd3cccd2..08095ab60bb 100644 --- a/plugins/modules/ecs_service_info.py +++ b/plugins/modules/ecs_service_info.py @@ -29,7 +29,7 @@ type: bool cluster: description: - - The cluster ARNS in which to list the services. If not provided, all clusters are listed. + - The cluster ARNS in which to list the services. If not provided, all clusters are listed. required: false type: list elements: str From f63e6250e871c1f86db17a859b4d874424c769c6 Mon Sep 17 00:00:00 2001 From: Mark Otting Date: Tue, 19 Mar 2024 17:52:33 +0100 Subject: [PATCH 08/10] Update plugins/modules/ecs_service_info.py Co-authored-by: Markus Bergholz --- plugins/modules/ecs_service_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/ecs_service_info.py b/plugins/modules/ecs_service_info.py index 08095ab60bb..887976047d0 100644 --- a/plugins/modules/ecs_service_info.py +++ b/plugins/modules/ecs_service_info.py @@ -66,7 +66,7 @@ cluster: - test-cluster - prod-cluster - register: output + register: output """ RETURN = r""" From 049e728ff843694e581506d3f673a8f2d986151e Mon Sep 17 00:00:00 2001 From: Mark Otting Date: Tue, 19 Mar 2024 17:52:40 +0100 Subject: [PATCH 09/10] Update plugins/modules/ecs_service_info.py Co-authored-by: Markus Bergholz --- plugins/modules/ecs_service_info.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/modules/ecs_service_info.py b/plugins/modules/ecs_service_info.py index 887976047d0..811d097fea3 100644 --- a/plugins/modules/ecs_service_info.py +++ b/plugins/modules/ecs_service_info.py @@ -60,7 +60,6 @@ - community.aws.ecs_service_info: details: true register: output - # Basic listing example for the list of services in two specific clusters - community.aws.ecs_service_info: cluster: From 456012ca2764047d834317ec4f988282a9a4339e Mon Sep 17 00:00:00 2001 From: "mark.otting" Date: Tue, 2 Apr 2024 00:38:04 +0200 Subject: [PATCH 10/10] Removed explicit error when no clusters or services exist --- plugins/modules/ecs_service_info.py | 3 +-- .../targets/ecs_cluster/tasks/10_ecs_cluster.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/plugins/modules/ecs_service_info.py b/plugins/modules/ecs_service_info.py index ee0dd3cccd2..3780f7681b6 100644 --- a/plugins/modules/ecs_service_info.py +++ b/plugins/modules/ecs_service_info.py @@ -175,9 +175,8 @@ def list_clusters(self): response = paginator.paginate().build_full_result() except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: self.module.fail_json_aws(e, msg="Couldn't list ECS clusters") + # Note that the clusters list can be emptu if there are no clusters (and thus no services) clusters = list(response["clusterArns"]) - if not clusters: - self.module.fail_json_aws(e, msg="Account does not have any ECS clusters") return clusters def list_services(self, cluster): diff --git a/tests/integration/targets/ecs_cluster/tasks/10_ecs_cluster.yml b/tests/integration/targets/ecs_cluster/tasks/10_ecs_cluster.yml index f2c868674f8..b593b337b9f 100644 --- a/tests/integration/targets/ecs_cluster/tasks/10_ecs_cluster.yml +++ b/tests/integration/targets/ecs_cluster/tasks/10_ecs_cluster.yml @@ -1,4 +1,16 @@ # cluster "{{ ecs_cluster_name }}" is used for ecs_service tests +- name: get existing service info + ecs_service_info: + details: true + events: false + register: ecs_service_info + +- name: check that no services exist and the module does not error + assert: + that: + - ecs_service_info.services is defined + - ecs_service_info.services | length == 0 + - name: create an ECS cluster ecs_cluster: name: "{{ ecs_cluster_name }}"