[WIP] add nodes_addition role

docs/how-to/add-nodes-to-cluster.md

@@ -0,0 +1,63 @@
+# How to: Add nodes to an existing cluster
+
+This guide will run through specifics for adding nodes to existing clusters.
+
+# Prereqs
+The following prerequisites must be fulfilled :
+- New nodes up & running
+- New nodes connected to Cloudera Manager (Agents installed, configured & running)
+
+The same way as for fresh install or upgrade:
+- New nodes are part of the inventory and member of the group 'cluster'
+- Host templates must be assigned to the new nodes
+
+# Configuration
+Two variables to configure:
+```yaml
+nodes_addition_host_templates_prefix: Prefix to be used for Host Templates name in Cloudera Manager (i.e. {{ nodes_addition_host_templates_prefix }}+{{ host_template_name_from_definition }})
+nodes_addition_hosts:
+  - cluster_name: Cluster Name to add below nodes to
+    hosts: List of hostnames to add to the above cluster
+```
+Example values:
+```yaml
+nodes_addition_host_templates_prefix: "nodes-addition-HostTemplate-"
+nodes_addition_hosts:
+  - cluster_name: "Basic Cluster"
+    hosts: "{{ groups.new_worker_nodes }}"
+```
+
+Example inventory:
+```ini
+...
+[cluster_worker_nodes]
+worker-001.lab.local
+worker-002.lab.local
+worker-003.lab.local
+
+[cluster_worker_nodes:vars]
+host_template=Workers
+
+[new_worker_nodes]
+worker-004.lab.local
+worker-005.lab.local
+worker-006.lab.local
+
+[new_worker_nodes:vars]
+host_template=Workers
+
+[cluster:children]
+...
+cluster_worker_nodes
+new_worker_nodes
+```
+
+# Deployment
+Run the role nodes_addition
+Example playbook:
+```yaml
+- name: Add new nodes
+  hosts: cloudera_manager
+  roles:
+    - cloudera.cluster.deployment.nodes_addition
+```

roles/deployment/nodes_addition/defaults/main.yml

@@ -0,0 +1,20 @@
+# Copyright 2023 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+nodes_addition_host_templates_prefix: nodes-addition-HostTemplate-
+nodes_addition_hosts:
+  - cluster_name: Basic Cluster
+    hosts: "{{ groups.new_worker_nodes }}"

roles/deployment/nodes_addition/meta/main.yml

@@ -0,0 +1,18 @@
+# Copyright 2023 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+dependencies:
+  - role: cloudera.cluster.cloudera_manager.api_hosts
+

roles/deployment/nodes_addition/tasks/main.yml

@@ -0,0 +1,62 @@
+# Copyright 2023 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+- name: Add nodes to existing cluster
+  cloudera.cluster.cm_api:
+    endpoint: /clusters/{{ __cluster.cluster_name | urlencode() }}/hosts
+    method: POST
+    body: "{{ lookup('template', 'hosts.j2', convert_data=False) }}"
+  loop: "{{ nodes_addition_hosts }}"
+  loop_control:
+    loop_var: __cluster
+
+- name: Add host templates to existing cluster
+  cloudera.cluster.cm_api:
+    endpoint: /clusters/{{ _cluster.name | urlencode() }}/hostTemplates
+    method: POST
+    body: "{{ lookup('template', 'hostTemplates.j2', convert_data=False) }}"
+  loop: "{{ definition.clusters }}"
+  loop_control:
+    loop_var: _cluster
+  register: host_templates_created
+
+- name: Extract and Group Items
+  set_fact:
+    host_templates_created_list: "{{ host_templates_created.results | json_query('[].json.items[].name') }}"
+
+- name: Wait for parcels to become distributed to new hosts
+  cloudera.cluster.cm_api:
+    endpoint: /clusters/{{ _cluster.name | urlencode() }}/parcels
+  register: parcels_response
+  until: parcels_response.json['items'] | rejectattr('stage', 'in', ["AVAILABLE_REMOTELY", "DOWNLOADED", "ACTIVATED", "UNAVAILABLE"]) | list | length == 0
+  retries: "{{ parcel_poll_max_retries | default(30) }}"
+  delay: "{{ parcel_poll_duration | default(60) }}"
+  loop: "{{ definition.clusters }}"
+  loop_control:
+    loop_var: _cluster
+
+- name: Apply host template to new nodes
+  cloudera.cluster.cm_api:
+    endpoint: "/clusters/{{ _cluster | urlencode() }}/hostTemplates/{{ _host_template }}/commands/applyHostTemplate"
+    method: POST
+    body: "{{ lookup('template', 'hosts_apply_template.j2', convert_data=False) }}"
+  loop: "{{ host_templates_created_list }}"
+  loop_control:
+    loop_var: _host_template
+  vars:
+    _host_template_name: "{{ _host_template | split (nodes_addition_host_templates_prefix) | last }}"
+    _cluster: "{{ definition.host_template_cluster_map[_host_template_name] }}"
+

roles/deployment/nodes_addition/templates/hostTemplates.j2

@@ -0,0 +1,47 @@
+{%- set template_joiner  = joiner(",") -%}
+{%- set new_nodes_json_query = '[?cluster_name == `' ~ _cluster.name ~ '`].hosts[]' -%}
+{%- set new_nodes = nodes_addition_hosts | json_query(new_nodes_json_query) -%}
+{
+  "items": [
+    {%- for (template_name, role_mapping) in _cluster.host_templates.items() -%}
+    {%- if groups['host_template_' ~ template_name] | intersect(new_nodes) | length > 0 -%}
+    {%- set role_config_group_refs = [] -%}
+    {%- set filtered_services = [] -%}
+
+    {%- for (service, roles) in role_mapping.items() -%}
+      {%- set service_ref = service | lower -%}
+      {%- if (_cluster.security.kerberos | default(false) and service == "HUE"
+            and "HUE_SERVER" in roles and not "KT_RENEWER" in roles) -%}
+        {{ roles.append("KT_RENEWER") }}
+      {%- endif -%}
+      {%- for role in roles -%}
+        {%- set (role_type, config_group) = role | cloudera.cluster.extract_role_and_group -%}
+        {{ role_config_group_refs.append("{}-{}-{}".format(service_ref, role_type, config_group)) }}
+      {%- endfor -%}
+  {%- endfor -%}
+  {%- if role_config_group_refs -%}
+  {{ template_joiner() }}
+
+    {
+      "name": "{{ nodes_addition_host_templates_prefix }}{{ template_name }}",
+      "clusterRef": {
+        "clusterName": "{{ _cluster.name }}",
+        "displayName": "{{ _cluster.name }}"
+      },
+      "roleConfigGroupRefs": [
+    {%- set rcg_sep = joiner(",") -%}
+      {%- for rcg in role_config_group_refs -%}
+        {{ rcg_sep() }}
+        {
+          "roleConfigGroupName": "{{ rcg }}"
+        }
+      {%- endfor -%}
+  {%- endif -%}
+
+      ]
+    }
+    {%- endif -%}
+    {%- endfor -%}
+
+  ]
+}

roles/deployment/nodes_addition/templates/hosts.j2

@@ -0,0 +1,12 @@
+{%- set host_joiner = joiner(",") -%}
+{
+  "items": [
+    {%- for host in __cluster.hosts -%}
+    {{ host_joiner() }}
+    {
+    "hostId": "{{ cloudera_manager_api_hosts[host]['id'] }}",
+    "hostname": "{{ cloudera_manager_api_hosts[host]['hostname'] }}"
+    }
+    {% endfor %}
+  ]
+}

roles/deployment/nodes_addition/templates/hosts_apply_template.j2

@@ -0,0 +1,14 @@
+{%- set host_joiner = joiner(",") -%}
+{%- set new_nodes_json_query = '[?cluster_name == `' ~ _cluster ~ '`].hosts[]' -%}
+{%- set new_nodes = nodes_addition_hosts | json_query(new_nodes_json_query) -%}
+{
+  "items": [
+    {%- for host in groups['host_template_' ~ _host_template_name] | intersect(new_nodes) -%}
+    {{ host_joiner() }}
+    {
+    "hostId": "{{ cloudera_manager_api_hosts[host]['id'] }}",
+    "hostname": "{{ cloudera_manager_api_hosts[host]['hostname'] }}"
+    }
+    {% endfor %}
+  ]
+}