Merge pull request #243 from shiftstack/idempotency

Make playbook idempotent

Author: chenwng

playbooks/install_stack.yaml

@@ -62,6 +62,13 @@
               - /opt/exported-data/extra-host-file-entries.json
               - /opt/exported-data/all-nodes-extra-map-data.json
 
+    - name: Check if TripleO has already been deployed
+      ansible.builtin.stat:
+        path: /etc/openstack/clouds.yaml
+      register: tripleo_deployed
+      become: true
+      become_user: root
+
     - name: Enable SSL
       when: ssl_enabled
       block:
@@ -75,7 +82,9 @@
               - /usr/share/openstack-tripleo-heat-templates/environments/ssl/inject-trust-anchor.yaml
 
     - name: Generate SSL self-signed certificate on localhost
-      when: ssl_enabled
+      when:
+        - ssl_enabled
+        - not tripleo_deployed.stat.exists
       become: false
       # We run this block on localhost because we don't want to put the CA key on the remote
       # server, which could lead to security problems.
@@ -96,7 +105,9 @@
             cert_name: standalone
 
     - name: Prepare the host for SSL
-      when: ssl_enabled
+      when:
+        - ssl_enabled
+        - not tripleo_deployed.stat.exists
       no_log: true
       block:
         - name: Read and clean SSL files
@@ -307,6 +318,7 @@
             neutron_bridge_mappings: "{{ base_bridge_mappings }}"
 
     - name: Create standalone_parameters.yaml
+      when: not tripleo_deployed.stat.exists
       no_log: true
       ansible.builtin.template:
         mode: '644'
@@ -506,7 +518,42 @@
       when:
         - ceph_enabled
 
+    - name: Clean up stale heat processes from previous deploy
+      become: true
+      become_user: root
+      when: not tripleo_deployed.stat.exists
+      block:
+        - name: Check if heat-all is already running (previous deploy with --keep-running)
+          ansible.builtin.command: pgrep -f heat-all
+          register: heat_running
+          failed_when: false
+          changed_when: false
+
+        - name: Kill stale heat-all processes
+          # Each tripleo deploy forks a new heat-all without killing old ones.
+          # Multiple heat-all processes sharing port 8006 via SO_REUSEPORT
+          # causes API requests to be randomly routed to the wrong instance,
+          # resulting in 'Stack create failed'.
+          ansible.builtin.command: pkill -9 -f heat-all
+          changed_when: true
+          when: heat_running.rc == 0
+
+        - name: Wait for heat processes to terminate
+          ansible.builtin.command: pgrep -f heat-all
+          register: heat_check
+          failed_when: false
+          changed_when: false
+          retries: 10
+          delay: 1
+          until: heat_check.rc != 0
+
+        - name: Remove stack update mark to ensure a clean stack create
+          ansible.builtin.file:
+            path: /var/lib/tripleo-heat-installer/update_mark_standalone
+            state: absent
+
     - name: Run TripleO deploy
+      when: not tripleo_deployed.stat.exists
       ansible.builtin.import_role:
         name: tripleo.operator.tripleo_deploy
       vars:

playbooks/network.yaml

@@ -67,19 +67,6 @@
             ipv6:
               enabled: false
 
-    # this saves the static route configuration including the default route
-    # prior to filtering later
-
-    - name: Save route info # noqa no-changed-when
-      ansible.builtin.command: nmstatectl show --json
-      register: pre
-
-    - name: Create fact for nmstate_routes
-      ansible.builtin.set_fact:
-        nmstate_routes: "{{ pre.stdout | from_json | json_query(q) }}"
-      vars:
-        q: "routes.config"
-
     # This works round a TripleO installation failure caused by NM-managed
     # interfaces 'failing' because:
     # * The default NM configuration specifies DHCP
@@ -112,24 +99,22 @@
       args:
         stdin: "{{ network_state | to_nice_json }}"
       vars:
+        # Do not include routes in the desired state. nmstatectl apply
+        # without a routes section leaves existing routes untouched.
+        # Previously we captured and replayed routes, but on re-run
+        # (idempotency) the captured routes reference interfaces that
+        # nmstate cannot manage (e.g. eno1 enslaved to br-ex, or
+        # br-hostonly which nmstate considers an unsupported tun type),
+        # causing VerificationError failures.
         network_state:
           interfaces: "{{ nmstate_ifs }}"
-          # add saved static routes
-          routes:
-            config: "{{ nmstate_routes }}"
       register: nmstateset
 
-    - name: Set fact for nmstate checkpoing on RHEL8
-      when:
-        - ansible_facts.distribution_major_version == "8"
-      ansible.builtin.set_fact:
-        checkpoint: "{{ (nmstateset.stdout_lines | last).split()[1] }}"
-
-    - name: Set fact for nmstate checkpoing on RHEL9
-      when:
-        - ansible_facts.distribution_major_version == "9"
+    - name: Extract nmstate checkpoint from output
       ansible.builtin.set_fact:
-        checkpoint: "{{ (nmstateset.stderr_lines | last).split()[-1] }}"
+        checkpoint: >-
+          {{ (nmstateset.stdout + nmstateset.stderr)
+             | regex_search('/org/freedesktop/NetworkManager/Checkpoint/\d+') }}
 
     - name: Fail if the checkpoint has not been found or is incorrect
       when: