nat64 - Fix molecule, and touch ups to stabilize

The molecule tests was failing because the interfaces did not end up in
the 'libvirt' firewall zone.

Added the zone attribute to the libvirt bridge element. This should work
according to libvirt documentation, but for some reason not in this
test environment. So, had to do that manually with firewalld.

Also:
 * Pin dnsmasq to the v2.89 tag.
   Should be more stable than running of the main branch with frequent changes.
 * Set do-daemonize: no in unbound config.
   This removes a warning logged by the unbound service, do-daemonize and
   use-systemd should not be used togheter.

There is a lot of "crude" debug info gathering added in the molecule
tests, I think we should keep that around as is to make troubleshooting
any potentital future issues in this job.

docs/dictionary/en-custom.txt

@@ -152,6 +152,7 @@ fci
 fedoraproject
 fil
 fips
+firewalld
 flbxutz
 fmw
 fqdn

roles/nat64_appliance/README.md

@@ -10,6 +10,7 @@
 * `cifmw_nat64_appliance_workdir`: (String) Working directory. Defaults to `{{ cifmw_nat64_appliance_basedir }}/nat64_appliance`.
 * `cifmw_nat64_appliance_venv_dir`: (String) Python virtual environment directory. Defaults to `{{ cifmw_nat64_appliance_workdir }}/venv`.
 * `cifmw_nat64_libvirt_uri`: (String) The libvirt URI for the hypervisor to deploy on. Defaults to `qemu:///system`.
+* `cifmw_nat64_firewall_zone`: (String) The value for the firewalld `zone` attribute on the libvirt networks. Defaults to `libvirt`.
 * `cifmw_nat64_network_ipv4_name`: (String) Name of the nat64 IPv4 libvirt network. Defaults to: `nat64-net-v4`.
 * `cifmw_nat64_network_ipv4_bridge_name`: (String) Bridge name for the nat64 IPv4 libvirt network. Defaults to: `br-64v4`.
 * `cifmw_nat64_network_ipv4_address`: (String) IP address for the nat64 IPv4 libvirt network. Defaults to: `172.31.255.1`.

roles/nat64_appliance/defaults/main.yml

@@ -22,6 +22,7 @@ cifmw_nat64_appliance_workdir: "{{ cifmw_nat64_appliance_basedir }}/nat64_applia
 cifmw_nat64_appliance_venv_dir: "{{ cifmw_nat64_appliance_workdir }}/venv"
 
 cifmw_nat64_libvirt_uri: "qemu:///system"
+cifmw_nat64_firewall_zone: libvirt
 cifmw_nat64_network_ipv4_name: nat64-net-v4
 cifmw_nat64_network_ipv4_bridge_name: br-64v4
 cifmw_nat64_network_ipv4_address: 172.31.255.1

roles/nat64_appliance/files/elements/nat64-router/install.d/70-dnsmasq

@@ -21,6 +21,7 @@ set -o pipefail
 
 pushd /root
 
+# TODO(hjensas): Make the branch (tag) configurable via variable.
 git clone --depth 1 --branch v2.89 git://thekelleys.org.uk/dnsmasq.git
 
 pushd dnsmasq

roles/nat64_appliance/molecule/default/converge.yml

@@ -18,7 +18,7 @@
   hosts: instance
   vars:
     ansible_user_dir: "{{ lookup('env', 'HOME') }}"
-    cifmw_basedir: "/opt/basedir"
+    cifmw_basedir: "{{ ansible_user_dir }}/ci-framework-data"
   tasks:
     - name: Crate SSH keypair
       register: _test_key
@@ -27,6 +27,18 @@
         path: "{{ (ansible_user_dir, '.ssh/id_test') | path_join }}"
         type: "ecdsa"
 
+    - name: Enable forwarding in the libvirt zone
+      become: true
+      ansible.builtin.command:
+        cmd: >-
+          firewall-cmd --permanent --zone libvirt --add-forward
+
+    - name: Restart firewalld.service
+      become: true
+      ansible.builtin.systemd_service:
+        name: firewalld
+        state: restarted
+
     - name: Discover latest image
       when:
         - cifmw_discovered_image_url is not defined
@@ -53,6 +65,15 @@
       ansible.builtin.include_role:
         name: nat64_appliance
 
+    - name: Fix permissions on logs dir - because we ran dib as root
+      become: true
+      ansible.builtin.file:
+        path: "{{ cifmw_basedir }}/logs"
+        state: directory
+        recurse: true
+        owner: "{{ ansible_user_id }}"
+        group: "{{ ansible_user_gid }}"
+
     - name: Fix permissions on nat64_appliance dir - because we ran dib as root
       become: true
       ansible.builtin.file:
@@ -82,7 +103,7 @@
           <network>
             <name>br-mol</name>
             <forward mode='open'/>
-            <bridge name='br-mol' stp='on' delay='0'/>
+            <bridge name='br-mol' zone='libvirt' stp='on' delay='0'/>
             <ip family='ipv6' address='2620:cf:cf:aaaa::1' prefix='64'/>
             <dns>
               <forwarder addr='2620:cf:cf:fc00::2'/>
@@ -108,6 +129,23 @@
         name: br-mol
         uri: 'qemu:///system'
 
+    # TODO(hjensas): With the zone set in libvirt XML this bridge "should"
+    # already be in this zone. But logs indicate that it is not.
+    # See: https://libvirt.org/formatnetwork.html
+    - name: Make sure br-mol bridge is in the libvirt firewalld zone
+      become: true
+      ansible.posix.firewalld:
+        zone: libvirt
+        interface: br-mol
+        state: enabled
+        permanent: true
+
+    - name: Restart firewalld.service
+      become: true
+      ansible.builtin.systemd_service:
+        name: firewalld
+        state: restarted
+
     - name: Generate test node UUID
       ansible.builtin.set_fact:
         test_node_uuid: "{{ 99999999 | random | to_uuid | lower }}"
@@ -261,18 +299,149 @@
       ansible.builtin.add_host:
         name: test-node
         groups:
-          - test_nodes
+          - testnodes
         ansible_host: '2620:cf:cf:aaaa::101'
         ansible_ssh_user: 'cloud-user'
         ansible_ssh_private_key_file: "{{ _test_key.filename }}"
         ansible_ssh_extra_args: '-o StrictHostKeyChecking=no'
 
+    - name: Add nat64 appliance to the invetory
+      ansible.builtin.add_host:
+        name: nat64-appliance
+        groups:
+          - nat64appliances
+        ansible_host: '172.31.255.2'
+        ansible_ssh_user: 'cloud-user'
+        ansible_ssh_private_key_file: "{{ _test_key.filename }}"
+        ansible_ssh_extra_args: '-o StrictHostKeyChecking=no'
+
     # Even tough the node respond on port 22, let some time to ensure
     # ssh access for users is available.
     - name: Wait a little to let the test instance boot.
       ansible.builtin.pause:
         seconds: 20
 
+    - name: Run some commands to test the nat64-appliance DNS64 functions
+      register: _nat64_appliance_dns64_debug
+      ansible.builtin.shell: |
+        echo "--- ping the DNS service listening address ---"
+        ping -c 4 2620:cf:cf:fc00::2 || true
+        echo
+        echo "--- Try to resolve a name using 2620:cf:cf:fc00::2 ---"
+        dig AAAA @2620:cf:cf:fc00::2 example.com
+        echo
+        echo "--- Try to resolve a name forwarding on test-network 2620:cf:cf:aaaa::1 ---"
+        dig AAAA @2620:cf:cf:aaaa::1 example.com
+        echo
+        echo "--- Try to ping exmaple.com ---"
+        ping -c 4 example.com || true
+
+    - name: Grab some info from the test node
+      become: true
+      delegate_to: test-node
+      register: _test_node_debug_info
+      ansible.builtin.shell: |
+        echo "--- ip addr show ---"
+        ip addr show
+        echo
+        echo "--- ip -6 route show ---"
+        ip -6 route show
+        echo
+        echo "--- /etc/resolv.conf ---"
+        cat /etc/resolv.conf
+        echo
+        echo "--- NetworkManager system-connections ---"
+        for file in $(find /etc/NetworkManager/system-connections/ -type f); do
+          echo "--- $file ---"
+          cat $file
+          echo
+        done
+
+    - name: Grab some info from the nat64 appliance
+      become: true
+      delegate_to: nat64-appliance
+      register: _nat64_appliance_debug_info
+      ansible.builtin.shell: |
+        echo "--- ip addr show ---"
+        ip addr show
+        echo
+        echo "--- ip -4 route show ---"
+        ip -4 route show
+        echo
+        echo "--- ip -6 route show ---"
+        ip -6 route show
+        echo
+        echo "--- /etc/resolv.conf ---"
+        cat /etc/resolv.conf
+        echo
+        echo "--- NetworkManager system-connections ---"
+        for file in $(find /etc/NetworkManager/system-connections/ -type f); do
+          echo "--- $file ---"
+          cat $file
+          echo
+        done
+        echo
+        echo "--- systemctl status unbound.service ---"
+        systemctl status unbound.service
+        echo
+        echo "--- systemctl status nat64-v4-dnsmasq.service ---"
+        systemctl status nat64-v4-dnsmasq.service
+        echo
+        echo "--- systemctl status nat64-v6-dnsmasq.service ---"
+        systemctl status nat64-v6-dnsmasq.service
+
+    - name: Grab the journal from the nat64 appliance
+      become: true
+      delegate_to: nat64-appliance
+      register: _nat64_appliance_journal
+      ansible.builtin.shell: |
+        journalctl -b --no-pager
+
+    - name: Grab some info from hypervisor
+      become: true
+      delegate_to: instance
+      register: _hypervisor_info
+      ansible.builtin.shell: |
+        echo "--- ip addr show ---"
+        ip addr show
+        echo
+        echo "--- ip -4 route show ---"
+        ip -4 route show
+        echo
+        echo "--- ip -6 route show ---"
+        ip -6 route show
+        echo
+        echo "--- firewall - list-all-zones"
+        firewall-cmd --list-all-zones
+        echo
+        echo "--- sysctl - list all"
+        sysctl -a
+
+    - name: Write test-node info to file
+      ansible.builtin.copy:
+        dest: "{{ cifmw_basedir }}/logs/test_node_info.log"
+        content: "{{ _test_node_debug_info.stdout }}"
+
+    - name: Write nat64-appliance info to file
+      ansible.builtin.copy:
+        dest: "{{ cifmw_basedir }}/logs/nat64_appliance_node_info.log"
+        content: "{{ _nat64_appliance_debug_info.stdout }}"
+
+    - name: Write nat64-appliance journal to file
+      ansible.builtin.copy:
+        dest: "{{ cifmw_basedir }}/logs/nat64_appliance_journal.log"
+        content: "{{ _nat64_appliance_journal.stdout }}"
+
+    - name: Write nat64-appliance DNS64 debug to file
+      ansible.builtin.copy:
+        dest: "{{ cifmw_basedir }}/logs/nat64_appliance_dns64_debug.log"
+        content: "{{ _nat64_appliance_dns64_debug.stdout }}"
+
+    - name: Write hypervisor info to file
+      ansible.builtin.copy:
+        dest: "{{ cifmw_basedir }}/logs/hypervisor_info.log"
+        content: "{{ _hypervisor_info.stdout }}"
+
     - name: Ping example.com (delegate to test-node)
       delegate_to: test-node
       register: _ping_example_com

roles/nat64_appliance/molecule/default/prepare.yml

@@ -17,7 +17,8 @@
 - name: Prepare
   hosts: all
   vars:
-    cifmw_basedir: "/opt/basedir"
+    ansible_user_dir: "{{ lookup('env', 'HOME') }}"
+    cifmw_basedir: "{{ ansible_user_dir }}/ci-framework-data"
   pre_tasks:
     - name: Create custom basedir
       become: true
@@ -27,6 +28,13 @@
         owner: zuul
         group: zuul
         mode: "0755"
+    - name: Install some debug utils
+      become: true
+      ansible.builtin.package:
+        name:
+          - bind-utils
+          - tcpdump
+        state: present
   roles:
     - role: test_deps
     - role: libvirt_manager

roles/nat64_appliance/tasks/deploy.yml

@@ -54,6 +54,26 @@
     name: "{{ cifmw_nat64_network_ipv6_name }}"
     uri: "{{ cifmw_nat64_libvirt_uri }}"
 
+# TODO(hjensas): With the zone set in libvirt XML these bridges "should"
+# already be in this zone. But logs indicate that they are not.
+# See: https://libvirt.org/formatnetwork.html
+- name: Make sure all bridges are in the libvirt firewalld zone
+  become: true
+  ansible.posix.firewalld:
+    zone: "{{ cifmw_nat64_firewall_zone }}"
+    interface: "{{ item }}"
+    state: enabled
+    permanent: true
+  loop:
+    - "{{ cifmw_nat64_network_ipv6_bridge_name }}"
+    - "{{ cifmw_nat64_network_ipv4_bridge_name }}"
+
+- name: Restart firewalld.service
+  become: true
+  ansible.builtin.systemd_service:
+    name: firewalld
+    state: restarted
+
 - name: "Generate nat64-appliance UUID"
   ansible.builtin.set_fact:
     nat64_appliance_uuid: "{{ 99999999 | random | to_uuid | lower }}"

roles/nat64_appliance/templates/ipv4_network.xml.j2

@@ -5,6 +5,7 @@
   />
   <bridge
     name='{{ cifmw_nat64_network_ipv4_bridge_name }}'
+    zone='{{ cifmw_nat64_firewall_zone }}'
     stp='on'
     delay='0'
   />

roles/nat64_appliance/templates/ipv6_network.xml.j2

@@ -5,6 +5,7 @@
   />
   <bridge
     name='{{ cifmw_nat64_network_ipv6_bridge_name }}'
+    zone='{{ cifmw_nat64_firewall_zone }}'
     stp='on'
     delay='0'
   />