Fix kernel installation on Ubuntu with some specific variants (like aws) + refactor code

Author: JGoutin

.ansible-lint

@@ -2,3 +2,4 @@
 skip_list:
   # Required to list with extra arguments
   - '303'
+  - role-name

README.md

@@ -21,6 +21,7 @@ Role Variables
   Default to any version.
 * **reboot_on_kernel_update**: If True, reboot the system if the kernel was updated.
   Default to `true`.
+* **kernel_variant**: If specified on a Debian based distributions, use the required kernel variant (like "", "common", "generic", "aws", "azure", ...) else use the current kernel variant.
 
 Example Playbook
 ----------------

azure-pipelines.yml

@@ -67,9 +67,13 @@ jobs:
         displayName: Test Ansible role
         condition: always()
         env:
-          ANSIBLE_STDOUT_CALLBACK: debug
+          ANSIBLE_DISPLAY_SKIPPED_HOSTS: "False"
           ANSIBLE_FORCE_COLOR: "True"
+          ANSIBLE_FORKS: "30"
           ANSIBLE_NOCOLOR": "False"
+          ANSIBLE_PIPELINING: "True"
+          ANSIBLE_SSH_ARGS: "-o ControlMaster=auto -o ControlPersist=60s"
+          ANSIBLE_STDOUT_CALLBACK: debug
 
   - job:
     displayName: Ansible Galaxy publish

filter_plugins/main.py

@@ -15,17 +15,14 @@ def _rhel_kernel_info(packages, kernel_version, current_version):
     """
     kernels = list()
 
-    # If current version match with required version, use this version
     if current_version.startswith(kernel_version):
         kernel_version = current_version.rsplit(".", 1)[0]
 
-    # List all available kernel version and associated repository
     for line in packages["stdout"].splitlines():
         if line.startswith("kernel.") and not line.startswith("kernel.src"):
             package = line.strip().split()
             kernels.append(dict(version=package[1], repo=package[2]))
 
-    # Return more recent kernel version that match version requirement
     for kernel in reversed(kernels):
         if kernel["version"].startswith(kernel_version):
             return kernel
@@ -66,139 +63,106 @@ def rhel_repo(packages, kernel_version, current_version):
     return _rhel_kernel_info(packages, kernel_version, current_version)["repo"]
 
 
-def deb_kernel(packages, kernel_version, current_version):
+def deb_kernel(packages, kernel_version, current_version, variant=None):
     """
     Return best matching kernel version.
 
     Args:
         packages (dict): apt-cache showpkg output.
         kernel_version (str): Kernel version to install.
         current_version (str): Current kernel version.
+        variant (str): Kernel variant to use ("common", ...) If not specified use
+            current variant.
 
     Returns:
        str: kernel version.
     """
-    kernels = set()
-
-    # If current version match with required version, use this version
-    if current_version.startswith(kernel_version):
-        kernel_version = current_version
+    if current_version.startswith(kernel_version) and not variant:
+        return current_version
 
-    # List all available kernel version and associated repository
-    for line in packages["stdout"].splitlines():
-        line = line.strip()
-        if line.startswith("Package: ") and (
-            line.endswith("-common") or line.endswith("-generic")  # Debian
-        ):  # Ubuntu
-            kernel = line.split()[1]
+    import re
 
-            for string in ("linux-headers-", "common", "generic"):
-                kernel = kernel.replace(string, "")
-            kernel = kernel.strip("-")
+    kernels = set()
+    kernels_add = kernels.add
+    current_version, current_variant = re.match(
+        r"^([0-9-.]+)(-[a-z0-9]+)?$", current_version
+    ).groups()
+    variant = "-" + (
+        variant
+        if not (variant is None or variant.startswith("__omit_place_holder__"))
+        else (current_variant or "")
+    ).lstrip("-")
+    match = re.compile(r"^Package: linux-headers-([a-z0-9-.]+%s)\s*$" % variant).match
 
-            if kernel:
-                kernels.add(kernel)
+    for line in packages["stdout"].splitlines():
+        line_match = match(line)
+        if line_match:
+            kernels_add(line_match.group(1))
 
-    # Sort Kernel versions
     versions = {}
     for kernel in kernels:
-        try:
-            version, build = kernel.split("-", 1)
-        except ValueError:
-            version = kernel
-            build = ""
+        version_info = kernel.split("-")
+        version = version_info[0]
+        build = version_info[1]
         versions[kernel] = list(int(ver) for ver in version.split(".")) + [build]
     kernels = sorted(versions.keys(), key=versions.get, reverse=True)
 
-    # Return more recent kernel package that match version requirement
     for kernel in kernels:
         if kernel.startswith(kernel_version):
             return kernel
 
     raise RuntimeError(
-        'No kernel matching to "%s". Available kernel versions: %s'
-        % (kernel_version, ", ".join(reversed(kernels)))
-    )
-
-
-def _deb_kernel_package(kernel, dist, arch, name):
-    """
-    Return kernel package name.
-
-    Args:
-        kernel (str): Kernel version.
-        dist (str): Distribution.
-        arch (str): Architecture.
-        name (str): Package name.
-
-    Returns:
-       str: kernel package.
-    """
-    # Define package suffix
-    if dist == "Ubuntu":
-        suffix = "generic"
-    elif name == "linux-image":
-        suffix = arch.replace("x86_64", "amd64")
-    else:
-        suffix = "common"
-
-    return "-".join((name, kernel, suffix))
-
-
-def deb_kernel_pkg(packages, kernel_version, current_version, dist, arch, name):
-    """
-    Return kernel package to install.
-
-    Args:
-        packages (dict): apt-cache showpkg output.
-        kernel_version (str): Kernel version to install.
-        current_version (str): Current kernel version.
-        dist (str): Distribution.
-        arch (str): Architecture.
-        name (str): Package name.
-
-    Returns:
-       str: kernel package to install.
-    """
-    return _deb_kernel_package(
-        deb_kernel(packages, kernel_version, current_version), dist, arch, name
+        'No kernel matching to "%s". Current version: %s. Available kernel versions: %s'
+        % (kernel_version, current_version, ", ".join(reversed(kernels)))
     )
 
 
-def deb_installed_kernel(installed, packages, kernel_version, current_version):
+def deb_installed_kernel(installed, kernel_version, arch):
     """
     Return old kernel packages to remove.
 
     Args:
         installed (dict): dpkg -l output.
-        packages (dict): apt-cache showpkg output.
         kernel_version (str): Kernel version to install.
-        current_version (str): Current kernel version.
+        arch (str): Architecture.
 
     Returns:
        list of str: Kernel packages to remove.
     """
-    # Filter installed package to keep
-    to_keep = deb_kernel(packages, kernel_version, current_version)
+    packages = ("linux-image-", "linux-headers-")
+    to_keep = tuple(deb_kernel_package(name, kernel_version, arch) for name in packages)
 
-    # Return installed package to remove
     to_remove = []
     for line in installed["stdout"].splitlines():
         if " linux-" not in line:
             continue
-
-        package = line.split()[1]
-        if (
-            package.startswith("linux-image-") or package.startswith("linux-headers-")
-        ) and not (
-            package.startswith("linux-image-" + to_keep)
-            or package.startswith("linux-headers-" + to_keep)
+        package = line.split()[1].strip()
+        if any(package.startswith(name) for name in packages) and not any(
+            package.startswith(name) for name in to_keep
         ):
             to_remove.append(package)
-
     return to_remove
 
 
+def deb_kernel_package(name, kernel_version, arch):
+    """
+    Check if kernel version match.
+
+    Args:
+        name (str): package name.
+        kernel_version (str): Kernel version to install.
+        arch (str): Architecture.
+
+    Returns:
+        str: Package name.
+    """
+    package = "%s-%s" % (name, kernel_version)
+    if name == "linux-image":
+        # Debian "image" packages does not end by the variant like headers
+        package = package.replace("-common", "-" + arch.replace("x86_64", "amd64"))
+    return package
+
+
 def kernel_match(kernel, kernel_spec):
     """
     Check if kernel version match.
@@ -223,7 +187,7 @@ def filters():
             "rhel_kernel": rhel_kernel,
             "rhel_repo": rhel_repo,
             "deb_kernel": deb_kernel,
-            "deb_kernel_pkg": deb_kernel_pkg,
             "deb_installed_kernel": deb_installed_kernel,
+            "deb_kernel_package": deb_kernel_package,
             "kernel_match": kernel_match,
         }

molecule/default/playbook.yml renamed to molecule/default/converge.yml

@@ -1,6 +1,7 @@
 ---
 - name: Converge
   hosts: all
+  strategy: free
   roles:
     - role: ansible-role-linux-kernel
   vars:

molecule/default/molecule.yml

@@ -24,13 +24,22 @@ provisioner:
     host_vars:
       # Warning: Always specify a version: bad version detected from container
       centos_7:
-        kernel_version: 3.10.0-693
+        kernel_version: "3.10.0-693"
       centos_8:
-        kernel_version: 4.18.0
+        kernel_version: "4.18.0"
       ubuntu_bionic:
-        kernel_version: 4.15.0-55
+        kernel_version: "4.15"
+        kernel_variant: generic
       debian_buster:
         kernel_version: 4.19.0
+        kernel_variant: common
 
 verifier:
   name: testinfra
+
+scenario:
+  test_sequence:
+    - create
+    - converge
+    - idempotence
+    - verify

tasks/main.yml

@@ -28,11 +28,10 @@
                      rhel_kernel(kernel_version, ansible_kernel) }}"
   when: yum_kernel_list is not skipped
 
-- name: Ensure kernel packages versions with YUM
+- name: "Ensure kernel {{ _kernel }} packages are installed with YUM"
   yum:
     name: "{{ item.name }}-{{ _kernel }}"
-    enablerepo: "{{ yum_kernel_list |
-                 rhel_repo(kernel_version, ansible_kernel) }}"
+    enablerepo: "{{ yum_kernel_list | rhel_repo(kernel_version, ansible_kernel) }}"
     allow_downgrade: true
   retries: 10
   delay: 1
@@ -68,11 +67,10 @@
                      rhel_kernel(kernel_version, ansible_kernel) }}"
   when: dnf_kernel_list is not skipped
 
-- name: Ensure kernel packages versions with DNF
+- name: "Ensure kernel {{ _kernel }} packages are installed with DNF"
   dnf:
     name: "{{ item.name }}-{{ _kernel }}"
-    enablerepo: "{{ dnf_kernel_list |
-                 rhel_repo(kernel_version, ansible_kernel) }}"
+    enablerepo: "{{ dnf_kernel_list | rhel_repo(kernel_version, ansible_kernel) }}"
     allow_downgrade: true
   retries: 10
   delay: 1
@@ -103,27 +101,24 @@
 - name: Get available kernel versions with APT
   command: apt-cache showpkg linux-headers-*
   changed_when: false
-  when:
-    - ansible_os_family == 'Debian'
   register: apt_kernel_list
+  when: ansible_os_family == 'Debian'
 
 - name: Get installed packages with APT
   command: dpkg -l
   changed_when: false
-  when:
-    - ansible_os_family == 'Debian'
   register: apt_packages_list
+  when: ansible_os_family == 'Debian'
 
 - name: Set target APT kernel version
   set_fact: _kernel="{{ apt_kernel_list |
-                     deb_kernel(kernel_version, ansible_kernel) }}"
+                     deb_kernel(kernel_version, ansible_kernel,
+                     kernel_variant | default(omit)) }}"
   when: apt_kernel_list is not skipped
 
-- name: Ensure kernel packages versions with APT
+- name: "Ensure kernel {{ _kernel }} packages are installed with APT"
   apt:
-    name: "{{ apt_kernel_list | deb_kernel_pkg(
-           kernel_version, ansible_kernel, ansible_distribution,
-           ansible_architecture, item.name) }}"
+    name: "{{ item.name | deb_kernel_package(_kernel, ansible_architecture) }}"
   retries: 10
   delay: 1
   register: _apt_install
@@ -139,33 +134,31 @@
 
 - name: Ensure any other kernel packages are removed with APT
   apt:
-    name: "{{ apt_packages_list | deb_installed_kernel(
-           apt_kernel_list, kernel_version, ansible_kernel) }}"
+    name: "{{ apt_packages_list |
+           deb_installed_kernel(_kernel, ansible_architecture) }}"
     state: absent
     purge: true
   when: ansible_os_family == 'Debian'
+  tags: molecule-idempotence-notest
 
 - name: Get /var/run/reboot-required stat
   # Note: Should exist on Debian based OS if reboot is required
   stat:
     path: /var/run/reboot-required
   register: reboot_flag
 
-- name: Notify about Kernel update and reboot requierement
-  debug:
-    msg: 'REBOOT REQUIRED, kernel version changed
-          from {{ ansible_kernel }} to {{ _kernel }}.'
-  when:
-    - (not (ansible_kernel | kernel_match(_kernel)) or reboot_flag.stat.exists)
-
-- name: Ensure the new kernel is enabled by rebooting
+- name: "Reboot if kernel changed (from {{ ansible_kernel }} to {{ _kernel }})."
   reboot:
+  register: linux_kernel_rebooted
   when:
     - reboot_on_kernel_update | bool
     - (not (ansible_kernel | kernel_match(_kernel)) or reboot_flag.stat.exists)
 
 - name: Update facts
   setup:
-  when:
-    - reboot_on_kernel_update | bool
-    - (not (ansible_kernel | kernel_match(_kernel)) or reboot_flag.stat.exists)
+  when: linux_kernel_rebooted.changed  # noqa no-handler
+
+- name: Show kernel version after reboot
+  debug:
+    msg: 'The current kernel version is now: {{ ansible_kernel }}.'
+  when: linux_kernel_rebooted.changed  # noqa no-handler