oci patchs: Platform External and CCM manifests

Author: mtulio

docs/guides/installing-agnostic-oci.md

@@ -153,11 +153,21 @@ OCI_COMPARTMENT_ID_IMAGE="<CHANGE_ME:ocid1.compartment.oc1.UUID>"
 EOF
 source ~/.oci/env
 
-cat <<EOF > ./vars-oci-ha.yaml
+cat <<EOF > ~/.openshift/env
+export OCP_CUSTOM_RELEASE="docker.io/mtulio/ocp-release:latest"
+
+OCP_RELEASE_413="quay.io/openshift-release-dev/ocp-release:4.13.0-ec.4-x86_64"
+EOF
+source ~/.openshift/env
+
+CLUSTER_NAME=oci-cr3cmo
+cat <<EOF > ./vars-oci-ha_${CLUSTER_NAME}.yaml
 provider: oci
-cluster_name: oci
+cluster_name: ${CLUSTER_NAME}
 config_cluster_region: us-sanjose-1
 
+#TODO: create compartment validations
+#TODO: allow create compartment from a parent
 oci_compartment_id: ${OCI_COMPARTMENT_ID}
 oci_compartment_id_dns: ${OCI_COMPARTMENT_ID_DNS}
 oci_compartment_id_image: ${OCI_COMPARTMENT_ID_IMAGE}
@@ -169,10 +179,10 @@ config_base_domain: splat-oci.devcluster.openshift.com
 config_ssh_key: "$(cat ~/.ssh/id_rsa.pub)"
 config_pull_secret_file: "${HOME}/.openshift/pull-secret-latest.json"
 
-config_cluster_version: 4.13.0-ec.3-x86_64
+#config_cluster_version: 4.13.0-ec.3-x86_64
 version: 4.13.0-ec.3
-#config_installer_environment:
-#  OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/openshift-release-dev/ocp-release:4.13.0-ec.4-x86_64"
+config_installer_environment:
+  OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "${OCP_CUSTOM_RELEASE}"
 
 controlplane_instance: VM.Standard3.Flex
 controlplane_instance_spec:
@@ -204,7 +214,41 @@ os_mirror_to_oci:
   image_type: QCOW2
   #image_type: VMDK
 
+
+## Apply patches to installer manifests (WIP)
+
+# TODO: we must keep the OCI CCM manifests patch more generic
+
+config_patches:
+- rm-capi-machines
+#- platform-external-kubelet # PROBLEM hangin kubelete (network)
+#- platform-external-kcmo
+- deploy-oci-ccm
+- yaml_patch # working for OCI, but need to know the path
+#- line_regex_patch # ideal, but not working as expected
+
+cfg_patch_yaml_patch_specs:
+    ## patch infra object to create External provider
+  - manifest: /manifests/cluster-infrastructure-02-config.yml
+    patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}'
+
+    ## OCI : Change the namespace from downloaded assets
+  #- manifest: /manifests/oci-cloud-controller-manager-02.yaml
+  #  patch: '{"metadata":{"namespace":"oci-cloud-controller-manager"}}'
+
+cfg_patch_line_regex_patch_specs:
+  - manifest: /manifests/oci-cloud-controller-manager-01-rbac.yaml
+    #search_string: 'namespace: kube-system'
+    regexp: '^(.*)(namespace\\: kube-system)$'
+    #line: 'namespace: oci-cloud-controller-manager'
+    line: '\\1namespace: oci-cloud-controller-manager'
+
+  - manifest:  /manifests/oci-cloud-controller-manager-02.yaml
+    regexp: '^(.*)(namespace\\: kube-system)$'
+    line: '\\1namespace: oci-cloud-controller-manager'
 EOF
+
+
 ```
 
 ### Install the clients
@@ -295,6 +339,28 @@ ansible-playbook mtulio.okd_installer.create_node \
 oc adm certificate approve $(oc get csr  -o json |jq -r '.items[] | select(.status.certificate == null).metadata.name')
 ```
 
+### Create all
+
+```bash
+ansible-playbook mtulio.okd_installer.create_all \
+    -e certs_max_retries=20 \
+    -e cert_wait_interval_sec=60 \
+    -e @./vars-oci-ha.yaml
+```
+
+> TO DO: measure total time
+
+## Review the cluster
+
+```bash
+export KUBECONFIG=${HOME}/.ansible/okd-installer/clusters/${cluster_name}/auth/kubeconfig
+
+oc get nodes
+oc get co
+```
+
+## OPCT setup
+
 - Create the OPCT [dedicated] node
 
 > https://redhat-openshift-ecosystem.github.io/provider-certification-tool/user/#option-a-command-line
@@ -344,30 +410,14 @@ oc adm taint node opct-01.priv.ocp.oraclevcn.com node-role.kubernetes.io/tests="
 ~/opct/bin/openshift-provider-cert-linux-amd64-v0.3.0 report *.tar.gz
 ```
 
+## Generate custom image
 
-### Create all
-
-```bash
-ansible-playbook mtulio.okd_installer.create_all \
-    -e @./vars-oci-ha.yaml \
-    -e certs_max_retries=20 \
-    -e cert_wait_interval_sec=60
 ```
 
-> TO DO: measure total time
-
-## Review the cluster
-
-```bash
-export KUBECONFIG=${HOME}/.ansible/okd-installer/clusters/${cluster_name}/auth/kubeconfig
-
-oc get nodes
-oc get co
 ```
 
 ## Destroy
 
 ```bash
-ansible-playbook mtulio.okd_installer.destroy_cluster \
-    -e @./vars-oci-ha.yaml
+ansible-playbook mtulio.okd_installer.destroy_cluster -e @./vars-oci-ha.yaml
 ```

playbooks/create_all.yaml

@@ -17,16 +17,16 @@
   vars:
     mode: create
 
+- name: OKD Installer | Create all | create stack | IAM
+  ansible.builtin.import_playbook: stack_iam.yaml
+
 - name: OKD Installer | Create all | os_mirror
   ansible.builtin.import_playbook: os_mirror.yaml
   when: os_mirror | d(false)
 
 - name: OKD Installer | Create all | create stack | network
   ansible.builtin.import_playbook: stack_network.yaml
 
-- name: OKD Installer | Create all | create stack | IAM
-  ansible.builtin.import_playbook: stack_iam.yaml
-
 - name: OKD Installer | Create all | create stack | DNS
   ansible.builtin.import_playbook: stack_dns.yaml
 

playbooks/group_vars/all.yaml

@@ -9,7 +9,7 @@ collection_cluster_dir:  "{{ bindir | d(collection_work_dir + '/clusters') }}"
 
 config_install_dir: "{{ collection_cluster_dir }}/{{ cluster_name }}"
 bin_openshift_install: "{{ collection_bin_dir }}/openshift-install"
-bin_oc: "{{ collection_bin_dir }}/openshift-install"
+bin_oc: "{{ collection_bin_dir }}/oc"
 
 ## export CONFIG_PULL_SECRET_FILE=${HOME}/.openshift/pull-secret-latest.jso
 config_pull_secret_file: "{{ lookup('ansible.builtin.env', 'CONFIG_PULL_SECRET_FILE') }}"

playbooks/vars/oci/profiles/ha/node-bootstrap.yaml

@@ -87,12 +87,21 @@ compute_resources:
       # availability_domain: Uocm:PHX-AD-1
       agent_config:
         are_all_plugins_disabled: true
-    
+
+      # Disk Configuration
+      preserve_boot_volume: false
       source_details:
         source_type: image
         boot_volume_size_in_gbs: 120
         boot_volume_vpus_per_gb: 30
 
+      # that config will prevent actions like stop/start (not desired)
+      # preemptible_instance_config:
+      #   preemption_action:
+      #     preserve_boot_volume: false
+      #     type: TERMINATE
+
+      # Network
       create_vnic_details:
         display_name: "{{ cluster_state.infra_id }}-bootstrap-vnic0"
         assign_public_ip: true
@@ -108,6 +117,10 @@ compute_resources:
       metadata:
         user_data: "{{ lookup('template', 'ocp-bootstrap-user-data.j2') | to_nice_json | string | b64encode }}"
 
+      # launch_options:
+      #   firmware: BIOS
+      #   boot_volume_type: PARAVIRTUALIZED
+
     callbacks:
     - name: nlb
       # nlb_name: ocp-nlb

playbooks/vars/oci/profiles/ha/node-compute.yaml

@@ -35,6 +35,7 @@ compute_resources:
     # OCI spec
     spec:
       state: present
+      wait: no
       compartment_id: "{{ oci_compartment_id }}"
       display_name: "{{ cluster_state.infra_id }}-worker-01"
       region: "{{ config_cluster_region }}"
@@ -95,6 +96,7 @@ compute_resources:
     # OCI spec
     spec:
       state: present
+      wait: no
       compartment_id: "{{ oci_compartment_id }}"
       display_name: "{{ cluster_state.infra_id }}-worker-02"
       region: "{{ config_cluster_region }}"
@@ -155,6 +157,7 @@ compute_resources:
     # OCI spec
     spec:
       state: present
+      wait: no
       compartment_id: "{{ oci_compartment_id }}"
       display_name: "{{ cluster_state.infra_id }}-worker-03"
       region: "{{ config_cluster_region }}"

playbooks/vars/oci/profiles/ha/node-controlplane.yaml

@@ -34,6 +34,7 @@ compute_resources:
     # OCI spec
     spec:
       state: present
+      wait: no
       compartment_id: "{{ oci_compartment_id }}"
       display_name: "{{ cluster_state.infra_id }}-master-01"
       region: "{{ config_cluster_region }}"
@@ -94,6 +95,7 @@ compute_resources:
     # OCI spec
     spec:
       state: present
+      wait: no
       compartment_id: "{{ oci_compartment_id }}"
       display_name: "{{ cluster_state.infra_id }}-master-02"
       region: "{{ config_cluster_region }}"
@@ -154,6 +156,7 @@ compute_resources:
     # OCI spec
     spec:
       state: present
+      wait: yes
       compartment_id: "{{ oci_compartment_id }}"
       display_name: "{{ cluster_state.infra_id }}-master-03"
       region: "{{ config_cluster_region }}"

roles/cloud_compute

@@ -0,0 +1,30 @@
+---
+- name: Patch | Create OCI CCM NS
+  ansible.builtin.template:
+    src: patches/oci-ccm-ns.yaml.j2
+    dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-00-namespace.yaml"
+    mode: 0644
+
+- name: Patch | Create OCI CCM RBAC
+  ansible.builtin.template:
+    src: patches/oci-cloud-controller-manager-rbac.yaml.j2
+    dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-01-rbac.yaml"
+    mode: 0644
+
+- name: Patch | Create OCI CCM Secret
+  ansible.builtin.set_fact:
+    oci_ccm_secret_data: "{{ lookup('template', 'patches/oci-ccm-secret-data.yaml.j2') | from_yaml }}"
+
+- name: Patch | Create OCI CCM Secret
+  ansible.builtin.template:
+    src: patches/oci-ccm-secret.yaml.j2
+    dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-01-secret.yaml"
+    mode: 0644
+  vars:
+    oci_compartment_id: oci_compartment_id
+
+- name: Patch | Create OCI CCM
+  ansible.builtin.template:
+    src: patches/oci-cloud-controller-manager.yaml.j2
+    dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-02.yaml"
+    mode: 0644

roles/cloud_network

@@ -0,0 +1,12 @@
+---
+- name: Config | Patch manifests | Line regex
+  ansible.builtin.lineinfile:
+    path: "{{ config_install_dir }}{{ patch_spec.manifest }}"
+    # search_string: "{{ patch_spec.search_string }}"
+    # line: "{{ patch_spec.line }}"
+    backrefs: true
+    regexp: "{{ patch_spec.regexp }}"
+    line: "{{ patch_spec.line }}"
+  loop: "{{ cfg_patch_line_regex_patch_specs }}"
+  loop_control:
+    loop_var: patch_spec