From 1d7f705af206d8eeff93f5f02fde88effdefc964 Mon Sep 17 00:00:00 2001 From: Marco Braga Date: Tue, 28 Mar 2023 01:15:27 -0300 Subject: [PATCH] creating release 4.13.0-rc.0 installing with ccm+csi --- docs/guides/OCI/index.md | 17 + docs/guides/OCI/oci-image-registry-bucket.md | 12 + docs/guides/OCI/oci-install-ccm.md | 107 +++++ .../OCI/oci-installing-quickly-examples.md | 90 ++++ docs/guides/OCI/oci-installing-steps.md | 285 ++++++++++++ docs/guides/OCI/oci-prerequisites.md | 115 +++++ .../OCI/platform-external-custom-release.md | 166 +++++++ .../{ => OCI}/validate-cluster-with-opct.md | 4 +- docs/guides/installing-agnostic-oci.md | 428 ------------------ examples/create-cluster.yaml | 14 + examples/vars/common.yaml | 8 + examples/vars/oci/common.yaml | 21 + .../oci/ha-platform-external-ccm-csi.yaml | 21 + .../vars/oci/ha-platform-external-ccm.yaml | 20 + examples/vars/oci/ha-platform-external.yaml | 19 + examples/vars/oci/ha-platform-none-csi.yaml | 6 + examples/vars/oci/ha-platform-none.yaml | 5 + roles/clients/tasks/main.yaml | 42 +- roles/cloud_compute | 2 +- roles/config/tasks/create-ignitions.yaml | 5 +- .../patches-manifests/deploy-oci-csi.yaml | 12 +- .../mc-kubelet-providerid.yaml | 4 +- .../patches/mc-iscsid-service.yaml.j2 | 14 + .../oci-csi-05-storage-class-00-bv.yaml.j2 | 12 +- .../oci/oci-sample-csi-00-namespace.yaml.j2 | 12 + .../oci/oci-sample-lb-00-deployment.yaml.j2 | 3 + .../oci/oci-sample-lb-01-service.yaml.j2 | 2 + .../patches/oci/oci-samples-namespace.j2 | 9 +- 28 files changed, 989 insertions(+), 466 deletions(-) create mode 100644 docs/guides/OCI/index.md create mode 100644 docs/guides/OCI/oci-image-registry-bucket.md create mode 100644 docs/guides/OCI/oci-install-ccm.md create mode 100644 docs/guides/OCI/oci-installing-quickly-examples.md create mode 100644 docs/guides/OCI/oci-installing-steps.md create mode 100644 docs/guides/OCI/oci-prerequisites.md create mode 100644 docs/guides/OCI/platform-external-custom-release.md rename docs/guides/{ => OCI}/validate-cluster-with-opct.md (97%) delete mode 100644 docs/guides/installing-agnostic-oci.md create mode 100644 examples/create-cluster.yaml create mode 100644 examples/vars/common.yaml create mode 100644 examples/vars/oci/common.yaml create mode 100644 examples/vars/oci/ha-platform-external-ccm-csi.yaml create mode 100644 examples/vars/oci/ha-platform-external-ccm.yaml create mode 100644 examples/vars/oci/ha-platform-external.yaml create mode 100644 examples/vars/oci/ha-platform-none-csi.yaml create mode 100644 examples/vars/oci/ha-platform-none.yaml create mode 100644 roles/config/templates/patches/mc-iscsid-service.yaml.j2 create mode 100644 roles/config/templates/patches/oci/oci-sample-csi-00-namespace.yaml.j2 diff --git a/docs/guides/OCI/index.md b/docs/guides/OCI/index.md new file mode 100644 index 0000000..36c0dab --- /dev/null +++ b/docs/guides/OCI/index.md @@ -0,0 +1,17 @@ +# Guides for Oracle Cloud Infrastructure + +> WIP + +> TODO: + +Create guides/docs for OCP/OKD on OCI: + +- Installing a cluster with agnostic installation quickly (Platform=None) +- Installing a cluster with Cloud Controller Manager using External provider (Platform=External) +- Installing a cluster with External Cloud provider integration: CCM and `Platform External` +- Installing a cluster with agnostic installation with Assisted Installer as a installation provider + +Generic guides: + +- Integrate new provider to the Ansible Collection (UPI stacks) +- Adding CCM to existing integrated external provider diff --git a/docs/guides/OCI/oci-image-registry-bucket.md b/docs/guides/OCI/oci-image-registry-bucket.md new file mode 100644 index 0000000..6befb8e --- /dev/null +++ b/docs/guides/OCI/oci-image-registry-bucket.md @@ -0,0 +1,12 @@ +# OCI Image Registry - Use S3 compatibility URL for persistent storage + +> WIP + +Steps to use the OCI S3 Compatibility API to set the persistent storage for the OpenShift Image Registry with OCI Bucket service. + +Steps: + +- Create access Key +- Create the secret used by image-registry +- Edit the image registry object adding the s3 configuration +- Test it \ No newline at end of file diff --git a/docs/guides/OCI/oci-install-ccm.md b/docs/guides/OCI/oci-install-ccm.md new file mode 100644 index 0000000..19968f1 --- /dev/null +++ b/docs/guides/OCI/oci-install-ccm.md @@ -0,0 +1,107 @@ +## Install a cluster on OCI with CCM + +## Requirements + +- Credentials +- Client installed + +## OCP Cluster Setup on OCI + +### Generate the vars file + +```bash +cat < ~/.oci/env +# Compartment that the cluster will be installed +OCI_COMPARTMENT_ID="" + +# Compartment that the DNS Zone is created (based domain) +# Only RR will be added +OCI_COMPARTMENT_ID_DNS="" + +# Compartment that the OS Image will be created +OCI_COMPARTMENT_ID_IMAGE="" +EOF +source ~/.oci/env + +cat < ~/.openshift/env +export OCP_CUSTOM_RELEASE="docker.io/mtulio/ocp-release:latest" + +OCP_RELEASE_413="quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" +EOF +source ~/.openshift/env + +CLUSTER_NAME=oci-t13 +VAR_FILE=./vars-oci-ha_${CLUSTER_NAME}.yaml + +cat < ${VAR_FILE} +provider: oci +cluster_name: ${CLUSTER_NAME} +config_cluster_region: us-sanjose-1 + +#TODO: create compartment validations +#TODO: allow create compartment from a parent +oci_compartment_id: ${OCI_COMPARTMENT_ID} +oci_compartment_id_dns: ${OCI_COMPARTMENT_ID_DNS} +oci_compartment_id_image: ${OCI_COMPARTMENT_ID_IMAGE} + +cluster_profile: ha +destroy_bootstrap: no + +config_base_domain: splat-oci.devcluster.openshift.com +config_ssh_key: "$(cat ~/.ssh/id_rsa.pub;cat ~/.ssh/openshift-dev.pub)" +config_pull_secret_file: "${HOME}/.openshift/pull-secret-latest.json" + +config_cluster_version: 4.13.0-rc.0 +version: 4.13.0-rc.0 +config_installer_environment: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "$OCP_RELEASE_413" + +# Define the OS Image mirror +os_mirror: yes +os_mirror_from: stream_artifacts +os_mirror_stream: + architecture: x86_64 + artifact: openstack + format: qcow2.gz + +os_mirror_to_provider: oci +os_mirror_to_oci: + compartment_id: ${OCI_COMPARTMENT_ID_IMAGE} + bucket: rhcos-images + image_type: QCOW2 + +## Apply patches to installer manifests (WIP) +# TODO: we must keep the OCI CCM manifests patch more generic + +config_patches: +- rm-capi-machines +- mc-kubelet-providerid +- deploy-oci-ccm +- deploy-oci-csi +- yaml_patch + +cfg_patch_yaml_patch_specs: + ## patch infra object to create External provider + - manifest: /manifests/cluster-infrastructure-02-config.yml + patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}' + +cfg_patch_kubelet_providerid_script: | + PROVIDERID=\$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id); + +EOF +``` + +### Install the cluster + +```bash +ansible-playbook mtulio.okd_installer.create_all \ + -e certs_max_retries=20 \ + -e cert_wait_interval_sec=60 \ + -e @$VAR_FILE +``` + +## Destroy + +```bash +ansible-playbook mtulio.okd_installer.destroy_cluster -e @$VAR_FILE +``` diff --git a/docs/guides/OCI/oci-installing-quickly-examples.md b/docs/guides/OCI/oci-installing-quickly-examples.md new file mode 100644 index 0000000..3fa764a --- /dev/null +++ b/docs/guides/OCI/oci-installing-quickly-examples.md @@ -0,0 +1,90 @@ +# Installing in OCI with build-in examples + +## Export variables + +```bash +export OKD_CONFIG_BASE_DOMAIN="splat-oci.devcluster.openshift.com" +export OCI_COMPARTMENT_ID="ocid1.compartment.oc1..aaaaaaaaovt4qpeqi7l4cfefzh7tnggbmbj26ccb3dmv23jwle3gsusqpkjq" +export OCI_COMPARTMENT_ID_DNS="ocid1.compartment.oc1..aaaaaaaabfwx2ct2vckgbmvqybfjvz5pblv4ctjcfxmfhdfspyfojyxtaj6q" +export OCI_COMPARTMENT_ID_IMAGE="ocid1.compartment.oc1..aaaaaaaabfwx2ct2vckgbmvqybfjvz5pblv4ctjcfxmfhdfspyfojyxtaj6q" +export OS_MIRROR_IMAGE_BUCKET_NAME="rhcos-images" +``` + +### Default vars + + +## Installing + + +### Installing a cluster on OCI with Platform Agnostic/None + +> TODO + +```bash +ansible-playbook examples/create-cluster.yaml \ + -e cluster_name=name \ + -e @./examples/vars/common.yaml \ + -e @./examples/vars/oci/common.yaml \ + -e @./examples/vars/oci/ha-platform-none.yaml +``` + +### Installing a cluster on OCI with Platform Agnostic/None with CSI Driver + +```bash +ansible-playbook examples/create-cluster.yaml \ + -e cluster_name=name \ + -e @./examples/vars/common.yaml \ + -e @./examples/vars/oci/common.yaml \ + -e @./examples/vars/oci/ha-platform-none-csi.yaml +``` + +### Installing a cluster on OCI with Platform External + +```bash +ansible-playbook examples/create-cluster.yaml \ + -e cluster_name=name \ + -e @./examples/vars/common.yaml \ + -e @./examples/vars/oci/common.yaml \ + -e @./examples/vars/oci/ha-platform-external.yaml +``` + +### Installing a cluster on OCI with Platform External with CCM + +```bash +ansible-playbook examples/create-cluster.yaml \ + -e cluster_name=name \ + -e @./examples/vars/common.yaml \ + -e @./examples/vars/oci/common.yaml \ + -e @./examples/vars/oci/ha-platform-external-ccm.yaml +``` + +### Installing a cluster on OCI with Platform External with CCM and CSI Driver + +```bash +ansible-playbook examples/create-cluster.yaml \ + -e cluster_name=name \ + -e @./examples/vars/common.yaml \ + -e @./examples/vars/oci/common.yaml \ + -e @./examples/vars/oci/ha-platform-external-ccm-csi.yaml +``` + +### Installing a cluster on OCI with Platform External with CSI Driver + +> TODO: OCI CSI Driver can be installed in Platform None with manual changes + + + +### Destroy a cluster + +```bash +ansible-playbook mtulio.okd_installer.destroy_cluster \ + -e cluster_name=name +``` \ No newline at end of file diff --git a/docs/guides/OCI/oci-installing-steps.md b/docs/guides/OCI/oci-installing-steps.md new file mode 100644 index 0000000..991a774 --- /dev/null +++ b/docs/guides/OCI/oci-installing-steps.md @@ -0,0 +1,285 @@ +# Install OKD/OCP on OCI using an agnostic method + +> This document is under development on https://github.com/mtulio/ansible-collection-okd-installer/pull/26 + +Install OCP/OKD Cluster on Oracle Cloud Infrastructure using agnostic installation/UPI. + +- Prerequisites +- Installing OCP + - Install the Clientes + - Option 1 - Install quickly + - Option 2 - Install step-by-stack + - Create the Install config + - Create the manifests + - Setup IAM Stack + - Setup Network Stack + - Setup DNS Stack + - Setup Load Balancer Stack + - Patch the manifests + - Create the ignitions + - Setup Compute Stack + - Setup Bootstrap + - Setup Control Plane nodes + - Setup Compute nodes + - Check/Approve the certificates +- Review the Installation +- Destroy the Cluster + +## Prerequisites + +Read [here](./oci-prerequisites.md) + +## Installing OpenShift/OKD + +### Create the vars file + +```bash +cat < ~/.oci/env +# Compartment where the cluster will be installed +OCI_COMPARTMENT_ID="" + +# Compartment that the DNS Zone is created (based domain) +# Only RR will be added +OCI_COMPARTMENT_ID_DNS="" + +# Compartment that the OS Image will be created +OCI_COMPARTMENT_ID_IMAGE="" +EOF +source ~/.oci/env + +cat < ~/.openshift/env +export OCP_CUSTOM_RELEASE="quay.io/mtulio/ocp-release:latest" + +OCP_RELEASE_413="quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" +EOF +source ~/.openshift/env + +CLUSTER_NAME=oci-t20 +VAR_FILE=./vars-oci-ha_${CLUSTER_NAME}.yaml + +cat < ${VAR_FILE} +provider: oci +cluster_name: ${CLUSTER_NAME} +config_cluster_region: us-sanjose-1 + +#TODO: create compartment validations +#TODO: allow create compartment from a parent +oci_compartment_id: ${OCI_COMPARTMENT_ID} +oci_compartment_id_dns: ${OCI_COMPARTMENT_ID_DNS} +oci_compartment_id_image: ${OCI_COMPARTMENT_ID_IMAGE} + +cluster_profile: ha +destroy_bootstrap: no + +config_base_domain: splat-oci.devcluster.openshift.com +config_ssh_key: "$(cat ~/.ssh/id_rsa.pub;cat ~/.ssh/openshift-dev.pub)" +config_pull_secret_file: "${HOME}/.openshift/pull-secret-latest.json" + +config_cluster_version: 4.13.0-rc.0 +version: 4.13.0-rc.0 +config_installer_environment: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" + +# Define the OS Image mirror +# custom_image_id: rhcos-412.86.202212081411-0-openstack.x86_64 + +os_mirror: yes +os_mirror_from: stream_artifacts +os_mirror_stream: + architecture: x86_64 + artifact: openstack + format: qcow2.gz + +os_mirror_to_provider: oci +os_mirror_to_oci: + compartment_id: ${OCI_COMPARTMENT_ID_IMAGE} + bucket: rhcos-images + image_type: QCOW2 + +## Apply patches to installer manifests (WIP) +# TODO: we must keep the OCI CCM manifests patch more generic + +config_patches: +- rm-capi-machines +- mc-kubelet-providerid +- deploy-oci-ccm +- deploy-oci-csi +- yaml_patch + +cfg_patch_yaml_patch_specs: + ## patch infra object to create External provider + - manifest: /manifests/cluster-infrastructure-02-config.yml + patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}' + +cfg_patch_kubelet_providerid_script: | + PROVIDERID=\$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id); + +EOF + +``` + +### Install the clients + +```bash +ansible-playbook mtulio.okd_installer.install_clients -e @$VAR_FILE +``` + +### Installing option 1: quickly install + +```bash +ansible-playbook mtulio.okd_installer.create_all \ + -e certs_max_retries=20 \ + -e cert_wait_interval_sec=60 \ + -e @$VAR_FILE +``` + +### Installing option 2: step-by-step + +#### Create the Installer Configuration + +Create the installation configuration: + + +```bash +ansible-playbook mtulio.okd_installer.config -e mode=create-config -e @$VAR_FILE +``` + +The rendered install-config.yaml will be available on the following path: + +- `~/.ansible/okd-installer/clusters/$CLUSTER_NAME/install-config.yaml` + +If you want to skip this part, place your own install-config.yaml on the same +path and go to the next step. + +#### Create the Installer manifests + +Create the installation configuration: + +```bash +ansible-playbook mtulio.okd_installer.config -e mode=create-manifests -e @$VAR_FILE +``` + +The manifests will be rendered and saved on the install directory: + +- `~/.ansible/okd-installer/clusters/$CLUSTER_NAME/` + +If you want to skip that part, with your manifests, you must be able to run +the `openshift-install create manifests` under the install directory, and the file +`manifests/cluster-config.yaml` is created correctly. + +The infrastructure manifest also must exist on the path: `manifests/cluster-infrastructure-02-config.yml`. + + +**After this stage, the file `$install_dir/cluster_state.json` will be created and populated with the stack results.** + +#### IAM Stack + +N/A + +> TODO: create Compartment validations + +#### Create the Network Stack + +```bash +ansible-playbook mtulio.okd_installer.stack_network -e @$VAR_FILE +``` + +#### DNS Stack + +```bash +ansible-playbook mtulio.okd_installer.stack_dns -e @$VAR_FILE +``` + +#### Load Balancer Stack + +```bash +ansible-playbook mtulio.okd_installer.stack_loadbalancer -e @$VAR_FILE +``` + +#### Config Commit + +This stage allows the user to modify the cluster configurations (manifests), +then generate the ignition files used to create the cluster. + +##### Manifest patches (pre-ign) + +In this step, the playbooks will apply any patches to the manifests, +according to the vars file `config_patches`. + +The `config_patches` are predefined tasks that will run to reach specific goals. + +If you wouldn't like to apply patches, leave the empty value `config_patches: []`. + +If you would like to apply patches manually, you can do it by changing the manifests +on the install dir. Default install dir path: `~/.ansible/okd-installer/clusters/${cluster_name}/*` + +```bash +ansible-playbook mtulio.okd_installer.config -e mode=patch-manifests -e @$VAR_FILE +``` + +##### Config generation (ignitions) + +These steps should be the last before the configuration be 'committed': + +- `create ignitions` when using `openshift-install` as the config provider + +```bash +ansible-playbook mtulio.okd_installer.config -e mode=create-ignitions -e @$VAR_FILE +``` + +#### Mirror OS boot image + +- Download the image from the URL provided by openshift-install coreos-stream + +> Example: `$ jq -r '.architectures["x86_64"].artifacts.openstack.formats["qcow2.gz"].disk.location' ~/.ansible/okd-installer/clusters/ocp-oci/coreos-stream.json` + +```bash +ansible-playbook mtulio.okd_installer.os_mirror -e @$VAR_FILE +``` + +#### Compute Stack + +##### Bootstrap node + +- Upload the bootstrap ignition to blob and Create the Bootstrap Instance + +```bash +ansible-playbook mtulio.okd_installer.create_node -e node_role=bootstrap -e @$VAR_FILE +``` + +##### Control Plane nodes + +- Create the Control Plane nodes + +```bash +ansible-playbook mtulio.okd_installer.create_node -e node_role=controlplane -e @$VAR_FILE +``` + +##### Compute/worker nodes + +- Create the Compute nodes + +```bash +ansible-playbook mtulio.okd_installer.create_node -e node_role=compute -e @$VAR_FILE +``` + +- Approve worker nodes' certificates signing requests (CSR) + +```bash +oc adm certificate approve $(oc get csr -o json |jq -r '.items[] | select(.status.certificate == null).metadata.name') +``` + +## Review the installation + +```bash +export KUBECONFIG=${HOME}/.ansible/okd-installer/clusters/${cluster_name}/auth/kubeconfig + +oc get nodes +oc get co +``` + +## Destroy cluster + +```bash +ansible-playbook mtulio.okd_installer.destroy_cluster -e @$VAR_FILE +``` diff --git a/docs/guides/OCI/oci-prerequisites.md b/docs/guides/OCI/oci-prerequisites.md new file mode 100644 index 0000000..77e1615 --- /dev/null +++ b/docs/guides/OCI/oci-prerequisites.md @@ -0,0 +1,115 @@ +# OCI PoC - Prerequisites + +The steps described on this document can be changed from the final version. + +The goal is to quickly setup the PoC environment installing all the dependencies and Oracle Cloud Infrastructure identities to use the CLI/SDK with Ansible. + +### Setup Ansible project + +> This steps should be made only when OCI provider is under development - not merged to `main` branch. Then the normal install flow should be used. + +- Setup your ansible workdir (optional, you can use the defaults) + +```bash +cat < ansible.cfg +[defaults] +inventory = ./inventories +collections_path=./collections +callbacks_enabled=ansible.posix.profile_roles,ansible.posix.profile_tasks +hash_behavior=merge + +[inventory] +enable_plugins = yaml, ini + +[callback_profile_tasks] +task_output_limit=1000 +sort_order=none +EOF +``` + +- Create a virtual ennv + +```bash +python3.9 -m venv ./.venv-oci +source ./.venv-oci/bin/activate +``` + +- Donwload requirements files + +``` +wget https://raw.githubusercontent.com/mtulio/ansible-collection-okd-installer/main/requirements.yml +wget https://raw.githubusercontent.com/mtulio/ansible-collection-okd-installer/main/requirements.txt +``` + +- Update with OCI requirements + +```bash +cat <> requirements.txt + +# Oracle Cloud Infrastructure +oci +EOF + +cat <> requirements.yml + +# Oracle Cloud Infrastructure Ansible Collections +# https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.11.0/installation/index.html +- name: oracle.oci + version: '>=4.11.0,<4.12.0' +EOF +``` + +- Install ansible and dependencies + +```bash +pip install -r requirements.txt +``` + +- Install the Collections + +```bash +ansible-galaxy collection install -r requirements.yml +``` + +- Get the latest (under development) okd-installer for OCI + +> https://github.com/mtulio/ansible-collection-okd-installer/pull/26 + +```bash +git clone -b feat-added-provider-oci --recursive \ + git@github.com:mtulio/ansible-collection-okd-installer.git \ + collections/ansible_collections/mtulio/okd_installer +``` + +- Check if the collection is present + + +```bash +$ ansible-galaxy collection list |egrep "(okd_installer|^oracle)" +mtulio.okd_installer 0.0.0-latest +oracle.oci 4.11.0 +``` + +### Setup OCI credentials + +- See [API Key Authentication](https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.11.0/guides/authentication.html#api-key-authentication): +- See https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm#two + + +Make sure your credentials have been set correctly on the file `~/.oci/config` and you can use the OCI ansible collection: + +- Get the User ID from the documentation + +```bash +export oci_user_id=$(grep ^user ~/.oci/config | awk -F '=' '{print$2}') +``` + +- Retrieve facts from the user + +```bash +ansible localhost \ + -m oracle.oci.oci_identity_user_facts \ + -a user_id=${oci_user_id} +``` + +You must be able to collect the user information. \ No newline at end of file diff --git a/docs/guides/OCI/platform-external-custom-release.md b/docs/guides/OCI/platform-external-custom-release.md new file mode 100644 index 0000000..7e26e1e --- /dev/null +++ b/docs/guides/OCI/platform-external-custom-release.md @@ -0,0 +1,166 @@ +# Platform External - creating a custom release to support it on 4.13 + +This guide describes how to create a custom OCP release image with minimal changes to enable Platform `External` to be considered 'external' on the `library-go` - `IsCloudProviderExternal()`, signalizing the Kubelet (MCO) and Kube Controller Manager (KCMO) flag `--cloud-provider` be external, waiting for an external CCM be deployed on install time (in this case [OCI CCM](https://github.com/oracle/oci-cloud-controller-manager)) + +This is part of a PoC to enable Platform External to install CCM on install time. All the work has been mapped on the [Enhancement Proposal 1353](https://github.com/openshift/enhancements/pull/1353). + +## Update the API + +### API + +> The minimal changes on API have been created on 4.13. It's not required for this PoC. + +References: + +- https://github.com/openshift/api/pull/1301 +- https://github.com/openshift/api/pull/1409 + +### library-go + +- Clone the Library-go + +- Make the changes: https://github.com/openshift/library-go/compare/release-4.13...mtulio:library-go:release-4.13-platexternal?expand=1#diff-478af36e9fb994fc80d37b7d2f6ae207c67d8c43b94f98f6ae3e420808958ba9R40-R41 + +- Push to your account + + +## Rebuilding KCMO + +Steps to propagate the library-go change to kube-controller-manager-operator. + +- Clone the repo https://github.com/openshift/cluster-kube-controller-manager-operator + +- Update the go.mod to use your version of library-go https://github.com/openshift/cluster-kube-controller-manager-operator/compare/release-4.13...mtulio:cluster-kube-controller-manager-operator:release-4.13-platexternal?expand=1 + +`go.mod` +``` +replace github.com/openshift/library-go => github.com/mtulio/library-go v0.0.0-20230313023417-78e409222bff +``` + +- upload your custom changes (optional) + +```bash +$ git remote -v +mtulio git@github.com:mtulio/cluster-kube-controller-manager-operator.git (fetch) +mtulio git@github.com:mtulio/cluster-kube-controller-manager-operator.git (push) +origin git@github.com:openshift/cluster-kube-controller-manager-operator.git (fetch) +$ git push --set-upstream mtulio release-4.13-platexternal -f +``` + +- Build a custom image + + +```bash +QUAY_USER=mrbraga +REPO_NAME=cluster-kube-controller-manager-operator + +podman build \ + --authfile ${PULL_SECRET} \ + -f Dockerfile.rhel7 \ + -t quay.io/${QUAY_USER}/${REPO_NAME}:latest \ + && podman push quay.io/${QUAY_USER}/${REPO_NAME}:latest + +TS=$(date +%Y%m%d%H%M) +podman tag quay.io/${QUAY_USER}/${REPO_NAME}:latest \ + "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" && \ + podman push "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" +``` + +## Building MCO + +Steps to propagate the library-go change to machine-config-operator. + +- Clone the repo https://github.com/openshift/machine-config-operator + +- Update the go.mod to use your version of library-go + +`go.mod` +``` +replace github.com/openshift/library-go => github.com/mtulio/library-go v0.0.0-20230313023417-78e409222bff +``` + +- Build a custom image + +```shell +QUAY_USER=mrbraga +REPO_NAME=machine-config-operator + +podman build -f Dockerfile.rhel7 \ + -t quay.io/${QUAY_USER}/${REPO_NAME}:latest && \ + podman push quay.io/${QUAY_USER}/${REPO_NAME}:latest + +TS=$(date +%Y%m%d%H%M) +podman tag quay.io/${QUAY_USER}/${REPO_NAME}:latest \ + "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" && \ + podman push "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" +``` + +## Building CCCMO + +Steps to propagate the library-go change to cluster-cloud-controller-manager-operator. + +- Clone the repo https://github.com/mtulio/cluster-cloud-controller-manager-operator + +- Update the go.mod to use your version of library-go + +- Build a custom image + +```bash +QUAY_USER=mrbraga +REPO_NAME=cluster-cloud-controller-manager-operator + +podman build \ + --authfile ${PULL_SECRET} \ + -f Dockerfile \ + -t quay.io/${QUAY_USER}/${REPO_NAME}:latest \ + && podman push quay.io/${QUAY_USER}/${REPO_NAME}:latest + +TS=$(date +%Y%m%d%H%M) +podman tag quay.io/${QUAY_USER}/${REPO_NAME}:latest \ + "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" && \ + podman push "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" +``` + +## Create a new release + +- Choose the base image on https://openshift-release.apps.ci.l2s4.p1.openshiftapps.com/ + +- Run the command + +```bash +VERSION_BASE="4.13.0-rc.0-x86_64" +OCP_RELEASE_BASE="quay.io/openshift-release-dev/ocp-release:${VERSION_BASE}" +CUSTOM_IMAGE_NAMESPACE="quay.io/${QUAY_USER}" +NEW_RELEASE_IMAGE="docker.io/mtulio/ocp-release" + +$(which time) -v oc adm release new -n origin \ + --server https://api.ci.openshift.org \ + -a ${PULL_SECRET} \ + --from-release ${OCP_RELEASE_BASE} \ + --to-image "${NEW_RELEASE_IMAGE}:latest" \ + machine-config-operator=${CUSTOM_IMAGE_NAMESPACE}/machine-config-operator:latest \ + cluster-kube-controller-manager-operator=${CUSTOM_IMAGE_NAMESPACE}/cluster-kube-controller-manager-operator:latest \ + cluster-cloud-controller-manager-operator=${CUSTOM_IMAGE_NAMESPACE}/cluster-cloud-controller-manager-operator:latest +``` + +- Mirror it creating custom labels to identify the customization and base image + +```bash +podman pull "${NEW_RELEASE_IMAGE}:latest" + +podman tag "${NEW_RELEASE_IMAGE}:latest" \ + "${CUSTOM_IMAGE_NAMESPACE}/ocp-release:latest" && \ + podman push "${CUSTOM_IMAGE_NAMESPACE}/ocp-release:latest" +podman tag "${NEW_RELEASE_IMAGE}:latest" \ + "${CUSTOM_IMAGE_NAMESPACE}/ocp-release:${VERSION_BASE}_platexternal-kcmo-mco-3cmo" && \ + podman push "${CUSTOM_IMAGE_NAMESPACE}/ocp-release:${VERSION_BASE}_platexternal-kcmo-mco-3cmo" +``` + +- Check if the release image `${NEW_RELEASE_IMAGE}:latest` was created + +- Use it + +```bash +OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" \ + openshift-install create cluster --dir my-install-dir/ +``` \ No newline at end of file diff --git a/docs/guides/validate-cluster-with-opct.md b/docs/guides/OCI/validate-cluster-with-opct.md similarity index 97% rename from docs/guides/validate-cluster-with-opct.md rename to docs/guides/OCI/validate-cluster-with-opct.md index 5f58e8d..4466f9a 100644 --- a/docs/guides/validate-cluster-with-opct.md +++ b/docs/guides/OCI/validate-cluster-with-opct.md @@ -16,11 +16,11 @@ ansible-playbook mtulio.okd_installer.create_node \ ```bash -# Set the OPCT requirements (registry, labels, wait-for COs stable) oc label node opct-01.priv.ocp.oraclevcn.com node-role.kubernetes.io/tests="" oc adm taint node opct-01.priv.ocp.oraclevcn.com node-role.kubernetes.io/tests="":NoSchedule -ansible-playbook ../opct/hack/opct-runner/opct-run-tool-preflight.yaml -e cluster_name=oci-t11 -D +# Set the OPCT requirements (registry, labels, wait-for COs stable) +ansible-playbook ../opct/hack/opct-runner/opct-run-tool-preflight.yaml -e @$VAR_FILE -D ``` diff --git a/docs/guides/installing-agnostic-oci.md b/docs/guides/installing-agnostic-oci.md deleted file mode 100644 index bcec6ae..0000000 --- a/docs/guides/installing-agnostic-oci.md +++ /dev/null @@ -1,428 +0,0 @@ -# Install OKD/OCP on OCI using agnostic method - -> This document is under development on https://github.com/mtulio/ansible-collection-okd-installer/pull/26 - -Install OCP/OKD Cluster on Oracle Cloud Infrastructure using agnostic installation/UPI. - -ToC - -- Prerequisites - - Setup Ansible Project - - Setup OCI Credentials -- OCP/OKD Cluster setup on OCI - - Install the Clientes - - Setup the installer artifacts - - Setup IAM Stack - - Setup Network Stack - - Setup DNS Stack - - Setup Load Balancer Stack - - Setup Compute Stack - - Setup Bootstrap - - Setup Control Plane - - Setup Compute Pool -- Review the Installation -- Destroy the Clueter - -## Prerequisites - -### Setup Ansible project - -> This steps should be made only when OCI provider is under development - not merged to `main` branch. Then the normal install flow should be used. - -- Setup your ansible workdir (optional, you can use the defaults) - -```bash -cat < ansible.cfg -[defaults] -inventory = ./inventories -collections_path=./collections -callbacks_enabled=ansible.posix.profile_roles,ansible.posix.profile_tasks -hash_behavior=merge - -[inventory] -enable_plugins = yaml, ini - -[callback_profile_tasks] -task_output_limit=1000 -sort_order=none -EOF -``` - -- Create a virtual ennv - -```bash -python3.9 -m venv ./.venv-oci -source ./.venv-oci/bin/activate -``` - -- Donwload requirements files - -``` -wget https://raw.githubusercontent.com/mtulio/ansible-collection-okd-installer/main/requirements.yml -wget https://raw.githubusercontent.com/mtulio/ansible-collection-okd-installer/main/requirements.txt -``` - -- Update with OCI requirements - -```bash -cat <> requirements.txt - -# Oracle Cloud Infrastructure -oci -EOF - -cat <> requirements.yml - -# Oracle Cloud Infrastructure Ansible Collections -# https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.11.0/installation/index.html -- name: oracle.oci - version: '>=4.11.0,<4.12.0' -EOF -``` - -- Install ansible and dependencies - -```bash -pip install -r requirements.txt -``` - -- Install the Collections - -```bash -ansible-galaxy collection install -r requirements.yml -``` - -- Get the latest (under development) okd-installer for OCI - -> https://github.com/mtulio/ansible-collection-okd-installer/pull/26 - -```bash -git clone -b feat-added-provider-oci --recursive \ - git@github.com:mtulio/ansible-collection-okd-installer.git \ - collections/ansible_collections/mtulio/okd_installer -``` - -- Check if the collection is present - - -```bash -$ ansible-galaxy collection list |egrep "(okd_installer|^oracle)" -mtulio.okd_installer 0.0.0-latest -oracle.oci 4.11.0 -``` - -### Setup OCI credentials - -- See [API Key Authentication](https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.11.0/guides/authentication.html#api-key-authentication): -- See https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm#two - - -Make sure your credentials have been set correctly on the file `~/.oci/config` and you can use the OCI ansible collection: - -- Get the User ID from the documentation - -```bash -export oci_user_id=$(grep ^user ~/.oci/config | awk -F '=' '{print$2}') -``` - -- Retrieve facts from the user - -```bash -ansible localhost \ - -m oracle.oci.oci_identity_user_facts \ - -a user_id=${oci_user_id} -``` - -You must be able to collect the user information. - -## OCP Cluster Setup on OCI - -### Generate the vars file - -```bash -cat < ~/.oci/env -# Compartment that the cluster will be installed -OCI_COMPARTMENT_ID="" - -# Compartment that the DNS Zone is created (based domain) -# Only RR will be added -OCI_COMPARTMENT_ID_DNS="" - -# Compartment that the OS Image will be created -OCI_COMPARTMENT_ID_IMAGE="" -EOF -source ~/.oci/env - -cat < ~/.openshift/env -export OCP_CUSTOM_RELEASE="docker.io/mtulio/ocp-release:latest" - -OCP_RELEASE_413="quay.io/openshift-release-dev/ocp-release:4.13.0-ec.4-x86_64" -EOF -source ~/.openshift/env - -CLUSTER_NAME=oci-t13 -VAR_FILE=./vars-oci-ha_${CLUSTER_NAME}.yaml - -cat < ${VAR_FILE} -provider: oci -cluster_name: ${CLUSTER_NAME} -config_cluster_region: us-sanjose-1 - -#TODO: create compartment validations -#TODO: allow create compartment from a parent -oci_compartment_id: ${OCI_COMPARTMENT_ID} -oci_compartment_id_dns: ${OCI_COMPARTMENT_ID_DNS} -oci_compartment_id_image: ${OCI_COMPARTMENT_ID_IMAGE} - -cluster_profile: ha -destroy_bootstrap: no - -config_base_domain: splat-oci.devcluster.openshift.com -config_ssh_key: "$(cat ~/.ssh/id_rsa.pub;cat ~/.ssh/openshift-dev.pub)" -config_pull_secret_file: "${HOME}/.openshift/pull-secret-latest.json" - -#config_cluster_version: 4.13.0-ec.3-x86_64 -version: 4.13.0-ec.3 -config_installer_environment: - OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "${OCP_CUSTOM_RELEASE}" - -controlplane_instance: VM.Standard3.Flex -controlplane_instance_spec: - cpu_count: 8 - memory_gb: 16 - -compute_instance: VM.Standard3.Flex -compute_instance_spec: - cpu_count: 8 - memory_gb: 16 - -# Define the OS Image mirror -# custom_image_id: rhcos-412.86.202212081411-0-openstack.x86_64 - -os_mirror: yes -os_mirror_from: stream_artifacts -os_mirror_stream: - architecture: x86_64 - artifact: openstack - format: qcow2.gz - # TO test: - #artifact: aws - #format: vmdk.gz - -os_mirror_to_provider: oci -os_mirror_to_oci: - compartment_id: ${OCI_COMPARTMENT_ID_IMAGE} - bucket: rhcos-images - image_type: QCOW2 - #image_type: VMDK - - -## Apply patches to installer manifests (WIP) - -# TODO: we must keep the OCI CCM manifests patch more generic - -config_patches: -- rm-capi-machines -#- mc-kubelet-env-workaround # PROBLEM hangin kubelet (network) -- mc-kubelet-providerid -#- platform-external-kcmo -- deploy-oci-ccm -#- deploy-oci-csi -- yaml_patch # working for OCI, but need to know the path -#- line_regex_patch # ideal, but not working as expected - -cfg_patch_yaml_patch_specs: - ## patch infra object to create External provider - - manifest: /manifests/cluster-infrastructure-02-config.yml - patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}' - -cfg_patch_line_regex_patch_specs: - - manifest: /manifests/oci-cloud-controller-manager-01-rbac.yaml - #search_string: 'namespace: kube-system' - regexp: '^(.*)(namespace\\: kube-system)$' - #line: 'namespace: oci-cloud-controller-manager' - line: '\\1namespace: oci-cloud-controller-manager' - - - manifest: /manifests/oci-cloud-controller-manager-02.yaml - regexp: '^(.*)(namespace\\: kube-system)$' - line: '\\1namespace: oci-cloud-controller-manager' - -cfg_patch_kubelet_providerid_script: | - PROVIDERID=\$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id); - -EOF - -``` - -### Install the clients - -```bash -ansible-playbook mtulio.okd_installer.install_clients -e @$VAR_FILE -``` - -### Create the Installer Configuration - -Create the installation configuration: - - -```bash -ansible-playbook mtulio.okd_installer.config -e mode=create-config -e @$VAR_FILE -``` - -The rendered install-config.yaml will be available on the following path: - -- `~/.ansible/okd-installer/clusters/$CLUSTER_NAME/install-config.yaml` - -If you want to skip this part, place your own install-config.yaml on the same -path and go to the next step. - -### Create the Installer manifests - -Create the installation configuration: - -```bash -ansible-playbook mtulio.okd_installer.config -e mode=create-manifests -e @$VAR_FILE -``` - -The manifests will be rendered and saved on the install directory: - -- `~/.ansible/okd-installer/clusters/$CLUSTER_NAME/` - -If you want to skip that part, with your own manifests, you must be able to run -the `openshift-install create manifests` under the install dir, and the file -`manifests/cluster-config.yaml` is created correctly. - -The infrastructure manifest also must exist on path: `manifests/cluster-infrastructure-02-config.yml`. - - -**After this stage, the file `$install_dir/cluster_state.json` will be created and populated with the stack results.** - -### IAM Stack - -N/A - -> TODO: create Compartment validations - -### Create the Network Stack - -```bash -ansible-playbook mtulio.okd_installer.stack_network -e @$VAR_FILE -``` - -### DNS Stack - -```bash -ansible-playbook mtulio.okd_installer.stack_dns -e @$VAR_FILE -``` - -### Load Balancer Stack - -```bash -ansible-playbook mtulio.okd_installer.stack_loadbalancer -e @$VAR_FILE -``` - -### Config Commit - -This stage allows the user to modify the cluster configurations (manifests), -then generate the ignition files used to create the cluster. - -#### Manifest patches (pre-ign) - -In this step the playbooks will apply any patchs to the manifests, -according to the vars file `config_patches`. - -The `config_patches` are predefined tasks that will run to reach specific goals. - -If you wouldn't like to apply patches, leave the empty value `config_patches: []`. - -If you would like to apply patches manually, you can do it changing the manifests -on the install dir. Default install dir path: `~/.ansible/okd-installer/clusters/${cluster_name}/*` - -```bash -ansible-playbook mtulio.okd_installer.config -e mode=patch-manifests -e @$VAR_FILE -``` - -#### Config generation (ignitions) - -> TODO/WIP - -This steps should be the last before the configuration be 'commited': - -- `create ignitions` when using `openshift-install` as config provider -- `` when using `assisted installer` as a config provider - -```bash -ansible-playbook mtulio.okd_installer.config -e mode=create-ignitions -e @$VAR_FILE -``` - - - -### Mirror OS boot image - -- Download image from URL provided by openshift-install coreos-stream - -> Example: `$ jq -r '.architectures["x86_64"].artifacts.openstack.formats["qcow2.gz"].disk.location' ~/.ansible/okd-installer/clusters/ocp-oci/coreos-stream.json` - -```bash -ansible-playbook mtulio.okd_installer.os_mirror -e @$VAR_FILE -``` - -### Compute Stack - -#### Bootstrap node - -- Upload the bootstrap ignition to blob and Create the Bootstrap Instance - -```bash -ansible-playbook mtulio.okd_installer.create_node -e node_role=bootstrap -e @$VAR_FILE -``` - -#### Control Plane nodes - -- Create the Control Plane nodes - -```bash -ansible-playbook mtulio.okd_installer.create_node -e node_role=controlplane -e @$VAR_FILE -``` - -#### Compute/worker nodes - -- Create the Compute nodes - -```bash -ansible-playbook mtulio.okd_installer.create_node -e node_role=compute -e @$VAR_FILE -``` - -> TODO: create instance Pool - -- Approve worker nodes certificates signing requests (CSR) - -```bash -oc adm certificate approve $(oc get csr -o json |jq -r '.items[] | select(.status.certificate == null).metadata.name') -``` - -### Create all - -```bash -ansible-playbook mtulio.okd_installer.create_all \ - -e certs_max_retries=20 \ - -e cert_wait_interval_sec=60 \ - -e @$VAR_FILE -``` - -## Review the cluster - -```bash -export KUBECONFIG=${HOME}/.ansible/okd-installer/clusters/${cluster_name}/auth/kubeconfig - -oc get nodes -oc get co -``` - -## Destroy - -```bash -ansible-playbook mtulio.okd_installer.destroy_cluster -e @$VAR_FILE -``` diff --git a/examples/create-cluster.yaml b/examples/create-cluster.yaml new file mode 100644 index 0000000..12d5b62 --- /dev/null +++ b/examples/create-cluster.yaml @@ -0,0 +1,14 @@ +--- +# Usage examples +# +# OCI Platform None/Agnostic: +# $ ansible-playbook examples/create-cluster.yaml -e cluster_name +# -e @./examples/vars/common.yaml -e @./examples/vars/oci/common.yaml +# -e @./examples/vars/oci/ha-platform-none.yaml +# + +- name: install clients + ansible.builtin.import_playbook: mtulio.okd_installer.install_clients.yaml + +- name: create cluster + ansible.builtin.import_playbook: mtulio.okd_installer.create_all.yaml \ No newline at end of file diff --git a/examples/vars/common.yaml b/examples/vars/common.yaml new file mode 100644 index 0000000..333ef5a --- /dev/null +++ b/examples/vars/common.yaml @@ -0,0 +1,8 @@ +--- +config_base_domain: "{{ ansible_env['OKD_CONFIG_BASE_DOMAIN'] }}" +config_ssh_key: "{{ lookup('file', ansible_env['HOME'] + '/.ssh/id_rsa.pub') }}" +config_pull_secret_file: "{{ ansible_env['HOME'] }}/.openshift/pull-secret-latest.json" + +version: 4.12.8 + +destroy_bootstrap: no \ No newline at end of file diff --git a/examples/vars/oci/common.yaml b/examples/vars/oci/common.yaml new file mode 100644 index 0000000..eb951b4 --- /dev/null +++ b/examples/vars/oci/common.yaml @@ -0,0 +1,21 @@ +--- + +provider: oci +config_cluster_region: us-sanjose-1 + +oci_compartment_id: "{{ ansible_env['OCI_COMPARTMENT_ID'] }}" +oci_compartment_id_dns: "{{ ansible_env['OCI_COMPARTMENT_ID_DNS'] }}" +oci_compartment_id_image: "{{ ansible_env['OCI_COMPARTMENT_ID_IMAGE'] }}" + +os_mirror: yes +os_mirror_from: stream_artifacts +os_mirror_stream: + architecture: x86_64 + artifact: openstack + format: qcow2.gz + +os_mirror_to_provider: oci +os_mirror_to_oci: + compartment_id: "{{ oci_compartment_id_image }}" + bucket: "{{ oci_image_bucket | d(ansible_env['OS_MIRROR_IMAGE_BUCKET_NAME']) }}" + image_type: QCOW2 \ No newline at end of file diff --git a/examples/vars/oci/ha-platform-external-ccm-csi.yaml b/examples/vars/oci/ha-platform-external-ccm-csi.yaml new file mode 100644 index 0000000..ab48cc5 --- /dev/null +++ b/examples/vars/oci/ha-platform-external-ccm-csi.yaml @@ -0,0 +1,21 @@ +--- +cluster_profile: ha + +version: 4.13.0-rc.0 +config_installer_environment: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" + +config_patches: +- rm-capi-machines +- mc-kubelet-providerid +- deploy-oci-ccm +- deploy-oci-csi +- yaml_patch + +cfg_patch_yaml_patch_specs: + ## patch infra object to create External provider + - manifest: /manifests/cluster-infrastructure-02-config.yml + patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}' + +cfg_patch_kubelet_providerid_script: | + PROVIDERID=\$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id); \ No newline at end of file diff --git a/examples/vars/oci/ha-platform-external-ccm.yaml b/examples/vars/oci/ha-platform-external-ccm.yaml new file mode 100644 index 0000000..fa1d3f6 --- /dev/null +++ b/examples/vars/oci/ha-platform-external-ccm.yaml @@ -0,0 +1,20 @@ +--- +cluster_profile: ha + +version: 4.13.0-rc.0 +config_installer_environment: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" + +config_patches: +- rm-capi-machines +- mc-kubelet-providerid +- deploy-oci-ccm +- yaml_patch + +cfg_patch_yaml_patch_specs: + ## patch infra object to create External provider + - manifest: /manifests/cluster-infrastructure-02-config.yml + patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}' + +cfg_patch_kubelet_providerid_script: | + PROVIDERID=\$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id); \ No newline at end of file diff --git a/examples/vars/oci/ha-platform-external.yaml b/examples/vars/oci/ha-platform-external.yaml new file mode 100644 index 0000000..bba75b0 --- /dev/null +++ b/examples/vars/oci/ha-platform-external.yaml @@ -0,0 +1,19 @@ +--- +cluster_profile: ha + +version: 4.13.0-rc.0 +config_installer_environment: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" + +config_patches: +- rm-capi-machines +- mc-kubelet-providerid +- yaml_patch + +cfg_patch_yaml_patch_specs: + ## patch infra object to create External provider + - manifest: /manifests/cluster-infrastructure-02-config.yml + patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}' + +cfg_patch_kubelet_providerid_script: | + PROVIDERID=\$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id); \ No newline at end of file diff --git a/examples/vars/oci/ha-platform-none-csi.yaml b/examples/vars/oci/ha-platform-none-csi.yaml new file mode 100644 index 0000000..a21f7d6 --- /dev/null +++ b/examples/vars/oci/ha-platform-none-csi.yaml @@ -0,0 +1,6 @@ +--- +cluster_profile: ha + +config_patches: +- rm-capi-machines +- deploy-oci-csi \ No newline at end of file diff --git a/examples/vars/oci/ha-platform-none.yaml b/examples/vars/oci/ha-platform-none.yaml new file mode 100644 index 0000000..f45d765 --- /dev/null +++ b/examples/vars/oci/ha-platform-none.yaml @@ -0,0 +1,5 @@ +--- +cluster_profile: ha + +config_patches: +- rm-capi-machines \ No newline at end of file diff --git a/roles/clients/tasks/main.yaml b/roles/clients/tasks/main.yaml index 1efdeac..232fc2f 100644 --- a/roles/clients/tasks/main.yaml +++ b/roles/clients/tasks/main.yaml @@ -14,17 +14,17 @@ - name: Mount the binary names ansible.builtin.set_fact: - bin_installer: "openshift-install-{{ client_os }}-{{ version | d(release_version) }}" bin_clients: "openshift-client-{{ client_os }}-{{ version | d(release_version) }}" - bin_oc: "oc-{{ client_os }}-{{ version | d(release_version) }}" - bin_kubectl: "kubectl-{{ client_os }}-{{ version | d(release_version) }}" - bin_butane: "{{ cli_butane_bin }}-{{ cli_butane_version }}" + _bin_installer: "openshift-install-{{ client_os }}-{{ version | d(release_version) }}" + _bin_oc: "oc-{{ client_os }}-{{ version | d(release_version) }}" + _bin_kubectl: "kubectl-{{ client_os }}-{{ version | d(release_version) }}" + _bin_butane: "{{ cli_butane_bin }}-{{ cli_butane_version }}" # Client: openshift-install - name: openshift-install | Check is present ansible.builtin.stat: - path: "{{ collection_bin_dir }}/{{ bin_installer }}" + path: "{{ collection_bin_dir }}/{{ _bin_installer }}" register: check_bin_inst - name: openshift-install | Install client @@ -32,7 +32,7 @@ block: - name: openshift-install | Check tarball ansible.builtin.stat: - path: "{{ collection_tmp_dir }}/{{ bin_installer }}.tar.gz" + path: "{{ collection_tmp_dir }}/{{ _bin_installer }}.tar.gz" register: check_tar_inst - name: openshift-install | Download msg @@ -71,7 +71,7 @@ - name: openshift-install | Extract tarball ansible.builtin.unarchive: - src: "{{ collection_tmp_dir }}/{{ bin_installer }}.tar.gz" + src: "{{ collection_tmp_dir }}/{{ _bin_installer }}.tar.gz" dest: "{{ collection_tmp_dir }}" when: not(check_bin_inst.stat.exists) register: ext_bin_inst @@ -79,7 +79,7 @@ - name: openshift-install | Copy to bin dir ansible.builtin.copy: src: "{{ collection_tmp_dir }}/openshift-install" - dest: "{{ collection_bin_dir }}/{{ bin_installer }}" + dest: "{{ collection_bin_dir }}/{{ _bin_installer }}" mode: 0755 when: ext_bin_inst.changed @@ -93,7 +93,7 @@ - name: oc | Check openshift client is present ansible.builtin.stat: - path: "{{ collection_bin_dir }}/{{ bin_oc }}" + path: "{{ collection_bin_dir }}/{{ _bin_oc }}" register: check_bin_cli - name: oc | Install clients @@ -109,7 +109,7 @@ - name: oc | Copy to bin path ansible.builtin.copy: src: "{{ collection_tmp_dir }}/oc" - dest: "{{ collection_bin_dir }}/{{ bin_oc }}" + dest: "{{ collection_bin_dir }}/{{ _bin_oc }}" mode: 0755 when: ext_bin_cli.changed @@ -122,7 +122,7 @@ - name: kubectl | Copy to bin path ansible.builtin.copy: src: "{{ collection_tmp_dir }}/kubectl" - dest: "{{ collection_bin_dir }}/{{ bin_kubectl }}" + dest: "{{ collection_bin_dir }}/{{ _bin_kubectl }}" mode: 0755 when: ext_bin_cli.changed @@ -136,7 +136,7 @@ - name: butane | Check butane client is present ansible.builtin.stat: - path: "{{ collection_bin_dir }}/{{ bin_butane }}" + path: "{{ collection_bin_dir }}/{{ _bin_butane }}" register: check_bin_butane - name: butane | Install clients @@ -152,7 +152,7 @@ - name: butane | Copy to bin path ansible.builtin.copy: src: "{{ collection_tmp_dir }}/butane" - dest: "{{ collection_bin_dir }}/{{ bin_butane }}" + dest: "{{ collection_bin_dir }}/{{ _bin_butane }}" mode: 0755 when: ext_bin_butane.changed @@ -170,13 +170,13 @@ src: "{{ collection_bin_dir }}/{{ item.src }}" dest: "{{ collection_bin_dir }}/{{ item.link }}" with_items: - - src: "{{ bin_installer }}" + - src: "{{ _bin_installer }}" link: openshift-install - - src: "{{ bin_oc }}" + - src: "{{ _bin_oc }}" link: oc - - src: "{{ bin_kubectl }}" + - src: "{{ _bin_kubectl }}" link: kubectl - - src: "{{ bin_butane }}" + - src: "{{ _bin_butane }}" link: butane - name: Show clients path @@ -185,11 +185,11 @@ - "binary=[{{ collection_bin_dir }}/{{ item.src }}]" - "link=[{{ collection_bin_dir }}/{{ item.link }}]" with_items: - - src: "{{ bin_installer }}" + - src: "{{ _bin_installer }}" link: openshift-install - - src: "{{ bin_oc }}" + - src: "{{ _bin_oc }}" link: oc - - src: "{{ bin_kubectl }}" + - src: "{{ _bin_kubectl }}" link: kubectl - - src: "{{ bin_butane }}" + - src: "{{ _bin_butane }}" link: butane diff --git a/roles/cloud_compute b/roles/cloud_compute index 839e555..2c60e24 160000 --- a/roles/cloud_compute +++ b/roles/cloud_compute @@ -1 +1 @@ -Subproject commit 839e5557ea006cda0f6f264b268fa007b53c1131 +Subproject commit 2c60e240bd23a3f5612c08a02dd4edaae1fba002 diff --git a/roles/config/tasks/create-ignitions.yaml b/roles/config/tasks/create-ignitions.yaml index 2bfdcb9..348e5b9 100644 --- a/roles/config/tasks/create-ignitions.yaml +++ b/roles/config/tasks/create-ignitions.yaml @@ -14,7 +14,4 @@ - name: Create | Create ignition configs ansible.builtin.shell: | {{ bin_openshift_install }} create ignition-configs --dir {{ config_install_dir }} - environment: "{{ config_installer_environment | d(omit) }}" - -# Set data from ignitions -#: TODO + environment: "{{ config_installer_environment | d(omit) }}" \ No newline at end of file diff --git a/roles/config/tasks/patches-manifests/deploy-oci-csi.yaml b/roles/config/tasks/patches-manifests/deploy-oci-csi.yaml index c734c71..6ae1a8e 100644 --- a/roles/config/tasks/patches-manifests/deploy-oci-csi.yaml +++ b/roles/config/tasks/patches-manifests/deploy-oci-csi.yaml @@ -9,7 +9,7 @@ ansible.builtin.set_fact: oci_ccm_secret_data: "{{ lookup('template', 'patches/oci/oci-ccm-01-secret-data.yaml.j2') | from_yaml }}" -- name: Patch | OCI | CSI | Create Manifests {{ manifest }} +- name: Patch | OCI | CSI | Create Manifests to install dir manifests/ ansible.builtin.template: src: "patches/oci/{{ manifest }}.j2" dest: "{{ config_install_dir }}/manifests/{{ manifest }}" @@ -30,3 +30,13 @@ - oci-csi-04-node-driver-04-daemonset.yaml - oci-csi-05-storage-class-00-bv.yaml - oci-csi-05-storage-class-01-bv-enc.yaml + +- name: Patch | OCI | CSI | Create MachineConfig iscsid.service + ansible.builtin.template: + src: patches/mc-iscsid-service.yaml.j2 + dest: "{{ config_install_dir }}/openshift//99_openshift-machineconfig_99-{{ machine_role }}-iscsid.yaml" + loop_control: + loop_var: machine_role + loop: + - master + - worker \ No newline at end of file diff --git a/roles/config/tasks/patches-manifests/mc-kubelet-providerid.yaml b/roles/config/tasks/patches-manifests/mc-kubelet-providerid.yaml index f2fc7be..1b13826 100644 --- a/roles/config/tasks/patches-manifests/mc-kubelet-providerid.yaml +++ b/roles/config/tasks/patches-manifests/mc-kubelet-providerid.yaml @@ -17,7 +17,7 @@ dest: "{{ cluster_tmp_dir }}" state: directory -- name: Patch | Crete kubelet providerID +- name: Patch | mc-kubelet-providerid | Create kubelet providerID ansible.builtin.template: src: patches/mc-kubelet-providerid.bu.j2 dest: "{{ cluster_tmp_dir }}/99_openshift-machineconfig_00-{{ machine_role }}-kubelet-providerid.bu" @@ -27,7 +27,7 @@ - master - worker -- name: Patch | Render butane config +- name: Patch | mc-kubelet-providerid | Render butane config ansible.builtin.shell: | {{ bin_butane }} \ {{ cluster_tmp_dir }}/99_openshift-machineconfig_00-{{ machine_role }}-kubelet-providerid.bu \ diff --git a/roles/config/templates/patches/mc-iscsid-service.yaml.j2 b/roles/config/templates/patches/mc-iscsid-service.yaml.j2 new file mode 100644 index 0000000..f454bf6 --- /dev/null +++ b/roles/config/templates/patches/mc-iscsid-service.yaml.j2 @@ -0,0 +1,14 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: {{ machine_role }} + name: 99-{{ machine_role }}-iscsid +spec: + config: + ignition: + version: 3.1.0 + systemd: + units: + - enabled: true + name: iscsid.service \ No newline at end of file diff --git a/roles/config/templates/patches/oci/oci-csi-05-storage-class-00-bv.yaml.j2 b/roles/config/templates/patches/oci/oci-csi-05-storage-class-00-bv.yaml.j2 index e0746fe..b0d6d3b 100644 --- a/roles/config/templates/patches/oci/oci-csi-05-storage-class-00-bv.yaml.j2 +++ b/roles/config/templates/patches/oci/oci-csi-05-storage-class-00-bv.yaml.j2 @@ -6,9 +6,9 @@ provisioner: blockvolume.csi.oraclecloud.com volumeBindingMode: WaitForFirstConsumer allowVolumeExpansion: true reclaimPolicy: Delete -allowedTopologies: -- matchLabelExpressions: - - key: topology.kubernetes.io/zone - values: - - US-SANJOSE-1-AD-1 - - "" +#allowedTopologies: +#- matchLabelExpressions: +# - key: topology.kubernetes.io/zone +# values: +# - US-SANJOSE-1-AD-1 +# - "" diff --git a/roles/config/templates/patches/oci/oci-sample-csi-00-namespace.yaml.j2 b/roles/config/templates/patches/oci/oci-sample-csi-00-namespace.yaml.j2 new file mode 100644 index 0000000..eae219e --- /dev/null +++ b/roles/config/templates/patches/oci/oci-sample-csi-00-namespace.yaml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: oci-samples + labels: + "pod-security.kubernetes.io/enforce": "privileged" + "pod-security.kubernetes.io/audit": "privileged" + "pod-security.kubernetes.io/warn": "privileged" + "security.openshift.io/scc.podSecurityLabelSync": "false" + "openshift.io/run-level": "0" + "pod-security.kubernetes.io/enforce-version": "v1.24" diff --git a/roles/config/templates/patches/oci/oci-sample-lb-00-deployment.yaml.j2 b/roles/config/templates/patches/oci/oci-sample-lb-00-deployment.yaml.j2 index 27f4920..69283de 100644 --- a/roles/config/templates/patches/oci/oci-sample-lb-00-deployment.yaml.j2 +++ b/roles/config/templates/patches/oci/oci-sample-lb-00-deployment.yaml.j2 @@ -6,6 +6,9 @@ metadata: namespace: {{ oci_samples_namespace | d('oci-samples') }} spec: replicas: 1 + selector: + matchLabels: + app: nginx template: metadata: labels: diff --git a/roles/config/templates/patches/oci/oci-sample-lb-01-service.yaml.j2 b/roles/config/templates/patches/oci/oci-sample-lb-01-service.yaml.j2 index dcc9780..c717141 100644 --- a/roles/config/templates/patches/oci/oci-sample-lb-01-service.yaml.j2 +++ b/roles/config/templates/patches/oci/oci-sample-lb-01-service.yaml.j2 @@ -4,6 +4,8 @@ apiVersion: v1 metadata: name: nginx-service namespace: {{ oci_samples_namespace | d('oci-samples') }} + labels: + app: nginx spec: selector: app: nginx diff --git a/roles/config/templates/patches/oci/oci-samples-namespace.j2 b/roles/config/templates/patches/oci/oci-samples-namespace.j2 index 6cf0563..cbceb1d 100644 --- a/roles/config/templates/patches/oci/oci-samples-namespace.j2 +++ b/roles/config/templates/patches/oci/oci-samples-namespace.j2 @@ -1,5 +1,12 @@ --- -apiVersion: apps/v1 +apiVersion: v1 kind: Namespace metadata: name: {{ oci_samples_namespace | d('oci-samples') }} + labels: + "pod-security.kubernetes.io/enforce": "privileged" + "pod-security.kubernetes.io/audit": "privileged" + "pod-security.kubernetes.io/warn": "privileged" + "pod-security.kubernetes.io/enforce-version": "v1.24" + "security.openshift.io/scc.podSecurityLabelSync": "false" + "openshift.io/run-level": "0"