diff --git a/docs/documentation/integrate-provider.md b/docs/documentation/integrate-provider.md new file mode 100644 index 0000000..335fdb5 --- /dev/null +++ b/docs/documentation/integrate-provider.md @@ -0,0 +1,3 @@ +# Steps to Integrate a new Provider into okd-installer + +> Placeholder diff --git a/docs/guides/OCI/index.md b/docs/guides/OCI/index.md new file mode 100644 index 0000000..36c0dab --- /dev/null +++ b/docs/guides/OCI/index.md @@ -0,0 +1,17 @@ +# Guides for Oracle Cloud Infrastructure + +> WIP + +> TODO: + +Create guides/docs for OCP/OKD on OCI: + +- Installing a cluster with agnostic installation quickly (Platform=None) +- Installing a cluster with Cloud Controller Manager using External provider (Platform=External) +- Installing a cluster with External Cloud provider integration: CCM and `Platform External` +- Installing a cluster with agnostic installation with Assisted Installer as a installation provider + +Generic guides: + +- Integrate new provider to the Ansible Collection (UPI stacks) +- Adding CCM to existing integrated external provider diff --git a/docs/guides/OCI/oci-image-registry-bucket.md b/docs/guides/OCI/oci-image-registry-bucket.md new file mode 100644 index 0000000..6befb8e --- /dev/null +++ b/docs/guides/OCI/oci-image-registry-bucket.md @@ -0,0 +1,12 @@ +# OCI Image Registry - Use S3 compatibility URL for persistent storage + +> WIP + +Steps to use the OCI S3 Compatibility API to set the persistent storage for the OpenShift Image Registry with OCI Bucket service. + +Steps: + +- Create access Key +- Create the secret used by image-registry +- Edit the image registry object adding the s3 configuration +- Test it \ No newline at end of file diff --git a/docs/guides/OCI/oci-install-ccm.md b/docs/guides/OCI/oci-install-ccm.md new file mode 100644 index 0000000..19968f1 --- /dev/null +++ b/docs/guides/OCI/oci-install-ccm.md @@ -0,0 +1,107 @@ +## Install a cluster on OCI with CCM + +## Requirements + +- Credentials +- Client installed + +## OCP Cluster Setup on OCI + +### Generate the vars file + +```bash +cat < ~/.oci/env +# Compartment that the cluster will be installed +OCI_COMPARTMENT_ID="" + +# Compartment that the DNS Zone is created (based domain) +# Only RR will be added +OCI_COMPARTMENT_ID_DNS="" + +# Compartment that the OS Image will be created +OCI_COMPARTMENT_ID_IMAGE="" +EOF +source ~/.oci/env + +cat < ~/.openshift/env +export OCP_CUSTOM_RELEASE="docker.io/mtulio/ocp-release:latest" + +OCP_RELEASE_413="quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" +EOF +source ~/.openshift/env + +CLUSTER_NAME=oci-t13 +VAR_FILE=./vars-oci-ha_${CLUSTER_NAME}.yaml + +cat < ${VAR_FILE} +provider: oci +cluster_name: ${CLUSTER_NAME} +config_cluster_region: us-sanjose-1 + +#TODO: create compartment validations +#TODO: allow create compartment from a parent +oci_compartment_id: ${OCI_COMPARTMENT_ID} +oci_compartment_id_dns: ${OCI_COMPARTMENT_ID_DNS} +oci_compartment_id_image: ${OCI_COMPARTMENT_ID_IMAGE} + +cluster_profile: ha +destroy_bootstrap: no + +config_base_domain: splat-oci.devcluster.openshift.com +config_ssh_key: "$(cat ~/.ssh/id_rsa.pub;cat ~/.ssh/openshift-dev.pub)" +config_pull_secret_file: "${HOME}/.openshift/pull-secret-latest.json" + +config_cluster_version: 4.13.0-rc.0 +version: 4.13.0-rc.0 +config_installer_environment: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "$OCP_RELEASE_413" + +# Define the OS Image mirror +os_mirror: yes +os_mirror_from: stream_artifacts +os_mirror_stream: + architecture: x86_64 + artifact: openstack + format: qcow2.gz + +os_mirror_to_provider: oci +os_mirror_to_oci: + compartment_id: ${OCI_COMPARTMENT_ID_IMAGE} + bucket: rhcos-images + image_type: QCOW2 + +## Apply patches to installer manifests (WIP) +# TODO: we must keep the OCI CCM manifests patch more generic + +config_patches: +- rm-capi-machines +- mc-kubelet-providerid +- deploy-oci-ccm +- deploy-oci-csi +- yaml_patch + +cfg_patch_yaml_patch_specs: + ## patch infra object to create External provider + - manifest: /manifests/cluster-infrastructure-02-config.yml + patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}' + +cfg_patch_kubelet_providerid_script: | + PROVIDERID=\$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id); + +EOF +``` + +### Install the cluster + +```bash +ansible-playbook mtulio.okd_installer.create_all \ + -e certs_max_retries=20 \ + -e cert_wait_interval_sec=60 \ + -e @$VAR_FILE +``` + +## Destroy + +```bash +ansible-playbook mtulio.okd_installer.destroy_cluster -e @$VAR_FILE +``` diff --git a/docs/guides/OCI/oci-installing-quickly-examples.md b/docs/guides/OCI/oci-installing-quickly-examples.md new file mode 100644 index 0000000..18875ae --- /dev/null +++ b/docs/guides/OCI/oci-installing-quickly-examples.md @@ -0,0 +1,90 @@ +# Installing in OCI with build-in examples + +## Export variables + +```bash +export OKD_CONFIG_BASE_DOMAIN="" +export OCI_COMPARTMENT_ID="" +export OCI_COMPARTMENT_ID_DNS="" +export OCI_COMPARTMENT_ID_IMAGE="" +export OS_MIRROR_IMAGE_BUCKET_NAME="rhcos-images" +``` + +### Default vars + + +## Installing + + +### Installing a cluster on OCI with Platform Agnostic/None + +> TODO + +```bash +ansible-playbook examples/create-cluster.yaml \ + -e cluster_name=name \ + -e @./examples/vars/common.yaml \ + -e @./examples/vars/oci/common.yaml \ + -e @./examples/vars/oci/ha-platform-none.yaml +``` + +### Installing a cluster on OCI with Platform Agnostic/None with CSI Driver + +```bash +ansible-playbook examples/create-cluster.yaml \ + -e cluster_name=name \ + -e @./examples/vars/common.yaml \ + -e @./examples/vars/oci/common.yaml \ + -e @./examples/vars/oci/ha-platform-none-csi.yaml +``` + +### Installing a cluster on OCI with Platform External + +```bash +ansible-playbook examples/create-cluster.yaml \ + -e cluster_name=name \ + -e @./examples/vars/common.yaml \ + -e @./examples/vars/oci/common.yaml \ + -e @./examples/vars/oci/ha-platform-external.yaml +``` + +### Installing a cluster on OCI with Platform External with CCM + +```bash +ansible-playbook examples/create-cluster.yaml \ + -e cluster_name=name \ + -e @./examples/vars/common.yaml \ + -e @./examples/vars/oci/common.yaml \ + -e @./examples/vars/oci/ha-platform-external-ccm.yaml +``` + +### Installing a cluster on OCI with Platform External with CCM and CSI Driver + +```bash +ansible-playbook examples/create-cluster.yaml \ + -e cluster_name=name \ + -e @./examples/vars/common.yaml \ + -e @./examples/vars/oci/common.yaml \ + -e @./examples/vars/oci/ha-platform-external-ccm-csi.yaml +``` + +### Installing a cluster on OCI with Platform External with CSI Driver + +> TODO: OCI CSI Driver can be installed in Platform None with manual changes + + + +### Destroy a cluster + +```bash +ansible-playbook mtulio.okd_installer.destroy_cluster \ + -e cluster_name=name +``` \ No newline at end of file diff --git a/docs/guides/OCI/oci-installing-steps.md b/docs/guides/OCI/oci-installing-steps.md new file mode 100644 index 0000000..e50c7f7 --- /dev/null +++ b/docs/guides/OCI/oci-installing-steps.md @@ -0,0 +1,293 @@ +# Install OKD/OCP on OCI using an agnostic method + +> This document is under development on https://github.com/mtulio/ansible-collection-okd-installer/pull/26 + +Install OCP/OKD Cluster on Oracle Cloud Infrastructure using agnostic installation/UPI. + +- Prerequisites +- Installing OCP + - Install the Clientes + - Option 1 - Install quickly + - Option 2 - Install step-by-stack + - Create the Install config + - Create the manifests + - Setup IAM Stack + - Setup Network Stack + - Setup DNS Stack + - Setup Load Balancer Stack + - Patch the manifests + - Create the ignitions + - Setup Compute Stack + - Setup Bootstrap + - Setup Control Plane nodes + - Setup Compute nodes + - Check/Approve the certificates +- Review the Installation +- Destroy the Cluster + +## Prerequisites + +Read [here](./oci-prerequisites.md) + +## Installing OpenShift/OKD + +### Create the vars file + +```bash +cat < ~/.oci/env +# Compartment where the cluster will be installed +OCI_COMPARTMENT_ID="" + +# Compartment that the DNS Zone is created (based domain) +# Only RR will be added +OCI_COMPARTMENT_ID_DNS="" + +# Compartment that the OS Image will be created +OCI_COMPARTMENT_ID_IMAGE="" +EOF +source ~/.oci/env + +cat < ~/.openshift/env +export OCP_CUSTOM_RELEASE="quay.io/mtulio/ocp-release:latest" + +OCP_RELEASE_413="quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" +EOF +source ~/.openshift/env + +CLUSTER_NAME=oci-bm2 +VAR_FILE=./vars-oci-ha_${CLUSTER_NAME}.yaml + +cat < ${VAR_FILE} +provider: oci +cluster_name: ${CLUSTER_NAME} +config_cluster_region: us-sanjose-1 + +#TODO: create compartment validations +#TODO: allow create compartment from a parent +oci_compartment_id: ${OCI_COMPARTMENT_ID} +oci_compartment_id_dns: ${OCI_COMPARTMENT_ID_DNS} +oci_compartment_id_image: ${OCI_COMPARTMENT_ID_IMAGE} + +cluster_profile: ha +destroy_bootstrap: no + +config_base_domain: splat-oci.devcluster.openshift.com +config_ssh_key: "$(cat ~/.ssh/id_rsa.pub;cat ~/.ssh/openshift-dev.pub)" +config_pull_secret_file: "${HOME}/.openshift/pull-secret-latest.json" + +config_cluster_version: 4.13.0-rc.0 +version: 4.13.0-rc.0 +config_installer_environment: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" + +# Define the OS Image mirror +# custom_image_id: rhcos-412.86.202212081411-0-openstack.x86_64 + +os_mirror: yes +os_mirror_from: stream_artifacts +os_mirror_stream: + architecture: x86_64 + artifact: openstack + format: qcow2.gz + +os_mirror_to_provider: oci +os_mirror_to_oci: + compartment_id: ${OCI_COMPARTMENT_ID_IMAGE} + bucket: rhcos-images + image_type: QCOW2 + +## Apply patches to installer manifests (WIP) +# TODO: we must keep the OCI CCM manifests patch more generic + +config_patches: +- rm-capi-machines +- mc-kubelet-providerid +- deploy-oci-ccm +- deploy-oci-csi +- yaml_patch + +cfg_patch_yaml_patch_specs: + ## patch infra object to create External provider + - manifest: /manifests/cluster-infrastructure-02-config.yml + patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}' + +cfg_patch_kubelet_providerid_script: | + PROVIDERID=\$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id); + +# Customize instance type +#compute_shape: "BM.Standard.E2.64" +#compute_shape_config: {} + +EOF + +``` + +### Install the clients + +```bash +ansible-playbook mtulio.okd_installer.install_clients -e @$VAR_FILE +``` + +### Installing option 1: quickly install + +```bash +ansible-playbook mtulio.okd_installer.create_all \ + -e certs_max_retries=20 \ + -e cert_wait_interval_sec=60 \ + -e @$VAR_FILE +``` + +### Installing option 2: step-by-step + +#### Create the Installer Configuration + +Create the installation configuration: + + +```bash +ansible-playbook mtulio.okd_installer.config -e mode=create-config -e @$VAR_FILE +``` + +The rendered install-config.yaml will be available on the following path: + +- `~/.ansible/okd-installer/clusters/$CLUSTER_NAME/install-config.yaml` + +If you want to skip this part, place your own install-config.yaml on the same +path and go to the next step. + +#### Create the Installer manifests + +Create the installation configuration: + +```bash +ansible-playbook mtulio.okd_installer.config -e mode=create-manifests -e @$VAR_FILE +``` + +The manifests will be rendered and saved on the install directory: + +- `~/.ansible/okd-installer/clusters/$CLUSTER_NAME/` + +If you want to skip that part, with your manifests, you must be able to run +the `openshift-install create manifests` under the install directory, and the file +`manifests/cluster-config.yaml` is created correctly. + +The infrastructure manifest also must exist on the path: `manifests/cluster-infrastructure-02-config.yml`. + + +**After this stage, the file `$install_dir/cluster_state.json` will be created and populated with the stack results.** + +#### IAM Stack + +N/A + +> TODO: create Compartment validations + +#### Create the Network Stack + +```bash +ansible-playbook mtulio.okd_installer.stack_network -e @$VAR_FILE +``` + +#### DNS Stack + +```bash +ansible-playbook mtulio.okd_installer.stack_dns -e @$VAR_FILE +``` + +#### Load Balancer Stack + +```bash +ansible-playbook mtulio.okd_installer.stack_loadbalancer -e @$VAR_FILE +``` + +#### Config Commit + +This stage allows the user to modify the cluster configurations (manifests), +then generate the ignition files used to create the cluster. + +##### Manifest patches (pre-ign) + +In this step, the playbooks will apply any patches to the manifests, +according to the vars file `config_patches`. + +The `config_patches` are predefined tasks that will run to reach specific goals. + +If you wouldn't like to apply patches, leave the empty value `config_patches: []`. + +If you would like to apply patches manually, you can do it by changing the manifests +on the install dir. Default install dir path: `~/.ansible/okd-installer/clusters/${cluster_name}/*` + +```bash +ansible-playbook mtulio.okd_installer.config -e mode=patch-manifests -e @$VAR_FILE +``` + +##### Config generation (ignitions) + +These steps should be the last before the configuration be 'committed': + +- `create ignitions` when using `openshift-install` as the config provider + +```bash +ansible-playbook mtulio.okd_installer.config -e mode=create-ignitions -e @$VAR_FILE +``` + +#### Mirror OS boot image + +- Download the image from the URL provided by openshift-install coreos-stream + +> Example: `$ jq -r '.architectures["x86_64"].artifacts.openstack.formats["qcow2.gz"].disk.location' ~/.ansible/okd-installer/clusters/ocp-oci/coreos-stream.json` + +```bash +ansible-playbook mtulio.okd_installer.os_mirror -e @$VAR_FILE +``` + +#### Compute Stack + +##### Bootstrap node + +- Upload the bootstrap ignition to blob and Create the Bootstrap Instance + +```bash +ansible-playbook mtulio.okd_installer.create_node -e node_role=bootstrap -e @$VAR_FILE +``` + +##### Control Plane nodes + +- Create the Control Plane nodes + +```bash +ansible-playbook mtulio.okd_installer.create_node -e node_role=controlplane -e @$VAR_FILE +``` + +##### Compute/worker nodes + +- Create the Compute nodes + +```bash +ansible-playbook mtulio.okd_installer.create_node -e node_role=compute -e @$VAR_FILE +``` + +- Approve worker nodes' certificates signing requests (CSR) + +```bash +oc adm certificate approve $(oc get csr -o json |jq -r '.items[] | select(.status.certificate == null).metadata.name') + +# OR + +oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs oc adm certificate approve +``` + +## Review the installation + +```bash +export KUBECONFIG=${HOME}/.ansible/okd-installer/clusters/${cluster_name}/auth/kubeconfig + +oc get nodes +oc get co +``` + +## Destroy cluster + +```bash +ansible-playbook mtulio.okd_installer.destroy_cluster -e @$VAR_FILE +``` diff --git a/docs/guides/OCI/oci-prerequisites.md b/docs/guides/OCI/oci-prerequisites.md new file mode 100644 index 0000000..77e1615 --- /dev/null +++ b/docs/guides/OCI/oci-prerequisites.md @@ -0,0 +1,115 @@ +# OCI PoC - Prerequisites + +The steps described on this document can be changed from the final version. + +The goal is to quickly setup the PoC environment installing all the dependencies and Oracle Cloud Infrastructure identities to use the CLI/SDK with Ansible. + +### Setup Ansible project + +> This steps should be made only when OCI provider is under development - not merged to `main` branch. Then the normal install flow should be used. + +- Setup your ansible workdir (optional, you can use the defaults) + +```bash +cat < ansible.cfg +[defaults] +inventory = ./inventories +collections_path=./collections +callbacks_enabled=ansible.posix.profile_roles,ansible.posix.profile_tasks +hash_behavior=merge + +[inventory] +enable_plugins = yaml, ini + +[callback_profile_tasks] +task_output_limit=1000 +sort_order=none +EOF +``` + +- Create a virtual ennv + +```bash +python3.9 -m venv ./.venv-oci +source ./.venv-oci/bin/activate +``` + +- Donwload requirements files + +``` +wget https://raw.githubusercontent.com/mtulio/ansible-collection-okd-installer/main/requirements.yml +wget https://raw.githubusercontent.com/mtulio/ansible-collection-okd-installer/main/requirements.txt +``` + +- Update with OCI requirements + +```bash +cat <> requirements.txt + +# Oracle Cloud Infrastructure +oci +EOF + +cat <> requirements.yml + +# Oracle Cloud Infrastructure Ansible Collections +# https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.11.0/installation/index.html +- name: oracle.oci + version: '>=4.11.0,<4.12.0' +EOF +``` + +- Install ansible and dependencies + +```bash +pip install -r requirements.txt +``` + +- Install the Collections + +```bash +ansible-galaxy collection install -r requirements.yml +``` + +- Get the latest (under development) okd-installer for OCI + +> https://github.com/mtulio/ansible-collection-okd-installer/pull/26 + +```bash +git clone -b feat-added-provider-oci --recursive \ + git@github.com:mtulio/ansible-collection-okd-installer.git \ + collections/ansible_collections/mtulio/okd_installer +``` + +- Check if the collection is present + + +```bash +$ ansible-galaxy collection list |egrep "(okd_installer|^oracle)" +mtulio.okd_installer 0.0.0-latest +oracle.oci 4.11.0 +``` + +### Setup OCI credentials + +- See [API Key Authentication](https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.11.0/guides/authentication.html#api-key-authentication): +- See https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm#two + + +Make sure your credentials have been set correctly on the file `~/.oci/config` and you can use the OCI ansible collection: + +- Get the User ID from the documentation + +```bash +export oci_user_id=$(grep ^user ~/.oci/config | awk -F '=' '{print$2}') +``` + +- Retrieve facts from the user + +```bash +ansible localhost \ + -m oracle.oci.oci_identity_user_facts \ + -a user_id=${oci_user_id} +``` + +You must be able to collect the user information. \ No newline at end of file diff --git a/docs/guides/OCI/platform-external-custom-release.md b/docs/guides/OCI/platform-external-custom-release.md new file mode 100644 index 0000000..7e26e1e --- /dev/null +++ b/docs/guides/OCI/platform-external-custom-release.md @@ -0,0 +1,166 @@ +# Platform External - creating a custom release to support it on 4.13 + +This guide describes how to create a custom OCP release image with minimal changes to enable Platform `External` to be considered 'external' on the `library-go` - `IsCloudProviderExternal()`, signalizing the Kubelet (MCO) and Kube Controller Manager (KCMO) flag `--cloud-provider` be external, waiting for an external CCM be deployed on install time (in this case [OCI CCM](https://github.com/oracle/oci-cloud-controller-manager)) + +This is part of a PoC to enable Platform External to install CCM on install time. All the work has been mapped on the [Enhancement Proposal 1353](https://github.com/openshift/enhancements/pull/1353). + +## Update the API + +### API + +> The minimal changes on API have been created on 4.13. It's not required for this PoC. + +References: + +- https://github.com/openshift/api/pull/1301 +- https://github.com/openshift/api/pull/1409 + +### library-go + +- Clone the Library-go + +- Make the changes: https://github.com/openshift/library-go/compare/release-4.13...mtulio:library-go:release-4.13-platexternal?expand=1#diff-478af36e9fb994fc80d37b7d2f6ae207c67d8c43b94f98f6ae3e420808958ba9R40-R41 + +- Push to your account + + +## Rebuilding KCMO + +Steps to propagate the library-go change to kube-controller-manager-operator. + +- Clone the repo https://github.com/openshift/cluster-kube-controller-manager-operator + +- Update the go.mod to use your version of library-go https://github.com/openshift/cluster-kube-controller-manager-operator/compare/release-4.13...mtulio:cluster-kube-controller-manager-operator:release-4.13-platexternal?expand=1 + +`go.mod` +``` +replace github.com/openshift/library-go => github.com/mtulio/library-go v0.0.0-20230313023417-78e409222bff +``` + +- upload your custom changes (optional) + +```bash +$ git remote -v +mtulio git@github.com:mtulio/cluster-kube-controller-manager-operator.git (fetch) +mtulio git@github.com:mtulio/cluster-kube-controller-manager-operator.git (push) +origin git@github.com:openshift/cluster-kube-controller-manager-operator.git (fetch) +$ git push --set-upstream mtulio release-4.13-platexternal -f +``` + +- Build a custom image + + +```bash +QUAY_USER=mrbraga +REPO_NAME=cluster-kube-controller-manager-operator + +podman build \ + --authfile ${PULL_SECRET} \ + -f Dockerfile.rhel7 \ + -t quay.io/${QUAY_USER}/${REPO_NAME}:latest \ + && podman push quay.io/${QUAY_USER}/${REPO_NAME}:latest + +TS=$(date +%Y%m%d%H%M) +podman tag quay.io/${QUAY_USER}/${REPO_NAME}:latest \ + "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" && \ + podman push "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" +``` + +## Building MCO + +Steps to propagate the library-go change to machine-config-operator. + +- Clone the repo https://github.com/openshift/machine-config-operator + +- Update the go.mod to use your version of library-go + +`go.mod` +``` +replace github.com/openshift/library-go => github.com/mtulio/library-go v0.0.0-20230313023417-78e409222bff +``` + +- Build a custom image + +```shell +QUAY_USER=mrbraga +REPO_NAME=machine-config-operator + +podman build -f Dockerfile.rhel7 \ + -t quay.io/${QUAY_USER}/${REPO_NAME}:latest && \ + podman push quay.io/${QUAY_USER}/${REPO_NAME}:latest + +TS=$(date +%Y%m%d%H%M) +podman tag quay.io/${QUAY_USER}/${REPO_NAME}:latest \ + "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" && \ + podman push "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" +``` + +## Building CCCMO + +Steps to propagate the library-go change to cluster-cloud-controller-manager-operator. + +- Clone the repo https://github.com/mtulio/cluster-cloud-controller-manager-operator + +- Update the go.mod to use your version of library-go + +- Build a custom image + +```bash +QUAY_USER=mrbraga +REPO_NAME=cluster-cloud-controller-manager-operator + +podman build \ + --authfile ${PULL_SECRET} \ + -f Dockerfile \ + -t quay.io/${QUAY_USER}/${REPO_NAME}:latest \ + && podman push quay.io/${QUAY_USER}/${REPO_NAME}:latest + +TS=$(date +%Y%m%d%H%M) +podman tag quay.io/${QUAY_USER}/${REPO_NAME}:latest \ + "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" && \ + podman push "quay.io/${QUAY_USER}/${REPO_NAME}:${TS}" +``` + +## Create a new release + +- Choose the base image on https://openshift-release.apps.ci.l2s4.p1.openshiftapps.com/ + +- Run the command + +```bash +VERSION_BASE="4.13.0-rc.0-x86_64" +OCP_RELEASE_BASE="quay.io/openshift-release-dev/ocp-release:${VERSION_BASE}" +CUSTOM_IMAGE_NAMESPACE="quay.io/${QUAY_USER}" +NEW_RELEASE_IMAGE="docker.io/mtulio/ocp-release" + +$(which time) -v oc adm release new -n origin \ + --server https://api.ci.openshift.org \ + -a ${PULL_SECRET} \ + --from-release ${OCP_RELEASE_BASE} \ + --to-image "${NEW_RELEASE_IMAGE}:latest" \ + machine-config-operator=${CUSTOM_IMAGE_NAMESPACE}/machine-config-operator:latest \ + cluster-kube-controller-manager-operator=${CUSTOM_IMAGE_NAMESPACE}/cluster-kube-controller-manager-operator:latest \ + cluster-cloud-controller-manager-operator=${CUSTOM_IMAGE_NAMESPACE}/cluster-cloud-controller-manager-operator:latest +``` + +- Mirror it creating custom labels to identify the customization and base image + +```bash +podman pull "${NEW_RELEASE_IMAGE}:latest" + +podman tag "${NEW_RELEASE_IMAGE}:latest" \ + "${CUSTOM_IMAGE_NAMESPACE}/ocp-release:latest" && \ + podman push "${CUSTOM_IMAGE_NAMESPACE}/ocp-release:latest" +podman tag "${NEW_RELEASE_IMAGE}:latest" \ + "${CUSTOM_IMAGE_NAMESPACE}/ocp-release:${VERSION_BASE}_platexternal-kcmo-mco-3cmo" && \ + podman push "${CUSTOM_IMAGE_NAMESPACE}/ocp-release:${VERSION_BASE}_platexternal-kcmo-mco-3cmo" +``` + +- Check if the release image `${NEW_RELEASE_IMAGE}:latest` was created + +- Use it + +```bash +OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" \ + openshift-install create cluster --dir my-install-dir/ +``` \ No newline at end of file diff --git a/docs/guides/OCI/validate-cluster-with-opct.md b/docs/guides/OCI/validate-cluster-with-opct.md new file mode 100644 index 0000000..4466f9a --- /dev/null +++ b/docs/guides/OCI/validate-cluster-with-opct.md @@ -0,0 +1,51 @@ +## OPCT setup + +- Create the OPCT [dedicated] node + +> https://redhat-openshift-ecosystem.github.io/provider-certification-tool/user/#option-a-command-line + +```bash +# Create OPCT node +ansible-playbook mtulio.okd_installer.create_node \ + -e node_role=generic -e sufix=opct-01 -e cpu=4 -e mem=16 \ + -e subnet=private -e nsg=compute \ + -e @$VAR_FILE +``` + +- OPCT dedicated node setup + +```bash + +oc label node opct-01.priv.ocp.oraclevcn.com node-role.kubernetes.io/tests="" +oc adm taint node opct-01.priv.ocp.oraclevcn.com node-role.kubernetes.io/tests="":NoSchedule + +# Set the OPCT requirements (registry, labels, wait-for COs stable) +ansible-playbook ../opct/hack/opct-runner/opct-run-tool-preflight.yaml -e @$VAR_FILE -D + +``` + +- OPCT regular + +```bash +# Run OPCT +~/opct/bin/openshift-provider-cert-linux-amd64-v0.3.0 run -w + +# Get the results and explore it +~/opct/bin/openshift-provider-cert-linux-amd64-v0.3.0 retrieve +~/opct/bin/openshift-provider-cert-linux-amd64-v0.3.0 results *.tar.gz +~/opct/bin/openshift-provider-cert-linux-amd64-v0.3.0 report *.tar.gz +``` + +- OPCT upgrade mode + +```bash +# from a cluster 4.12.1, run upgrade conformance to 4.13 +~/opct/bin/openshift-provider-cert-linux-amd64-v0.3.0 run -w \ + --mode=upgrade \ + --upgrade-to-image=$(oc adm release info 4.13.0-ec.2 -o jsonpath={.image}) + +# Get the results and explore it +~/opct/bin/openshift-provider-cert-linux-amd64-v0.3.0 retrieve +~/opct/bin/openshift-provider-cert-linux-amd64-v0.3.0 results *.tar.gz +~/opct/bin/openshift-provider-cert-linux-amd64-v0.3.0 report *.tar.gz +``` diff --git a/examples/create-cluster.yaml b/examples/create-cluster.yaml new file mode 100644 index 0000000..12d5b62 --- /dev/null +++ b/examples/create-cluster.yaml @@ -0,0 +1,14 @@ +--- +# Usage examples +# +# OCI Platform None/Agnostic: +# $ ansible-playbook examples/create-cluster.yaml -e cluster_name +# -e @./examples/vars/common.yaml -e @./examples/vars/oci/common.yaml +# -e @./examples/vars/oci/ha-platform-none.yaml +# + +- name: install clients + ansible.builtin.import_playbook: mtulio.okd_installer.install_clients.yaml + +- name: create cluster + ansible.builtin.import_playbook: mtulio.okd_installer.create_all.yaml \ No newline at end of file diff --git a/examples/vars/common.yaml b/examples/vars/common.yaml new file mode 100644 index 0000000..333ef5a --- /dev/null +++ b/examples/vars/common.yaml @@ -0,0 +1,8 @@ +--- +config_base_domain: "{{ ansible_env['OKD_CONFIG_BASE_DOMAIN'] }}" +config_ssh_key: "{{ lookup('file', ansible_env['HOME'] + '/.ssh/id_rsa.pub') }}" +config_pull_secret_file: "{{ ansible_env['HOME'] }}/.openshift/pull-secret-latest.json" + +version: 4.12.8 + +destroy_bootstrap: no \ No newline at end of file diff --git a/examples/vars/oci/common.yaml b/examples/vars/oci/common.yaml new file mode 100644 index 0000000..eb951b4 --- /dev/null +++ b/examples/vars/oci/common.yaml @@ -0,0 +1,21 @@ +--- + +provider: oci +config_cluster_region: us-sanjose-1 + +oci_compartment_id: "{{ ansible_env['OCI_COMPARTMENT_ID'] }}" +oci_compartment_id_dns: "{{ ansible_env['OCI_COMPARTMENT_ID_DNS'] }}" +oci_compartment_id_image: "{{ ansible_env['OCI_COMPARTMENT_ID_IMAGE'] }}" + +os_mirror: yes +os_mirror_from: stream_artifacts +os_mirror_stream: + architecture: x86_64 + artifact: openstack + format: qcow2.gz + +os_mirror_to_provider: oci +os_mirror_to_oci: + compartment_id: "{{ oci_compartment_id_image }}" + bucket: "{{ oci_image_bucket | d(ansible_env['OS_MIRROR_IMAGE_BUCKET_NAME']) }}" + image_type: QCOW2 \ No newline at end of file diff --git a/examples/vars/oci/ha-platform-external-ccm-csi.yaml b/examples/vars/oci/ha-platform-external-ccm-csi.yaml new file mode 100644 index 0000000..ab48cc5 --- /dev/null +++ b/examples/vars/oci/ha-platform-external-ccm-csi.yaml @@ -0,0 +1,21 @@ +--- +cluster_profile: ha + +version: 4.13.0-rc.0 +config_installer_environment: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" + +config_patches: +- rm-capi-machines +- mc-kubelet-providerid +- deploy-oci-ccm +- deploy-oci-csi +- yaml_patch + +cfg_patch_yaml_patch_specs: + ## patch infra object to create External provider + - manifest: /manifests/cluster-infrastructure-02-config.yml + patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}' + +cfg_patch_kubelet_providerid_script: | + PROVIDERID=\$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id); \ No newline at end of file diff --git a/examples/vars/oci/ha-platform-external-ccm.yaml b/examples/vars/oci/ha-platform-external-ccm.yaml new file mode 100644 index 0000000..fa1d3f6 --- /dev/null +++ b/examples/vars/oci/ha-platform-external-ccm.yaml @@ -0,0 +1,20 @@ +--- +cluster_profile: ha + +version: 4.13.0-rc.0 +config_installer_environment: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" + +config_patches: +- rm-capi-machines +- mc-kubelet-providerid +- deploy-oci-ccm +- yaml_patch + +cfg_patch_yaml_patch_specs: + ## patch infra object to create External provider + - manifest: /manifests/cluster-infrastructure-02-config.yml + patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}' + +cfg_patch_kubelet_providerid_script: | + PROVIDERID=\$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id); \ No newline at end of file diff --git a/examples/vars/oci/ha-platform-external.yaml b/examples/vars/oci/ha-platform-external.yaml new file mode 100644 index 0000000..bba75b0 --- /dev/null +++ b/examples/vars/oci/ha-platform-external.yaml @@ -0,0 +1,19 @@ +--- +cluster_profile: ha + +version: 4.13.0-rc.0 +config_installer_environment: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/mrbraga/ocp-release:4.13.0-rc.0-x86_64_platexternal-kcmo-mco-3cmo" + +config_patches: +- rm-capi-machines +- mc-kubelet-providerid +- yaml_patch + +cfg_patch_yaml_patch_specs: + ## patch infra object to create External provider + - manifest: /manifests/cluster-infrastructure-02-config.yml + patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}' + +cfg_patch_kubelet_providerid_script: | + PROVIDERID=\$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id); \ No newline at end of file diff --git a/examples/vars/oci/ha-platform-none-csi.yaml b/examples/vars/oci/ha-platform-none-csi.yaml new file mode 100644 index 0000000..a21f7d6 --- /dev/null +++ b/examples/vars/oci/ha-platform-none-csi.yaml @@ -0,0 +1,6 @@ +--- +cluster_profile: ha + +config_patches: +- rm-capi-machines +- deploy-oci-csi \ No newline at end of file diff --git a/examples/vars/oci/ha-platform-none.yaml b/examples/vars/oci/ha-platform-none.yaml new file mode 100644 index 0000000..f45d765 --- /dev/null +++ b/examples/vars/oci/ha-platform-none.yaml @@ -0,0 +1,5 @@ +--- +cluster_profile: ha + +config_patches: +- rm-capi-machines \ No newline at end of file diff --git a/playbooks/vars/oci/profiles/default b/playbooks/vars/oci/profiles/default new file mode 120000 index 0000000..cca3261 --- /dev/null +++ b/playbooks/vars/oci/profiles/default @@ -0,0 +1 @@ +ha \ No newline at end of file diff --git a/playbooks/vars/oci/profiles/ha/destroy_resources.yaml b/playbooks/vars/oci/profiles/ha/destroy_resources.yaml new file mode 100644 index 0000000..50d26f8 --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/destroy_resources.yaml @@ -0,0 +1,26 @@ +--- +# placeholder +okd_cluster_destroy_instances_compartment_id: "{{ oci_compartment_id }}" +okd_cluster_destroy_instances: + - name: "{{ cluster_state.infra_id }}-bootstrap" + - name: "{{ cluster_state.infra_id }}-master-01" + - name: "{{ cluster_state.infra_id }}-master-02" + - name: "{{ cluster_state.infra_id }}-master-03" + - name: "{{ cluster_state.infra_id }}-worker-01" + - name: "{{ cluster_state.infra_id }}-worker-02" + - name: "{{ cluster_state.infra_id }}-worker-03" + - name: "{{ cluster_state.infra_id }}-opct-01" + wait: yes + wait_timeout: 120 + +okd_cluster_destroy_dns_compartment_id: "{{ oci_compartment_id_dns | d(oci_compartment_id) }}" +okd_cluster_destroy_dns_records: + zone_name_or_id: "{{ cluster_state.dns.base_domain }}" + patch_items_spec: + - operation: REMOVE + domain: "api.{{ cluster_state.dns.cluster_domain }}" + - operation: REMOVE + domain: "api-int.{{ cluster_state.dns.cluster_domain }}" + - operation: REMOVE + domain: "*.apps.{{ cluster_state.dns.cluster_domain }}" + diff --git a/playbooks/vars/oci/profiles/ha/dns.yaml b/playbooks/vars/oci/profiles/ha/dns.yaml new file mode 100644 index 0000000..0a9c737 --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/dns.yaml @@ -0,0 +1,24 @@ +--- + +# https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_dns_zone_module.html#ansible-collections-oracle-oci-oci-dns-zone-module + +cloud_dns_zones: + # public + - name: "{{ cluster_state.dns.base_domain }}" + provider: oci + spec: + # scope: GLOBAL + # zone_type: PRIMARY + compartment_id: "{{ oci_compartment_id_dns | d(oci_compartment_id) }}" + + # OCI is using subnet's zone. + # TODO: Need to check if will not conflict with custom private zone. + # Currently the cluster is resolving the DNS using the public zone. + # private + # - name: "{{ cluster_state.dns.cluster_domain }}" + # provider: oci + # view_name: "{{ cluster_state.infra_id }}-vcn" + # spec: + # scope: PRIVATE + # zone_type: PRIMARY + # compartment_id: "{{ oci_compartment_id_dns | d(oci_compartment_id) }}" diff --git a/playbooks/vars/oci/profiles/ha/iam.yaml b/playbooks/vars/oci/profiles/ha/iam.yaml new file mode 100644 index 0000000..02eb760 --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/iam.yaml @@ -0,0 +1,2 @@ +--- +# placeholder diff --git a/playbooks/vars/oci/profiles/ha/loadbalancer-router-default.yaml b/playbooks/vars/oci/profiles/ha/loadbalancer-router-default.yaml new file mode 100644 index 0000000..02eb760 --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/loadbalancer-router-default.yaml @@ -0,0 +1,2 @@ +--- +# placeholder diff --git a/playbooks/vars/oci/profiles/ha/loadbalancer.yaml b/playbooks/vars/oci/profiles/ha/loadbalancer.yaml new file mode 100644 index 0000000..69aface --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/loadbalancer.yaml @@ -0,0 +1,187 @@ +--- + +cloud_load_balancer_provider: oci + +# BackendSet +# https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_network_load_balancer_backend_set_module.html#ansible-collections-oracle-oci-oci-network-load-balancer-backend-set-module +# cloud_loadbalancer_targets: +# - name: "{{ cluster_state.infra_id }}-aext" +# provider: oci +# spec: +# name: "{{ cluster_state.infra_id }}-aext" +# compartment_id: "{{ oci_compartment_id }}" +# is_preserve_source: no +# ip_version: IPV4 +# #policy: TWO_TUPLE +# #backends: [] +# health_checker: +# port: 6443 +# protocol: HTTPS +# return_code: 200 +# url_path: /readyz +# interval_in_millis: 10000 +# timeout_in_millis: 3000 + + +# OCI NLB: https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_network_load_balancer_module.html#ansible-collections-oracle-oci-oci-network-load-balancer-module +cloud_loadbalancers: + - name: "{{ cluster_state.infra_id }}-nlb" + provider: oci + type: network + + # Is it supported multi-subnets? + subnet_name: "{{ cluster_state.infra_id }}-net-public" + nsg_name: "{{ cluster_state.infra_id }}-nsg-nlb" + spec: + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-nlb" + is_private: false + is_preserve_source_destination: false + nlb_ip_version: IPV4 + #freeform_tags: "{{ cluster_state.tags }}" + +# BackendSet +# https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_network_load_balancer_backend_set_module.html#ansible-collections-oracle-oci-oci-network-load-balancer-backend-set-module + backend_set: + - provider: oci + spec: + name: "{{ cluster_state.infra_id }}-api" + is_preserve_source: false + ip_version: IPV4 + policy: FIVE_TUPLE + #backends: [] + health_checker: + port: 6443 + protocol: HTTPS + return_code: 200 + url_path: /readyz + interval_in_millis: 10000 + timeout_in_millis: 3000 + + - provider: oci + spec: + name: "{{ cluster_state.infra_id }}-mcs" + is_preserve_source: false + ip_version: IPV4 + policy: FIVE_TUPLE + #backends: [] + health_checker: + port: 22623 + protocol: HTTPS + return_code: 200 + url_path: /healthz + interval_in_millis: 10000 + timeout_in_millis: 3000 + + - provider: oci + spec: + name: "{{ cluster_state.infra_id }}-ingress-http" + is_preserve_source: false + ip_version: IPV4 + policy: FIVE_TUPLE + #backends: [] # TCP/31794 + health_checker: + port: 80 + protocol: TCP + # return_code: 200 + # url_path: /healthz + interval_in_millis: 10000 + timeout_in_millis: 3000 + + - provider: oci + spec: + name: "{{ cluster_state.infra_id }}-ingress-https" + is_preserve_source: false + ip_version: IPV4 + #policy: TWO_TUPLE + #backends: [] # TCP/32186 + health_checker: + port: 443 + protocol: TCP + # return_code: 200 + # url_path: /healthz + interval_in_millis: 10000 + timeout_in_millis: 3000 + + # https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_network_load_balancer_listener_module.html#ansible-collections-oracle-oci-oci-network-load-balancer-listener-module + listeners: + - spec: + name: "{{ cluster_state.infra_id }}-api" + default_backend_set_name: "{{ cluster_state.infra_id }}-api" + ip_version: IPV4 + port: 6443 + protocol: TCP + + - spec: + name: "{{ cluster_state.infra_id }}-mcs" + default_backend_set_name: "{{ cluster_state.infra_id }}-mcs" + ip_version: IPV4 + port: 22623 + protocol: TCP + + - spec: + name: "{{ cluster_state.infra_id }}-ingress-http" + default_backend_set_name: "{{ cluster_state.infra_id }}-ingress-http" + ip_version: IPV4 + port: 80 + protocol: TCP + + - spec: + name: "{{ cluster_state.infra_id }}-ingress-https" + default_backend_set_name: "{{ cluster_state.infra_id }}-ingress-https" + ip_version: IPV4 + port: 443 + protocol: TCP + + callbacks: + - name: register_dns + rr_ip: public + spec: + zone_name_or_id: "{{ cluster_state.dns.base_domain }}" + compartment_id: "{{ oci_compartment_id_dns | d(oci_compartment_id) }}" + scope: GLOBAL + patch_items: + - domain: "api.{{ cluster_state.dns.cluster_domain }}" + rtype: A + ttl: 300 + - domain: "*.apps.{{ cluster_state.dns.cluster_domain }}" + rtype: A + ttl: 300 + - name: register_dns + rr_ip: private + spec: + zone_name_or_id: "{{ cluster_state.dns.base_domain }}" + compartment_id: "{{ oci_compartment_id_dns | d(oci_compartment_id) }}" + scope: GLOBAL + patch_items: + - domain: "api-int.{{ cluster_state.dns.cluster_domain }}" + rtype: A + ttl: 300 + + # # private address + # - name: register_dns + # rr_ip: private + # view_name: "{{ cluster_state.infra_id }}-vcn" + # spec: + # zone_name_or_id: "{{ cluster_state.dns.cluster_domain }}" + # compartment_id: "{{ oci_compartment_id }}" + # scope: PRIVATE + # patch_items: + # - domain: "api-int.{{ cluster_state.dns.cluster_domain }}" + # rtype: A + # ttl: 300 + + # - name: register_dns + # rr_ip: public + # view_name: "{{ cluster_state.infra_id }}-vcn" + # spec: + # zone_name_or_id: "{{ cluster_state.dns.cluster_domain }}" + # compartment_id: "{{ oci_compartment_id }}" + # scope: PRIVATE + # patch_items: + # - domain: "api.{{ cluster_state.dns.cluster_domain }}" + # rtype: A + # ttl: 300 + # - domain: "*.apps.{{ cluster_state.dns.cluster_domain }}" + # rtype: A + # ttl: 300 diff --git a/playbooks/vars/oci/profiles/ha/network.yaml b/playbooks/vars/oci/profiles/ha/network.yaml new file mode 100644 index 0000000..54869c6 --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/network.yaml @@ -0,0 +1,332 @@ +################################ +# AWS Networks +# AWS us-east-1: 10.0.0.0/16 (to 10.0.255.255/16) +# AWS : 10.23.0.0/16 (to 10.23.255.255/19) + +######################### + +# TODO: fix those rules to more restrictive. This is used to dev env. +# security_groups: [] + +cloud_networks: + ## OCI US San Jose 1 (HA topology) + - name: "{{ cluster_state.infra_id }}-vcn" + block: "{{ okd_net_default_cidr }}" + provider: oci + region: "{{ config_cluster_region }}" + compartment_id: "{{ oci_compartment_id }}" + + security_groups: "{{ security_groups | d([]) }}" + tags: "{{ cluster_state.tags | d({}) }}" + + spec_vcn: + dns_label: ocp + + internet_gateway: + spec: + is_enabled: yes + display_name: "{{ cluster_state.infra_id }}-igw" + + nat_gateway: + spec: + display_name: "{{ cluster_state.infra_id }}-natgw" + # route_table_name: "{{ cluster_state.infra_id }}-rt-private" + + # tags: "{{ cluster_state.tags | d({}) }}" + # wait: false + # - name: "{{ cluster_state.infra_id }}-natgw-1b" + # subnet: "{{ cluster_state.infra_id }}-net-public-1b" + # tags: "{{ cluster_state.tags | d({}) }}" + # wait: false + # - name: "{{ cluster_state.infra_id }}-natgw-1c" + # subnet: "{{ cluster_state.infra_id }}-net-public-1c" + # tags: "{{ cluster_state.tags | d({}) }}" + # wait: true + + #> TODO use generic list + route_table_public: + spec: + display_name: "{{ cluster_state.infra_id }}-rt-public" + route_rules: + - spec: + destination: "0.0.0.0/0" + destination_type: "CIDR_BLOCK" + network_entity_type: internet-gateway + network_entity_name: "{{ cluster_state.infra_id }}-igw" + + route_table_private: + spec: + display_name: "{{ cluster_state.infra_id }}-rt-private" + route_rules: + - spec: + destination: "0.0.0.0/0" + destination_type: "CIDR_BLOCK" + network_entity_type: nat-gateway + network_entity_name: "{{ cluster_state.infra_id }}-natgw" + + # # Generic list + # route_tables: + # - public: yes + # spec: + # display_name: "{{ cluster_state.infra_id }}-rt-public" + # route_rules: + # - spec: + # destination: "0.0.0.0/0" + # destination_type: "CIDR_BLOCK" + # network_entity_type: internet-gateway + # network_entity_name: "{{ cluster_state.infra_id }}-igw" + + # - public: no + # spec: + # display_name: "{{ cluster_state.infra_id }}-rt-private" + # route_rules: + # - spec: + # destination: "0.0.0.0/0" + # destination_type: "CIDR_BLOCK" + # network_entity_type: nat-gateway + # network_entity_name: "{{ cluster_state.infra_id }}-natgw" + + # create permissive security Lists to force to use NSGs + # https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_network_security_list_module.html#ansible-collections-oracle-oci-oci-network-security-list-module + security_lists: + - spec: + display_name: "{{ cluster_state.infra_id }}-seclist-public" + ingress_security_rules: [] + # - protocol: all + # source: 0.0.0.0/0 + # is_stateless: false + # source_type: CIDR_BLOCK + # description: Allow all inbound + egress_security_rules: + - destination: 0.0.0.0/0 + protocol: all + destination_type: CIDR_BLOCK + is_stateless: false + description: Allow All Outbound + + - spec: + display_name: "{{ cluster_state.infra_id }}-seclist-private" + # TODO restrict only for I/O of listeners/backendSet + ingress_security_rules: [] + # - protocol: all + # source: 0.0.0.0/0 + # is_stateless: false + # source_type: CIDR_BLOCK + # description: Allow all inbound + egress_security_rules: + - destination: 0.0.0.0/0 + protocol: all + destination_type: CIDR_BLOCK + is_stateless: false + description: Allow All Outbound + # - destination: "10.0.0.0/16" + # protocol: 6 + # tcp_options: + # destination_port_range: + # min: 6443 + # max: 6443 + # destination_type: CIDR_BLOCK + # is_stateless: false + # description: Allow API Outbound + # - destination: "10.0.0.0/16" + # protocol: 6 + # tcp_options: + # destination_port_range: + # min: 22623 + # max: 22623 + # destination_type: CIDR_BLOCK + # is_stateless: false + # description: Allow MCS Outbound + + subnets: + # Best practice create dedicated LB Subnet/Rtb? + # https://docs.oracle.com/en-us/iaas/Content/GSG/Tasks/loadbalancing.htm#Update + # Your load balancer must reside in different subnets from your application instances. This configuration allows you to keep your application instances secured in subnets with stricter access rules, while allowing public internet traffic to the load balancer in the public subnets. + + # - public: no + # spec: + # display_name: "{{ cluster_state.infra_id }}-net-private" + # cidr_block: "10.0.0.0/22" + # prohibit_public_ip_on_vnic: true + + - spec: + display_name: "{{ cluster_state.infra_id }}-net-public" + cidr_block: "10.0.0.0/20" + dns_label: pub + prohibit_internet_ingress: false + public: true + route_table_name: "{{ cluster_state.infra_id }}-rt-public" + security_list_names: + - "{{ cluster_state.infra_id }}-seclist-public" + + - spec: + display_name: "{{ cluster_state.infra_id }}-net-private" + cidr_block: "10.0.16.0/20" + dns_label: priv + prohibit_internet_ingress: true + public: false + route_table_name: "{{ cluster_state.infra_id }}-rt-private" + security_list_names: + - "{{ cluster_state.infra_id }}-seclist-private" + + + # - public: no + # security_list_names: + # - "{{ cluster_state.infra_id }}-seclist-default" + # spec: + # cidr_block: "10.0.32.0/22" + # display_name: "{{ cluster_state.infra_id }}-net-private-nodes" + # prohibit_internet_ingress: true + # prohibit_public_ip_on_vnic: true + # dns_label: nodes + + # - public: yes + # route_table_name: "{{ cluster_state.infra_id }}-rt-public-lb" + # security_list_names: + # - "{{ cluster_state.infra_id }}-seclist-lb" + # spec: + # cidr_block: "10.0.0.0/24" + # display_name: "{{ cluster_state.infra_id }}-net-public-lb" + + + # https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_network_security_list_module.html#ansible-collections-oracle-oci-oci-network-security-list-module + network_security_groups: + - spec: + display_name: "{{ cluster_state.infra_id }}-nsg-controlplane" + rules: + spec: + security_rules: + # INGRESS + - description: allow all inbound subnet + source: "10.0.0.0/16" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: all + + - description: allow ssh + source: "0.0.0.0/0" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: 6 + tcp_options: + destination_port_range: + min: 22 + max: 22 + + - description: allow API + source: "0.0.0.0/0" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: 6 + tcp_options: + destination_port_range: + min: 6443 + max: 6443 + + # EGRESS + - description: allow all outbound subnet + destination: "0.0.0.0/0" + destination_type: "CIDR_BLOCK" + direction: EGRESS + is_stateless: false + protocol: all + + - spec: + display_name: "{{ cluster_state.infra_id }}-nsg-compute" + rules: + spec: + security_rules: + # INGRESS + - description: allow all inbound subnet + source: "10.0.0.0/16" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: all + + - description: allow ssh + source: "0.0.0.0/0" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: 6 + tcp_options: + destination_port_range: + min: 22 + max: 22 + + # EGRESS + - description: allow all outbound subnet + destination: "0.0.0.0/0" + destination_type: "CIDR_BLOCK" + direction: EGRESS + is_stateless: false + protocol: all + + - spec: + display_name: "{{ cluster_state.infra_id }}-nsg-nlb" + rules: + spec: + security_rules: + # INGRESS + - description: allow all inbound subnet + source: "10.0.0.0/16" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: all + + - description: allow KAPI + source: "0.0.0.0/0" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: 6 + tcp_options: + destination_port_range: + min: 6443 + max: 6443 + + - description: allow MCS + source: "0.0.0.0/0" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: 6 + tcp_options: + destination_port_range: + min: 22623 + max: 22623 + + - description: allow IG-HTTP + source: "0.0.0.0/0" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: 6 + tcp_options: + destination_port_range: + min: 80 + max: 80 + + - description: allow IG-HTTPS + source: "0.0.0.0/0" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: 6 + tcp_options: + destination_port_range: + min: 443 + max: 443 + + # EGRESS + - description: allow all outbound subnet + destination: "0.0.0.0/0" + destination_type: "CIDR_BLOCK" + direction: EGRESS + is_stateless: false + protocol: all diff --git a/playbooks/vars/oci/profiles/ha/node-bootstrap.yaml b/playbooks/vars/oci/profiles/ha/node-bootstrap.yaml new file mode 100644 index 0000000..2841235 --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/node-bootstrap.yaml @@ -0,0 +1,134 @@ +--- +_cluster_prefix: "{{ cluster_state.infra_id }}" + +# Vars used on Bootstrap +bootstrap_bucket: "{{ _cluster_prefix }}-infra" + +# Vars used on Machine/Compute Stack +_instance_type: "{{ bootstrap_instance | d('m6i.xlarge') }}" +_instance_profile: "{{ cluster_state.compute.iam_profile_bootstrap }}" +# _image_id: "{{ custom_image_id | d(cluster_state.compute.image_id) }}" +_image_id: "{{ custom_image_id }}" +_subnet_name: "{{ _cluster_prefix }}-net-public-1a" + +_machine_suffix: '' + +## User Data template +userdata_config_source: "{{ bootstrap_bucket_signed_url }}" + +## Common vars used in the Stack vars +# _common: +# prefix: "{{ _cluster_prefix }}-bootstrap" +# detailed_monitoring: yes +# ebs_optimized: no +# image_id: "{{ _image_id }}" +# instance_role: "{{ _instance_profile }}" +# instance_type: "{{ _instance_type }}" +# security_groups: +# - "{{ _cluster_prefix }}-bootstrap-sg" +# - "{{ _cluster_prefix }}-controlplane-sg" +# state: present +# tags: "{{ cluster_state.tags }}" +# termination_protection: no +# volumes: +# - device_name: /dev/xvda +# ebs: +# volume_size: 128 +# volume_type: gp3 +# delete_on_termination: true +# - device_name: /dev/xvdd +# ebs: +# volume_size: 32 +# volume_type: gp3 +# delete_on_termination: true + +# vpc_subnet_name: "{{ _subnet_name }}" +# wait: yes +# wait_timeout: 500 + +# Stack Compute (Ansible Role cloud_compute) options: +compute_resources: + # + # Node role: bootstrap + # Node: bootstrap + # + - provider: oci + type: machine + # name: "{{ cluster_state.infra_id }}-bootstrap{{ _machine_suffix }}" + + # RHCOS Custom Image + image_name: "{{ cluster_state.compute.image_id }}" + image_compartment_id: "{{ oci_compartment_id_image | d(oci_compartment_id) }}" + + # Network details + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-public" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-controlplane" + # OCI spec + spec: + state: present + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-bootstrap{{ _machine_suffix }}" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + # platform_config: + # type: AMD_VM + shape: "VM.Standard.E4.Flex" + shape_config: + ocpus: 4 + memory_in_gbs: 16 + #baseline_ocpu_utilization: BASELINE_1_8 + #nvmes: 1 + fault_domain: FAULT-DOMAIN-1 + # availability_domain: Uocm:PHX-AD-1 + agent_config: + are_all_plugins_disabled: true + + # Disk Configuration + preserve_boot_volume: false + source_details: + source_type: image + boot_volume_size_in_gbs: 120 + boot_volume_vpus_per_gb: 30 + + # that config will prevent actions like stop/start (not desired) + # preemptible_instance_config: + # preemption_action: + # preserve_boot_volume: false + # type: TERMINATE + + # Network + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-bootstrap-vnic0" + assign_public_ip: true + assign_private_dns_record: true + hostname_label: "bootstrap{{ _machine_suffix }}" + # defined_tags: {'Operations': {'CostCenter': 'US'}} + # freeform_tags: {'Department': 'Finance'} + # private_ip: private_ip_example + # skip_source_dest_check: true + # vlan_id: "ocid1.vlan.oc1..xxxxxxEXAMPLExxxxxx" + #subnet_id: "{{ machine_subnet_id }}" + #nsg_ids: "{{ machine_nsg_ids }}" + metadata: + user_data: "{{ lookup('template', 'ocp-bootstrap-user-data.j2') | to_nice_json | string | b64encode }}" + + # launch_options: + # firmware: BIOS + # boot_volume_type: PARAVIRTUALIZED + + callbacks: + - name: nlb + # nlb_name: ocp-nlb + nlb_name: "{{ cluster_state.infra_id }}-nlb" + backend_sets: + - name: "{{ cluster_state.infra_id }}-api" + port: 6443 + - name: "{{ cluster_state.infra_id }}-mcs" + port: 22623 + # - name: "6443" + # port: 6443 + # - name: "22623" + # port: 22623 diff --git a/playbooks/vars/oci/profiles/ha/node-compute.yaml b/playbooks/vars/oci/profiles/ha/node-compute.yaml new file mode 100644 index 0000000..4eec751 --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/node-compute.yaml @@ -0,0 +1,188 @@ +--- +# Vars used on Machine/Compute Stack +_userdata_path: "{{ config_install_dir }}/worker.ign" + +_shape_config_default: + ocpus: 4 + memory_in_gbs: 16 + #baseline_ocpu_utilization: BASELINE_1_8 + #nvmes: 1 + +_shape: "{{ compute_shape | d('VM.Standard.E4.Flex') }}" +_shape_config: "{{ compute_shape_config | d(_shape_config_default) }}" + +# Stack Compute (Ansible Role cloud_compute) options: +compute_resources: + # + # Node role: compute + # Node: worker-01 + # + - provider: oci + type: machine + + # RHCOS Custom Image + image_name: "{{ cluster_state.compute.image_id }}" + image_compartment_id: "{{ oci_compartment_id_image | d(oci_compartment_id) }}" + + # Network details + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-private" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-compute" + + # OCI spec + spec: + state: present + wait: yes + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-worker-01" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + fault_domain: FAULT-DOMAIN-1 + + # platform_config: + # type: AMD_VM + shape: "{{ _shape }}" + shape_config: "{{ _shape_config }}" + + agent_config: + are_all_plugins_disabled: true + + source_details: + source_type: image + boot_volume_size_in_gbs: 120 + boot_volume_vpus_per_gb: 20 + + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-worker-01-vnic0" + assign_public_ip: false + assign_private_dns_record: true + hostname_label: "worker-01" + metadata: + user_data: "{{ lookup('file', _userdata_path) | b64encode }}" + + callbacks: + - name: nlb + nlb_name: "{{ cluster_state.infra_id }}-nlb" + backend_sets: + - name: "{{ cluster_state.infra_id }}-ingress-http" + port: 80 + - name: "{{ cluster_state.infra_id }}-ingress-https" + port: 443 + + # + # Node role: compute + # Node: worker-02 + # + - provider: oci + type: machine + + # RHCOS Custom Image + image_name: "{{ cluster_state.compute.image_id }}" + image_compartment_id: "{{ oci_compartment_id_image | d(oci_compartment_id) }}" + + # Network details + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-private" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-compute" + + # OCI spec + spec: + state: present + wait: yes + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-worker-02" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + fault_domain: FAULT-DOMAIN-2 + + # platform_config: + # type: AMD_VM + shape: "{{ _shape }}" + shape_config: "{{ _shape_config }}" + + agent_config: + are_all_plugins_disabled: true + + source_details: + source_type: image + boot_volume_size_in_gbs: 120 + boot_volume_vpus_per_gb: 20 + + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-worker-02-vnic0" + assign_public_ip: false + assign_private_dns_record: true + hostname_label: "worker-02" + metadata: + user_data: "{{ lookup('file', _userdata_path) | b64encode }}" + + callbacks: + - name: nlb + nlb_name: "{{ cluster_state.infra_id }}-nlb" + backend_sets: + - name: "{{ cluster_state.infra_id }}-ingress-http" + port: 80 + - name: "{{ cluster_state.infra_id }}-ingress-https" + port: 443 + + # + # Node role: compute + # Node: worker-03 + # + - provider: oci + type: machine + + # RHCOS Custom Image + image_name: "{{ cluster_state.compute.image_id }}" + image_compartment_id: "{{ oci_compartment_id_image | d(oci_compartment_id) }}" + + # Network details + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-private" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-compute" + + # OCI spec + spec: + state: present + wait: no + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-worker-03" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + fault_domain: FAULT-DOMAIN-3 + + # platform_config: + # type: AMD_VM + shape: "{{ _shape }}" + shape_config: "{{ _shape_config }}" + + agent_config: + are_all_plugins_disabled: true + + source_details: + source_type: image + boot_volume_size_in_gbs: 120 + boot_volume_vpus_per_gb: 20 + + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-worker-03-vnic0" + assign_public_ip: false + assign_private_dns_record: true + hostname_label: "worker-03" + metadata: + user_data: "{{ lookup('file', _userdata_path) | b64encode }}" + + callbacks: + - name: nlb + nlb_name: "{{ cluster_state.infra_id }}-nlb" + backend_sets: + - name: "{{ cluster_state.infra_id }}-ingress-http" + port: 80 + - name: "{{ cluster_state.infra_id }}-ingress-https" + port: 443 \ No newline at end of file diff --git a/playbooks/vars/oci/profiles/ha/node-controlplane.yaml b/playbooks/vars/oci/profiles/ha/node-controlplane.yaml new file mode 100644 index 0000000..e145ec9 --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/node-controlplane.yaml @@ -0,0 +1,189 @@ +--- + +_userdata_path: "{{ config_install_dir }}/master.ign" + +# Stack Compute (Ansible Role cloud_compute) options: +compute_resources: + # + # Node role: controlplane + # Node: master-01 + # + - provider: oci + type: machine + + # RHCOS Custom Image + image_name: "{{ cluster_state.compute.image_id }}" + image_compartment_id: "{{ oci_compartment_id_image | d(oci_compartment_id) }}" + + # Network details + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-private" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-controlplane" + + # OCI spec + spec: + state: present + wait: no + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-master-01" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + fault_domain: FAULT-DOMAIN-1 + + # platform_config: + # type: AMD_VM + shape: "VM.Standard.E4.Flex" + shape_config: + ocpus: 4 + memory_in_gbs: 16 + #baseline_ocpu_utilization: BASELINE_1_8 + #nvmes: 1 + agent_config: + are_all_plugins_disabled: true + + source_details: + source_type: image + boot_volume_size_in_gbs: 120 + boot_volume_vpus_per_gb: 20 + + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-master-01-vnic0" + assign_public_ip: false + assign_private_dns_record: true + hostname_label: "master-01" + metadata: + user_data: "{{ lookup('file', _userdata_path) | b64encode }}" + + callbacks: + - name: nlb + nlb_name: "{{ cluster_state.infra_id }}-nlb" + backend_sets: + - name: "{{ cluster_state.infra_id }}-api" + port: 6443 + - name: "{{ cluster_state.infra_id }}-mcs" + port: 22623 + + + # + # Node role: controlplane + # Node: master-02 + # + - provider: oci + type: machine + + # RHCOS Custom Image + image_name: "{{ cluster_state.compute.image_id }}" + image_compartment_id: "{{ oci_compartment_id_image | d(oci_compartment_id) }}" + + # Network details + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-private" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-controlplane" + + # OCI spec + spec: + state: present + wait: no + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-master-02" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + fault_domain: FAULT-DOMAIN-2 + + # platform_config: + # type: AMD_VM + shape: "VM.Standard.E4.Flex" + shape_config: + ocpus: 4 + memory_in_gbs: 16 + #baseline_ocpu_utilization: BASELINE_1_8 + #nvmes: 1 + agent_config: + are_all_plugins_disabled: true + + source_details: + source_type: image + boot_volume_size_in_gbs: 120 + boot_volume_vpus_per_gb: 20 + + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-master-02-vnic0" + assign_public_ip: false + assign_private_dns_record: true + hostname_label: "master-02" + metadata: + user_data: "{{ lookup('file', _userdata_path) | b64encode }}" + + callbacks: + - name: nlb + nlb_name: "{{ cluster_state.infra_id }}-nlb" + backend_sets: + - name: "{{ cluster_state.infra_id }}-api" + port: 6443 + - name: "{{ cluster_state.infra_id }}-mcs" + port: 22623 + + # + # Node role: controlplane + # Node: master-03 + # + - provider: oci + type: machine + + # RHCOS Custom Image + image_name: "{{ cluster_state.compute.image_id }}" + image_compartment_id: "{{ oci_compartment_id_image | d(oci_compartment_id) }}" + + # Network details + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-private" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-controlplane" + + # OCI spec + spec: + state: present + wait: yes + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-master-03" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + fault_domain: FAULT-DOMAIN-3 + + # platform_config: + # type: AMD_VM + shape: "VM.Standard.E4.Flex" + shape_config: + ocpus: 4 + memory_in_gbs: 16 + #baseline_ocpu_utilization: BASELINE_1_8 + #nvmes: 1 + agent_config: + are_all_plugins_disabled: true + + source_details: + source_type: image + boot_volume_size_in_gbs: 120 + boot_volume_vpus_per_gb: 20 + + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-master-03-vnic0" + assign_public_ip: false + assign_private_dns_record: true + hostname_label: "master-03" + metadata: + user_data: "{{ lookup('file', _userdata_path) | b64encode }}" + + callbacks: + - name: nlb + nlb_name: "{{ cluster_state.infra_id }}-nlb" + backend_sets: + - name: "{{ cluster_state.infra_id }}-api" + port: 6443 + - name: "{{ cluster_state.infra_id }}-mcs" + port: 22623 diff --git a/playbooks/vars/oci/profiles/ha/node-generic.yaml b/playbooks/vars/oci/profiles/ha/node-generic.yaml new file mode 100644 index 0000000..8712ac7 --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/node-generic.yaml @@ -0,0 +1,57 @@ +--- +# Vars used on Machine/Compute Stack +_userdata_path: "{{ config_install_dir }}/worker.ign" + +# Stack Compute (Ansible Role cloud_compute) options: +compute_resources: + # + # Node role: compute + # Node: opct-01 + # + - provider: oci + type: machine + + # RHCOS Custom Image + image_name: "{{ cluster_state.compute.image_id }}" + image_compartment_id: "{{ oci_compartment_id_image | d(oci_compartment_id) }}" + + # Network details + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-{{ subnet | d('net-private') }}" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-{{ nsg | d('nsg-compute') }}" + + # OCI spec + spec: + state: present + wait: yes + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-{{ sufix | d ('generic-01') }}" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + fault_domain: FAULT-DOMAIN-1 + + # platform_config: + # type: AMD_VM + shape: "VM.Standard.E4.Flex" + shape_config: + ocpus: "{{ cpu | d(2) }}" + memory_in_gbs: "{{ mem | d(8) }}" + #baseline_ocpu_utilization: BASELINE_1_8 + #nvmes: 1 + agent_config: + are_all_plugins_disabled: true + + source_details: + source_type: image + boot_volume_size_in_gbs: 120 + boot_volume_vpus_per_gb: 20 + + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-{{ sufix | d ('generic-01') }}-vnic0" + assign_public_ip: false + assign_private_dns_record: true + hostname_label: "{{ sufix | d ('generic-01') }}" + metadata: + user_data: "{{ lookup('file', _userdata_path) | b64encode }}" diff --git a/roles/bootstrap/tasks/oci.yaml b/roles/bootstrap/tasks/oci.yaml new file mode 100644 index 0000000..1213d0b --- /dev/null +++ b/roles/bootstrap/tasks/oci.yaml @@ -0,0 +1,51 @@ +--- +# https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_object_storage_object_module.html#ansible-collections-oracle-oci-oci-object-storage-object-module +# https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_object_storage_object_module.html#ansible-collections-oracle-oci-oci-object-storage-object-module + +- name: OCI | Get the namespace + oracle.oci.oci_object_storage_namespace_facts: + compartment_id: "{{ oci_compartment_id }}" + register: _objns + +- name: OCI | Create bucket + oracle.oci.oci_object_storage_bucket: + compartment_id: "{{ oci_compartment_id }}" + name: "{{ bootstrap_bucket }}" + namespace_name: "{{ _objns.namespace }}" + state: present + +# TODO: Make it indepotent +- name: OCI | Upload bootstrap.ign + oracle.oci.oci_object_storage_object: + namespace_name: "{{ _objns.namespace }}" + bucket_name: "{{ bootstrap_bucket }}" + object_name: "/bootstrap.ign" + src: "{{ config_install_dir + '/' + bootstrap_src_ign }}" + force: false + register: _upload + +- name: OCI | Create expiration timestamp + ansible.builtin.command: "date +'%Y-%m-%dT%H:%M:%S%z' -d '+1 hour'" + register: _cmd + changed_when: false + +- name: OCI | Create preauthenticated_request + oracle.oci.oci_object_storage_preauthenticated_request: + name: par-bootstrap + access_type: ObjectRead + time_expires: "{{ _cmd.stdout }}" + namespace_name: "{{ _objns.namespace }}" + bucket_name: "{{ bootstrap_bucket }}" + object_name: "/bootstrap.ign" + register: _objpreauth + #when: _upload.changed + +- name: OCI | Show existing URLs + oracle.oci.oci_object_storage_preauthenticated_request_facts: + namespace_name: "{{ _objns.namespace }}" + bucket_name: "{{ bootstrap_bucket }}" + register: _pars + +- name: OCI | Create Signed URL to bootstrap_bucket_signed_url + ansible.builtin.set_fact: + bootstrap_bucket_signed_url: "https://objectstorage.{{ config_cluster_region }}.oraclecloud.com{{ _objpreauth.preauthenticated_request.access_uri }}" diff --git a/roles/clients/tasks/main.yaml b/roles/clients/tasks/main.yaml index 136345f..5498dde 100644 --- a/roles/clients/tasks/main.yaml +++ b/roles/clients/tasks/main.yaml @@ -193,4 +193,4 @@ msg: - "binary=[{{ collection_bin_dir }}/{{ item.src }}]" - "link=[{{ collection_bin_dir }}/{{ item.link }}]" - with_items: "{{ _clients_map }}" \ No newline at end of file + with_items: "{{ _clients_map }}" diff --git a/roles/cloud_compute b/roles/cloud_compute index 5a0ea8a..2c60e24 160000 --- a/roles/cloud_compute +++ b/roles/cloud_compute @@ -1 +1 @@ -Subproject commit 5a0ea8a31f4ac8fe82e628fd6af8ef492ccee451 +Subproject commit 2c60e240bd23a3f5612c08a02dd4edaae1fba002 diff --git a/roles/cloud_dns b/roles/cloud_dns index c73d41e..7963424 160000 --- a/roles/cloud_dns +++ b/roles/cloud_dns @@ -1 +1 @@ -Subproject commit c73d41ec4c4cdbccf9aa7626420ada0ed5eb2215 +Subproject commit 7963424148b12d994106d3eb957cddee49649a71 diff --git a/roles/cloud_load_balancer b/roles/cloud_load_balancer index 7c521f0..2de0b77 160000 --- a/roles/cloud_load_balancer +++ b/roles/cloud_load_balancer @@ -1 +1 @@ -Subproject commit 7c521f0960eaa3a62d77237cb00fa1e58a2a8f21 +Subproject commit 2de0b77004652d7be5a2bbbcbd879f11c4c1b957 diff --git a/roles/cloud_network b/roles/cloud_network index 8e745fc..c1bc9ed 160000 --- a/roles/cloud_network +++ b/roles/cloud_network @@ -1 +1 @@ -Subproject commit 8e745fcdf35aa9fcb3bd64ad1e7317f805a4326f +Subproject commit c1bc9ed40d2707057a9809cded3fd2c7cf8c670d diff --git a/roles/config/defaults/main.yaml b/roles/config/defaults/main.yaml index 92b3c56..f55270b 100644 --- a/roles/config/defaults/main.yaml +++ b/roles/config/defaults/main.yaml @@ -8,6 +8,7 @@ config_default_hyperthreading: Enabled config_valid_providers: - aws + - oci # Compute config_compute_replicas: 3 diff --git a/roles/config/tasks/create-coreostream.yml b/roles/config/tasks/create-coreostream.yml new file mode 100644 index 0000000..ee8c009 --- /dev/null +++ b/roles/config/tasks/create-coreostream.yml @@ -0,0 +1,9 @@ +--- + +- name: Create | Create CoreOS Stream JSON + when: not(_coreosstream.stat.exists) + ansible.builtin.shell: | + {{ bin_openshift_install }} coreos print-stream-json \ + > {{ config_install_dir }}/coreos-stream.json + environment: "{{ config_installer_environment | d(omit) }}" + diff --git a/roles/config/tasks/create.yaml b/roles/config/tasks/create.yaml index 90be653..19a04e4 100644 --- a/roles/config/tasks/create.yaml +++ b/roles/config/tasks/create.yaml @@ -3,4 +3,4 @@ # - create-config # - create-manifests # - patch-manifests -# - create-ignitions \ No newline at end of file +# - create-ignitions diff --git a/roles/config/tasks/patches-manifests/deploy-oci-ccm.yaml b/roles/config/tasks/patches-manifests/deploy-oci-ccm.yaml new file mode 100644 index 0000000..bcff630 --- /dev/null +++ b/roles/config/tasks/patches-manifests/deploy-oci-ccm.yaml @@ -0,0 +1,56 @@ +--- +- name: Patch | OCI | CCM | Set namespace oci_ccm_namespace + ansible.builtin.set_fact: + # default provided by repo is kube-system + oci_ccm_namespace: oci-cloud-controller-manager + when: oci_ccm_namespace is not defined + +- name: Patch | OCI | CCM | Create Namespace + ansible.builtin.template: + src: patches/oci/oci-ccm-00-namespace.yaml.j2 + dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-00-namespace.yaml" + mode: 0644 + +- name: Patch | OCI | CCM | Set subnet ID + ansible.builtin.set_fact: + _lb_subnet1: "{{ sb.state.id }}" + loop: "{{ (cluster_state.networks | first).subnets }}" + loop_control: + loop_var: sb + when: sb.public + +- name: Patch | OCI | CCM | Load OCI Secret data + ansible.builtin.set_fact: + oci_ccm_secret_data: "{{ lookup('template', 'patches/oci/oci-ccm-01-secret-data.yaml.j2') | from_yaml }}" + +- name: Patch | OCI | CCM | Create Secret + ansible.builtin.template: + src: patches/oci/oci-ccm-01-secret.yaml.j2 + dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-01-secret.yaml" + mode: 0644 + vars: + oci_compartment_id: oci_compartment_id + +- name: Patch | OCI | CCM | Create RBAC SA + ansible.builtin.template: + src: patches/oci/oci-ccm-02-rbac-sa.yaml.j2 + dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-02-rbac-sa.yaml" + mode: 0644 + +- name: Patch | OCI | CCM | Create RBAC CR + ansible.builtin.template: + src: patches/oci/oci-ccm-03-rbac-cr.yaml.j2 + dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-03-rbac-cr.yaml" + mode: 0644 + +- name: Patch | OCI | CCM | Create RBAC CRB + ansible.builtin.template: + src: patches/oci/oci-ccm-04-rbac-crb.yaml.j2 + dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-04-rbac-crb.yaml" + mode: 0644 + +- name: Patch | OCI | CCM | Create DaemonSet + ansible.builtin.template: + src: patches/oci/oci-ccm-05-daemonset.yaml.j2 + dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-05-daemonset.yaml" + mode: 0644 diff --git a/roles/config/tasks/patches-manifests/deploy-oci-csi.yaml b/roles/config/tasks/patches-manifests/deploy-oci-csi.yaml new file mode 100644 index 0000000..6ae1a8e --- /dev/null +++ b/roles/config/tasks/patches-manifests/deploy-oci-csi.yaml @@ -0,0 +1,42 @@ +--- +- name: Patch | OCI | CCM | Set namespace oci_ccm_namespace + ansible.builtin.set_fact: + # default provided by repo is kube-system + oci_csi_namespace: oci-csi + when: oci_csi_namespace is not defined + +- name: Patch | OCI | CSI | Load OCI Secret data + ansible.builtin.set_fact: + oci_ccm_secret_data: "{{ lookup('template', 'patches/oci/oci-ccm-01-secret-data.yaml.j2') | from_yaml }}" + +- name: Patch | OCI | CSI | Create Manifests to install dir manifests/ + ansible.builtin.template: + src: "patches/oci/{{ manifest }}.j2" + dest: "{{ config_install_dir }}/manifests/{{ manifest }}" + mode: 0644 + loop_control: + loop_var: manifest + loop: + - oci-csi-00-namespace.yaml + - oci-csi-01-secret.yaml + - oci-csi-02-node-rbac-00-sa.yaml + - oci-csi-02-node-rbac-01-cr.yaml + - oci-csi-02-node-rbac-02-crb.yaml + - oci-csi-03-controller-driver.yaml + - oci-csi-04-node-driver-00-csidriver-fss.yaml + - oci-csi-04-node-driver-01-csidriver-bv.yaml + - oci-csi-04-node-driver-02-cm-iscsi.yaml + - oci-csi-04-node-driver-03-cm-fss.yaml + - oci-csi-04-node-driver-04-daemonset.yaml + - oci-csi-05-storage-class-00-bv.yaml + - oci-csi-05-storage-class-01-bv-enc.yaml + +- name: Patch | OCI | CSI | Create MachineConfig iscsid.service + ansible.builtin.template: + src: patches/mc-iscsid-service.yaml.j2 + dest: "{{ config_install_dir }}/openshift//99_openshift-machineconfig_99-{{ machine_role }}-iscsid.yaml" + loop_control: + loop_var: machine_role + loop: + - master + - worker \ No newline at end of file diff --git a/roles/config/tasks/patches-manifests/line_regex_patch.yaml b/roles/config/tasks/patches-manifests/line_regex_patch.yaml new file mode 100644 index 0000000..b8b63da --- /dev/null +++ b/roles/config/tasks/patches-manifests/line_regex_patch.yaml @@ -0,0 +1,10 @@ +--- +- name: Config | Patch manifests | Line regex + ansible.builtin.lineinfile: + path: "{{ config_install_dir }}{{ patch_spec.manifest }}" + backrefs: true + regexp: "{{ patch_spec.regexp }}" + line: "{{ patch_spec.line }}" + loop: "{{ cfg_patch_line_regex_patch_specs }}" + loop_control: + loop_var: patch_spec diff --git a/roles/config/tasks/patches-manifests/mc-kubelet-env-workaround.yaml b/roles/config/tasks/patches-manifests/mc-kubelet-env-workaround.yaml new file mode 100644 index 0000000..f46612e --- /dev/null +++ b/roles/config/tasks/patches-manifests/mc-kubelet-env-workaround.yaml @@ -0,0 +1,16 @@ +--- +# NOTE: there is not guarantee that it would work. +# The Platform=External should have precedence before testing this approach. + +# Requires to cfg_patch_kubelet_env_workaround_content +## Each line should have the script generating the data to append to the +## kubelet workaround file. +- name: Crete kubelet config + ansible.builtin.template: + src: patches/mc-kubelet-env.yaml.j2 + dest: "{{ config_install_dir }}/openshift/99_openshift-machineconfig_00-{{ machine_role }}-kubelet-env-wa.yaml" + loop_control: + loop_var: machine_role + loop: + - master + - worker diff --git a/roles/config/tasks/patches-manifests/mc-kubelet-providerid.yaml b/roles/config/tasks/patches-manifests/mc-kubelet-providerid.yaml new file mode 100644 index 0000000..1b13826 --- /dev/null +++ b/roles/config/tasks/patches-manifests/mc-kubelet-providerid.yaml @@ -0,0 +1,40 @@ +--- +# NOTE: there is not guarantee that it would work. +# The Platform=External should have precedence before testing this approach. + +- name: Patch | mc-kubelet-providerid | Check requirements + ansible.builtin.assert: + that: + - cfg_patch_kubelet_providerid_script is defined + fail_msg: "'cfg_patch_kubelet_providerid_script' is required to use patch 'mc-kubelet-providerid'" + +- name: Patch | mc-kubelet-providerid | Set tmp dir + ansible.builtin.set_fact: + cluster_tmp_dir: "{{ config_install_dir }}/.tmp" + +- name: Patch | mc-kubelet-providerid | ensure tmp dir + ansible.builtin.file: + dest: "{{ cluster_tmp_dir }}" + state: directory + +- name: Patch | mc-kubelet-providerid | Create kubelet providerID + ansible.builtin.template: + src: patches/mc-kubelet-providerid.bu.j2 + dest: "{{ cluster_tmp_dir }}/99_openshift-machineconfig_00-{{ machine_role }}-kubelet-providerid.bu" + loop_control: + loop_var: machine_role + loop: + - master + - worker + +- name: Patch | mc-kubelet-providerid | Render butane config + ansible.builtin.shell: | + {{ bin_butane }} \ + {{ cluster_tmp_dir }}/99_openshift-machineconfig_00-{{ machine_role }}-kubelet-providerid.bu \ + -o {{ config_install_dir }}/openshift/99_openshift-machineconfig_00-{{ machine_role }}-kubelet-providerid.yaml + loop_control: + loop_var: machine_role + loop: + - master + - worker + diff --git a/roles/config/tasks/patches-manifests/yaml_patch.yaml b/roles/config/tasks/patches-manifests/yaml_patch.yaml new file mode 100644 index 0000000..a9e99d2 --- /dev/null +++ b/roles/config/tasks/patches-manifests/yaml_patch.yaml @@ -0,0 +1,6 @@ +--- + +- ansible.builtin.include_tasks: ./yaml_patch_run.yaml + loop: "{{ cfg_patch_yaml_patch_specs }}" + loop_control: + loop_var: patch_spec diff --git a/roles/config/tasks/patches-manifests/yaml_patch_run.yaml b/roles/config/tasks/patches-manifests/yaml_patch_run.yaml new file mode 100644 index 0000000..7db7321 --- /dev/null +++ b/roles/config/tasks/patches-manifests/yaml_patch_run.yaml @@ -0,0 +1,18 @@ +--- +# generic update executor + +- name: patch | reading file {{ config_install_dir + patch_spec.manifest }} + set_fact: + patch_file: "{{ lookup('file', config_install_dir + patch_spec.manifest) | from_yaml }}" + +- debug: var=patch_file +- debug: var=patch_spec.patch + +- name: patch | patching content + set_fact: + new_content: "{{ patch_file | combine(patch_spec.patch|from_json, recursive=True) }}" + +- name: patch | saving file {{ config_install_dir + patch_spec.manifest }} + copy: + dest: "{{ config_install_dir + patch_spec.manifest }}" + content: "{{ new_content | to_nice_yaml(indent=2) }}" diff --git a/roles/config/tasks/save-state.yaml b/roles/config/tasks/save-state.yaml index 93987b7..7124c8b 100644 --- a/roles/config/tasks/save-state.yaml +++ b/roles/config/tasks/save-state.yaml @@ -5,4 +5,4 @@ dest: "{{ config_install_dir + '/cluster_state.json' }}" content: "{{ cluster_state }}" mode: 0644 - changed_when: false + #changed_when: false diff --git a/roles/config/templates/patches/mc-iscsid-service.yaml.j2 b/roles/config/templates/patches/mc-iscsid-service.yaml.j2 new file mode 100644 index 0000000..f454bf6 --- /dev/null +++ b/roles/config/templates/patches/mc-iscsid-service.yaml.j2 @@ -0,0 +1,14 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: {{ machine_role }} + name: 99-{{ machine_role }}-iscsid +spec: + config: + ignition: + version: 3.1.0 + systemd: + units: + - enabled: true + name: iscsid.service \ No newline at end of file diff --git a/roles/config/templates/patches/mc-kubelet-env.yaml.j2 b/roles/config/templates/patches/mc-kubelet-env.yaml.j2 new file mode 100644 index 0000000..b85de19 --- /dev/null +++ b/roles/config/templates/patches/mc-kubelet-env.yaml.j2 @@ -0,0 +1,38 @@ +# https://github.com/openshift/machine-config-operator/blob/master/templates/common/aws/files/usr-local-bin-aws-kubelet-providerid.yaml +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 00-{{ machine_role }}-kubelet-env +spec: + config: + ignition: + version: 3.1.0 + systemd: + units: + - name: kubelet-env.service + enabled: false + contents: | + [Unit] + Description=Fetch kubelet environments from Metadata + # Wait for NetworkManager to report it's online + #Wants=network-online.target + #Requires=crio.service kubelet-auto-node-size.service + #After=network-online.target + #After=ostree-finalize-staged.service + After=NetworkManager-wait-online.service + Before=kubelet.service + [Service] + User=root + Group=root + ExecStart=/opt/libexec/kubelet-env-workaround.sh + Type=oneshot + [Install] + WantedBy=network-online.target + storage: + files: + - mode: 0755 + path: "/opt/libexec/kubelet-env-workaround.sh" + contents: + source: data:text/plain;charset=utf-8;base64,{{ lookup('template', './mc-kubelet-env_kubelet-providerID.sh.j2') | b64encode }} diff --git a/roles/config/templates/patches/mc-kubelet-env_kubelet-providerID.sh.j2 b/roles/config/templates/patches/mc-kubelet-env_kubelet-providerID.sh.j2 new file mode 100644 index 0000000..a3ea527 --- /dev/null +++ b/roles/config/templates/patches/mc-kubelet-env_kubelet-providerID.sh.j2 @@ -0,0 +1,9 @@ +#!/bin/bash + +{{ cfg_patch_kubelet_env_workaround_content }} + +echo "#> Setting permissions 0644 for /etc/kubernetes/kubelet-workaround" +sudo chmod 0755 /etc/kubernetes/kubelet-workaround + +echo "#> Checking value of /etc/kubernetes/kubelet-workaround" +cat /etc/kubernetes/kubelet-workaround diff --git a/roles/config/templates/patches/mc-kubelet-providerid.bu.j2 b/roles/config/templates/patches/mc-kubelet-providerid.bu.j2 new file mode 100644 index 0000000..b276f21 --- /dev/null +++ b/roles/config/templates/patches/mc-kubelet-providerid.bu.j2 @@ -0,0 +1,51 @@ +# https://github.com/openshift/machine-config-operator/blob/master/templates/common/aws/files/usr-local-bin-aws-kubelet-providerid.yaml +variant: openshift +version: 4.12.0 +metadata: + name: 00-{{ machine_role }}-kubelet-providerid + labels: + machineconfiguration.openshift.io/role: {{ machine_role }} +storage: + files: + - mode: 0755 + path: "/usr/local/bin/kubelet-providerid" + contents: + inline: | + #!/bin/bash + set -e -o pipefail + NODECONF=/etc/systemd/system/kubelet.service.d/20-providerid.conf + if [ -e "${NODECONF}" ]; then + echo "Not replacing existing ${NODECONF}" + exit 0 + fi + + {{ cfg_patch_kubelet_providerid_script }} + + cat > "${NODECONF}" <> /data/out.txt; sleep 5; done"] + volumeMounts: + - name: persistent-storage + mountPath: /data + volumes: + - name: persistent-storage + persistentVolumeClaim: + claimName: oci-bv-claim diff --git a/roles/config/templates/patches/oci/oci-sample-lb-00-deployment.yaml.j2 b/roles/config/templates/patches/oci/oci-sample-lb-00-deployment.yaml.j2 new file mode 100644 index 0000000..69283de --- /dev/null +++ b/roles/config/templates/patches/oci/oci-sample-lb-00-deployment.yaml.j2 @@ -0,0 +1,21 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + namespace: {{ oci_samples_namespace | d('oci-samples') }} +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 diff --git a/roles/config/templates/patches/oci/oci-sample-lb-01-service.yaml.j2 b/roles/config/templates/patches/oci/oci-sample-lb-01-service.yaml.j2 new file mode 100644 index 0000000..c717141 --- /dev/null +++ b/roles/config/templates/patches/oci/oci-sample-lb-01-service.yaml.j2 @@ -0,0 +1,16 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: nginx-service + namespace: {{ oci_samples_namespace | d('oci-samples') }} + labels: + app: nginx +spec: + selector: + app: nginx + type: LoadBalancer + ports: + - name: http + port: 81 + targetPort: 80 diff --git a/roles/config/templates/patches/oci/oci-samples-namespace.j2 b/roles/config/templates/patches/oci/oci-samples-namespace.j2 new file mode 100644 index 0000000..cbceb1d --- /dev/null +++ b/roles/config/templates/patches/oci/oci-samples-namespace.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ oci_samples_namespace | d('oci-samples') }} + labels: + "pod-security.kubernetes.io/enforce": "privileged" + "pod-security.kubernetes.io/audit": "privileged" + "pod-security.kubernetes.io/warn": "privileged" + "pod-security.kubernetes.io/enforce-version": "v1.24" + "security.openshift.io/scc.podSecurityLabelSync": "false" + "openshift.io/run-level": "0" diff --git a/roles/destroy/tasks/oci.yaml b/roles/destroy/tasks/oci.yaml new file mode 100644 index 0000000..3376a85 --- /dev/null +++ b/roles/destroy/tasks/oci.yaml @@ -0,0 +1,27 @@ +--- +- name: OCI | Load Balancer + ansible.builtin.include_tasks: "{{ provider }}/loadbalancer.yaml" + tags: loadbalancer + +- name: OCI | Compute + ansible.builtin.include_tasks: "{{ provider }}/compute.yaml" + tags: compute + +- name: OCI | IAM + ansible.builtin.include_tasks: "{{ provider }}/iam.yaml" + tags: iam + +- name: OCI | DNS + ansible.builtin.include_tasks: "{{ provider }}/dns.yaml" + tags: dns + +- name: OCI | Network + ansible.builtin.include_tasks: "{{ provider }}/network.yaml" + tags: network + loop: "{{ cloud_networks }}" + loop_control: + loop_var: vcn + +- name: OCI | Bucket + ansible.builtin.include_tasks: "{{ provider }}/bucket.yaml" + tags: bucket diff --git a/roles/destroy/tasks/oci/bucket.yaml b/roles/destroy/tasks/oci/bucket.yaml new file mode 100644 index 0000000..e566603 --- /dev/null +++ b/roles/destroy/tasks/oci/bucket.yaml @@ -0,0 +1,14 @@ +--- +- name: OCI | Bucket | Get the namespace + oracle.oci.oci_object_storage_namespace_facts: + compartment_id: "{{ oci_compartment_id }}" + register: _objns + +- name: OCI | Bucket | Delete bootstrap + oracle.oci.oci_object_storage_bucket: + state: absent + compartment_id: "{{ oci_compartment_id }}" + name: "{{ cluster_state.infra_id }}-infra" + namespace_name: "{{ _objns.namespace }}" + force: yes + diff --git a/roles/destroy/tasks/oci/compute.yaml b/roles/destroy/tasks/oci/compute.yaml new file mode 100644 index 0000000..cafbdef --- /dev/null +++ b/roles/destroy/tasks/oci/compute.yaml @@ -0,0 +1,37 @@ +--- +- name: OCI | Compute | Discovery Instance ID + tags: compute + loop_control: + loop_var: instance + loop: "{{ okd_cluster_destroy_instances }}" + register: _instances + oracle.oci.oci_compute_instance_facts: + compartment_id: "{{ okd_cluster_destroy_instances_compartment_id }}" + display_name: "{{ instance.name }}" + +- name: OCI | LB | Show Delete + tags: compute + loop_control: + loop_var: results + loop: "{{ _instances.results }}" + when: + - _instances.results | length > 0 + - results.instances is defined and results.instances | length > 0 + debug: + msg: "Deleting Instance: {{ results.instances[0].display_name }}({{ results.instances[0].id }})" + +- name: OCI | Compute | Delete instance + tags: compute + loop_control: + loop_var: inst + loop: "{{ _instances.results }}" + when: + - _instances.results | length > 0 + - inst.instances is defined and inst.instances | length > 0 + oracle.oci.oci_compute_instance: + state: absent + compartment_id: "{{ okd_cluster_destroy_instances_compartment_id }}" + id: "{{ inst.instances[0].id }}" + preserve_boot_volume: false + wait: no + diff --git a/roles/destroy/tasks/oci/dns.yaml b/roles/destroy/tasks/oci/dns.yaml new file mode 100644 index 0000000..162d979 --- /dev/null +++ b/roles/destroy/tasks/oci/dns.yaml @@ -0,0 +1,12 @@ +--- +- name: OCI | DNS | Remove records + oracle.oci.oci_dns_zone_records: + compartment_id: "{{ okd_cluster_destroy_dns_compartment_id }}" + zone_name_or_id: "{{ okd_cluster_destroy_dns_records.zone_name_or_id }}" + patch_items: "{{ okd_cluster_destroy_dns_records.patch_items_spec }}" + # - operation: REMOVE + # domain: "{{ rr.domain }}" + # loop: "{{ okd_cluster_destroy_dns_records }}" + # loop_control: + # loop_var: rr + # register: _rr diff --git a/roles/destroy/tasks/oci/iam.yaml b/roles/destroy/tasks/oci/iam.yaml new file mode 100644 index 0000000..02eb760 --- /dev/null +++ b/roles/destroy/tasks/oci/iam.yaml @@ -0,0 +1,2 @@ +--- +# placeholder diff --git a/roles/destroy/tasks/oci/loadbalancer.yaml b/roles/destroy/tasks/oci/loadbalancer.yaml new file mode 100644 index 0000000..0c07f9c --- /dev/null +++ b/roles/destroy/tasks/oci/loadbalancer.yaml @@ -0,0 +1,37 @@ +--- +- name: OCI | LB | Discovery IDs + loop: "{{ cloud_loadbalancers }}" + loop_control: + loop_var: lb + register: _lb_out + oracle.oci.oci_network_load_balancer_facts: + compartment_id: "{{ lb.spec.compartment_id }}" + display_name: "{{ lb.name }}" + +# - debug: var=_lb_out + +- name: OCI | LB | Show Delete + loop: "{{ _lb_out.results }}" + loop_control: + loop_var: lb + register: _lb_del_out + debug: + msg: "Deleting Load Balancer ID: {{ lb.network_load_balancers[0].id }}" + when: + - debug | d(false) + - _lb_out.results is defined and _lb_out.results | length > 0 + check_mode: no + +- name: OCI | LB | Delete + loop: "{{ _lb_out.results }}" + loop_control: + loop_var: lb + register: _lb_del_out + oracle.oci.oci_network_load_balancer: + state: absent + id: "{{ lb.network_load_balancers[0].id }}" + when: + - _lb_out.results is defined and _lb_out.results | length > 0 + - lb.network_load_balancers is defined and lb.network_load_balancers | length > 0 + +# - debug: var=_lb_del_out diff --git a/roles/destroy/tasks/oci/network.yaml b/roles/destroy/tasks/oci/network.yaml new file mode 100644 index 0000000..eb13539 --- /dev/null +++ b/roles/destroy/tasks/oci/network.yaml @@ -0,0 +1,78 @@ +--- +- name: OCI | Network | Subnet | Delete + oracle.oci.oci_network_subnet: + state: absent + compartment_id: "{{ vcn.compartment_id }}" + display_name: "{{ subnet.spec.display_name }}" + environment: + OCI_USE_NAME_AS_IDENTIFIER: true + loop: "{{ vcn.subnets }}" + loop_control: + loop_var: subnet + register: _del_subnet + until: "_del_subnet is not failed" + retries: 5 + delay: 5 + +- name: OCI | Network | Route Table Public | Delete + oracle.oci.oci_network_route_table: + state: absent + compartment_id: "{{ vcn.compartment_id }}" + display_name: "{{ vcn.route_table_public.spec.display_name }}" + environment: + OCI_USE_NAME_AS_IDENTIFIER: true + +- name: OCI | Network | Route Table Private | Delete + oracle.oci.oci_network_route_table: + state: absent + compartment_id: "{{ vcn.compartment_id }}" + display_name: "{{ vcn.route_table_private.spec.display_name }}" + environment: + OCI_USE_NAME_AS_IDENTIFIER: true + +- name: OCI | Network | NatGW + oracle.oci.oci_network_nat_gateway: + state: absent + compartment_id: "{{ vcn.compartment_id }}" + display_name: "{{ vcn.nat_gateway.spec.display_name }}" + environment: + OCI_USE_NAME_AS_IDENTIFIER: true + +- name: OCI | VCN | IGW | Delete + oracle.oci.oci_network_internet_gateway: + state: absent + compartment_id: "{{ vcn.compartment_id }}" + display_name: "{{ vcn.internet_gateway.spec.display_name }}" + environment: + OCI_USE_NAME_AS_IDENTIFIER: true + +- name: OCI | Network | NSG | Delete + oracle.oci.oci_network_security_group: + state: absent + compartment_id: "{{ vcn.compartment_id }}" + display_name: "{{ nsg.spec.display_name }}" + environment: + OCI_USE_NAME_AS_IDENTIFIER: true + loop: "{{ vcn.network_security_groups }}" + loop_control: + loop_var: nsg + +- name: OCI | Network | SecList | Delete + oracle.oci.oci_network_security_list: + state: absent + compartment_id: "{{ vcn.compartment_id }}" + display_name: "{{ seclist.spec.display_name }}" + environment: + OCI_USE_NAME_AS_IDENTIFIER: true + loop: "{{ vcn.security_lists }}" + loop_control: + loop_var: seclist + +- name: OCI | Network | VCN | Delete + oracle.oci.oci_network_vcn: + state: absent + compartment_id: "{{ vcn.compartment_id }}" + display_name: "{{ vcn.name }}" + environment: + OCI_USE_NAME_AS_IDENTIFIER: true + diff --git a/roles/os_mirror/tasks/oci.yaml b/roles/os_mirror/tasks/oci.yaml new file mode 100644 index 0000000..f87cd57 --- /dev/null +++ b/roles/os_mirror/tasks/oci.yaml @@ -0,0 +1,41 @@ +--- + +- name: OCI | Get the namespace + oracle.oci.oci_object_storage_namespace_facts: + compartment_id: "{{ os_mirror_to_oci.compartment_id }}" + register: _objns + +- name: OCI | Create bucket + oracle.oci.oci_object_storage_bucket: + state: present + compartment_id: "{{ os_mirror_to_oci.compartment_id }}" + name: "{{ os_mirror_to_oci.bucket }}" + namespace_name: "{{ _objns.namespace }}" + +- name: OCI | Show Summary of OS Mirroring + debug: + msg: + - "Bucket/image object: {{ os_mirror_to_oci.bucket }}/{{ cluster_state.compute.image_id }}" + - "Image name: {{ cluster_state.compute.image_id }}" + when: not(_st_image.stat.exists) + +- name: OCI | Upload image to bucket + oracle.oci.oci_object_storage_object: + namespace_name: "{{ _objns.namespace }}" + bucket_name: "{{ os_mirror_to_oci.bucket }}" + object_name: "{{ cluster_state.compute.image_id }}" + src: "{{ collection_bin_dir + '/' + cluster_state.compute.image_id }}" + force: false + register: _upload + +- name: OCI | Creating Custom Image + oracle.oci.oci_compute_image: + compartment_id: "{{ os_mirror_to_oci.compartment_id }}" + image_source_details: + bucket_name: "{{ os_mirror_to_oci.bucket }}" + namespace_name: "{{ _objns.namespace }}" + object_name: "{{ cluster_state.compute.image_id }}" + source_type: objectStorageTuple + source_image_type: "{{ os_mirror_to_oci.image_type }}" + launch_mode: PARAVIRTUALIZED + display_name: "{{ cluster_state.compute.image_id }}"