Azure UserProvisionedDNS: Update bootstrap, master and worker ignition files

1. Edit bootstrap, master and worker ignition files after they have
been created using EditIgnition() which was created as a common method
for all platforms supporting this feature
2. Provide EditIgnition() with the api and api-int LB IPs for Azure
3. Stop creating DNS entries for Azure when userProvisionedDNS is
enabled

Author: sadasu

pkg/infrastructure/azure/azure.go

@@ -31,6 +31,7 @@ import (
 	"github.com/openshift/installer/pkg/rhcos"
 	"github.com/openshift/installer/pkg/types"
 	aztypes "github.com/openshift/installer/pkg/types/azure"
+	"github.com/openshift/installer/pkg/types/dns"
 )
 
 const (
@@ -64,6 +65,7 @@ type Provider struct {
 	Tags                  map[string]*string
 	clientOptions         *arm.ClientOptions
 	computeClientOptions  *arm.ClientOptions
+	publicLBIP            string
 }
 
 var _ clusterapi.InfraReadyProvider = (*Provider)(nil)
@@ -471,6 +473,7 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 		lbBaps = loadBalancer.Properties.BackendAddressPools
 		extLBFQDN = *publicIP.Properties.DNSSettings.Fqdn
 		pubIPAddress = *publicIP.Properties.IPAddress
+		p.publicLBIP = pubIPAddress
 	}
 
 	// Save context for other hooks
@@ -483,8 +486,10 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 	p.NetworkClientFactory = networkClientFactory
 	p.lbBackendAddressPools = lbBaps
 
-	if err := createDNSEntries(ctx, in, extLBFQDN, pubIPAddress, resourceGroupName, p.clientOptions); err != nil {
-		return fmt.Errorf("error creating DNS records: %w", err)
+	if in.InstallConfig.Config.Azure.UserProvisionedDNS != dns.UserProvisionedDNSEnabled {
+		if err := createDNSEntries(ctx, in, extLBFQDN, pubIPAddress, resourceGroupName, p.clientOptions); err != nil {
+			return fmt.Errorf("error creating DNS records: %w", err)
+		}
 	}
 
 	return nil
@@ -714,7 +719,6 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 		return nil, fmt.Errorf("failed to get session: %w", err)
 	}
 
-	bootstrapIgnData := in.BootstrapIgnData
 	subscriptionID := session.Credentials.SubscriptionID
 
 	ignitionContainerName := "ignition"
@@ -739,6 +743,13 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 		logrus.Debugf("BlobIgnitionContainer.ID=%s", *blobIgnitionContainer.ID)
 	}
 
+	// Edit Bootstrap, Master and Worker ignition files if needed. Currently, these
+	// ignition files are updated only when userProvisionedDNS is enabled.
+	ignOutput, err := editIgnition(ctx, in, p.publicLBIP)
+	if err != nil {
+		return nil, fmt.Errorf("failed to edit bootstrap, master or worker ignition: %w", err)
+	}
+
 	sasURL := ""
 
 	if in.InstallConfig.Config.Azure.CustomerManagedKey == nil {
@@ -749,7 +760,7 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 			StorageAccountName: p.StorageAccountName,
 			StorageAccountKeys: p.StorageAccountKeys,
 			ClientOpts:         p.clientOptions,
-			BootstrapIgnData:   bootstrapIgnData,
+			BootstrapIgnData:   ignOutput.UpdatedBootstrapIgn,
 			CloudEnvironment:   in.InstallConfig.Azure.CloudName,
 			ContainerName:      ignitionContainerName,
 			BlobName:           blobName,
@@ -765,7 +776,7 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 		}
 	} else {
 		logrus.Debugf("Creating a Page Blob for ignition shim because Customer Managed Key is provided")
-		lengthBootstrapFile := int64(len(bootstrapIgnData))
+		lengthBootstrapFile := int64(len(ignOutput.UpdatedBootstrapIgn))
 		if lengthBootstrapFile%512 != 0 {
 			lengthBootstrapFile = (((lengthBootstrapFile / 512) + 1) * 512)
 		}
@@ -775,7 +786,7 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 			BlobURL:            blobURL,
 			ImageURL:           "",
 			StorageAccountName: p.StorageAccountName,
-			BootstrapIgnData:   bootstrapIgnData,
+			BootstrapIgnData:   ignOutput.UpdatedBootstrapIgn,
 			ImageLength:        lengthBootstrapFile,
 			StorageAccountKeys: p.StorageAccountKeys,
 			ClientOpts:         p.clientOptions,
@@ -791,7 +802,8 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 
 	ignSecrets := []*corev1.Secret{
 		clusterapi.IgnitionSecret(ignShim, in.InfraID, "bootstrap"),
-		clusterapi.IgnitionSecret(in.MasterIgnData, in.InfraID, "master"),
+		clusterapi.IgnitionSecret(ignOutput.UpdatedMasterIgn, in.InfraID, "master"),
+		clusterapi.IgnitionSecret(ignOutput.UpdatedWorkerIgn, in.InfraID, "worker"),
 	}
 
 	return ignSecrets, nil

pkg/infrastructure/azure/ignition.go

@@ -0,0 +1,54 @@
+package azure
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
+	"github.com/openshift/installer/pkg/infrastructure/clusterapi"
+	"github.com/openshift/installer/pkg/types/azure"
+	"github.com/openshift/installer/pkg/types/dns"
+)
+
+// editIgnition attempts to edit the contents of the bootstrap ignition when the user has selected
+// a custom DNS configuration. Find the public and private load balancer addresses and fill in the
+// infrastructure file within the ignition struct.
+func editIgnition(ctx context.Context, in clusterapi.IgnitionInput, publicIP string) (*clusterapi.IgnitionOutput, error) {
+	// ARO wants the ability to enable custom-dns on day-2. In that case, we might have to
+	// add LB IPs to Infra CR and within bootstrap Ignition even when `UserProvisionedDNS` is
+	// not enabled in install-config.
+	if in.InstallConfig.Config.Azure.UserProvisionedDNS != dns.UserProvisionedDNSEnabled {
+		return &clusterapi.IgnitionOutput{
+			UpdatedBootstrapIgn: in.BootstrapIgnData,
+			UpdatedMasterIgn:    in.MasterIgnData,
+			UpdatedWorkerIgn:    in.WorkerIgnData}, nil
+	}
+	logrus.Debugf("Azure: Editing Ignition files to start in-cluster DNS when UserProvisionedDNS is enabled")
+	azureCluster := &capz.AzureCluster{}
+	key := client.ObjectKey{
+		Name:      in.InfraID,
+		Namespace: capiutils.Namespace,
+	}
+	if err := in.Client.Get(ctx, key, azureCluster); err != nil {
+		return nil, fmt.Errorf("failed to get Azure cluster: %w", err)
+	}
+	if apiLB := azureCluster.Spec.NetworkSpec.APIServerLB; apiLB == nil || len(apiLB.FrontendIPs) == 0 {
+		return nil, fmt.Errorf("failed to get Azure cluster LB frontend IPs")
+	}
+
+	apiIntLBIP := azureCluster.Spec.NetworkSpec.APIServerLB.FrontendIPs[0].PrivateIPAddress
+	if apiIntLBIP == "" {
+		return nil, fmt.Errorf("failed to get Azure cluster API Server Internal LB IP")
+	}
+	apiLBIP := apiIntLBIP
+	// Update API LB IP for public clusters
+	if in.InstallConfig.Config.PublicAPI() && publicIP != "" {
+		apiLBIP = publicIP
+	}
+	logrus.Debugf("Azure: Editing Ignition files with API LB IP: %s and API Int LB IP: %s", apiLBIP, apiIntLBIP)
+	return clusterapi.EditIgnition(in, azure.Name, []string{apiLBIP}, []string{apiIntLBIP})
+}

pkg/infrastructure/clusterapi/ignition.go

@@ -21,6 +21,7 @@ import (
 	"github.com/openshift/installer/pkg/asset/machines"
 	"github.com/openshift/installer/pkg/asset/tls"
 	awstypes "github.com/openshift/installer/pkg/types/aws"
+	azuretypes "github.com/openshift/installer/pkg/types/azure"
 	gcptypes "github.com/openshift/installer/pkg/types/gcp"
 )
 
@@ -154,6 +155,10 @@ func addLoadBalancersToInfra(platform string, config *igntypes.Config, publicLBs
 				if infra.Status.PlatformStatus.AWS.CloudLoadBalancerConfig.DNSType == configv1.ClusterHostedDNSType {
 					infra.Status.PlatformStatus.AWS.CloudLoadBalancerConfig.ClusterHosted = &cloudLBInfo
 				}
+			case azuretypes.Name:
+				if infra.Status.PlatformStatus.Azure.CloudLoadBalancerConfig.DNSType == configv1.ClusterHostedDNSType {
+					infra.Status.PlatformStatus.Azure.CloudLoadBalancerConfig.ClusterHosted = &cloudLBInfo
+				}
 			default:
 				return fmt.Errorf("invalid platform %s", platform)
 			}