From a3f8b455e70198a224e472773e4b7602a3c78de9 Mon Sep 17 00:00:00 2001 From: Kirby Chin <37311900+kabicin@users.noreply.github.com> Date: Mon, 22 Apr 2024 14:38:16 -0400 Subject: [PATCH 1/6] Add AllowAPIServerAccessLabel --- config/manifests/apiServerAccessPatch.yaml | 4 + config/manifests/kustomization.yaml | 5 + config/rbac/role.yaml | 7 ++ .../openlibertyapplication_controller.go | 105 +++++++++++++++++- utils/utils.go | 24 +++- 5 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 config/manifests/apiServerAccessPatch.yaml diff --git a/config/manifests/apiServerAccessPatch.yaml b/config/manifests/apiServerAccessPatch.yaml new file mode 100644 index 000000000..e4bd24511 --- /dev/null +++ b/config/manifests/apiServerAccessPatch.yaml @@ -0,0 +1,4 @@ +- op: add + path: /spec/install/spec/deployments/0/spec/template/metadata/labels- + value: + openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true" \ No newline at end of file diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml index f3df7e4e0..9338c4476 100644 --- a/config/manifests/kustomization.yaml +++ b/config/manifests/kustomization.yaml @@ -23,6 +23,11 @@ patches: kind: ClusterServiceVersion name: open-liberty.v0.0.0 namespace: placeholder +- path: apiServerAccessPatch.yaml + target: + kind: ClusterServiceVersion + name: open-liberty.v0.0.0 + namespace: placeholder # [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. # Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. # These patches remove the unnecessary "cert" volume and its manager container volumeMount. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d5f387c5c..48feec15e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -116,6 +116,13 @@ rules: - list - update - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list - apiGroups: - "" resources: diff --git a/controllers/openlibertyapplication_controller.go b/controllers/openlibertyapplication_controller.go index 0ed98e17c..08d1226d2 100644 --- a/controllers/openlibertyapplication_controller.go +++ b/controllers/openlibertyapplication_controller.go @@ -27,6 +27,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" servingv1 "knative.dev/serving/pkg/apis/serving/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -38,8 +39,9 @@ import ( ) const ( - OperatorName = "open-liberty-operator" - OperatorShortName = "olo" + OperatorName = "open-liberty-operator" + OperatorShortName = "olo" + OperatorAllowAPIServerAccessLabel = "openlibertyapplications.apps.openliberty.io/allow-apiserver-access" ) // ReconcileOpenLiberty reconciles an OpenLibertyApplication object @@ -58,6 +60,7 @@ const applicationFinalizer = "finalizer.openlibertyapplications.apps.openliberty // +kubebuilder:rbac:groups=apps,resources=deployments;statefulsets,verbs=get;list;watch;create;update;delete,namespace=open-liberty-operator // +kubebuilder:rbac:groups=apps,resources=deployments/finalizers;statefulsets,verbs=update,namespace=open-liberty-operator // +kubebuilder:rbac:groups=core,resources=services;secrets;serviceaccounts;configmaps;persistentvolumeclaims,verbs=get;list;watch;create;update;delete,namespace=open-liberty-operator +// +kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list,namespace=open-liberty-operator // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;delete,namespace=open-liberty-operator // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;delete,namespace=open-liberty-operator // +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;delete,namespace=open-liberty-operator @@ -377,6 +380,69 @@ func (r *ReconcileOpenLiberty) Reconcile(ctx context.Context, request ctrl.Reque common.StatusConditionTypeReconciled, instance) } + // Kube API Server NetworkPolicy (based upon impl. by Martin Smithson) + apiServerNetworkPolicy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-egress-dns-and-apiserver-access", + Namespace: instance.Namespace, + }} + err = r.CreateOrUpdate(apiServerNetworkPolicy, instance, func() error { + apiServerNetworkPolicy.Spec.PodSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{ + OperatorAllowAPIServerAccessLabel: "true", + }, + } + apiServerNetworkPolicy.Spec.Egress = make([]networkingv1.NetworkPolicyEgressRule, 0) + + var dnsRule networkingv1.NetworkPolicyEgressRule + var usingPermissiveRule bool + // If allowed, add an Egress rule to access the OpenShift DNS or K8s CoreDNS. Otherwise, use a permissive cluster-wide Egress rule. + if r.IsOpenShift() { + usingPermissiveRule, dnsRule = r.getDNSEgressRule(reqLogger, "dns-default", "openshift-dns") + } else { + usingPermissiveRule, dnsRule = r.getDNSEgressRule(reqLogger, "kube-dns", "kube-system") + } + apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, dnsRule) + + // If allowed, add an Egress rule to access the API server. + // Otherwise, if the OpenShift DNS or K8s CoreDNS Egress rule does not provide permissive cluster-wide access + // and the K8s API server could not be found, use a permissive cluster-wide Egress rule. + if apiServerEndpoints, err := r.getEndpoints("kubernetes", "default"); err == nil { + rule := networkingv1.NetworkPolicyEgressRule{} + // Define the port + port := networkingv1.NetworkPolicyPort{} + port.Protocol = &apiServerEndpoints.Subsets[0].Ports[0].Protocol + var portNumber intstr.IntOrString = intstr.FromInt((int)(apiServerEndpoints.Subsets[0].Ports[0].Port)) + port.Port = &portNumber + rule.Ports = append(rule.Ports, port) + + // Add the endpoint address as ipBlock entries + for _, endpoint := range apiServerEndpoints.Subsets { + for _, address := range endpoint.Addresses { + peer := networkingv1.NetworkPolicyPeer{} + ipBlock := networkingv1.IPBlock{} + ipBlock.CIDR = address.IP + "/32" + + peer.IPBlock = &ipBlock + rule.To = append(rule.To, peer) + } + } + apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, rule) + reqLogger.Info("Found endpoints for kubernetes service in the default namespace") + } else if !usingPermissiveRule { + rule := networkingv1.NetworkPolicyEgressRule{} + apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, rule) + reqLogger.Info("Found endpoints for kubernetes service in the default namespace") + } + apiServerNetworkPolicy.Labels = ba.GetLabels() + apiServerNetworkPolicy.Annotations = oputils.MergeMaps(apiServerNetworkPolicy.Annotations, ba.GetAnnotations()) + apiServerNetworkPolicy.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeEgress} + return nil + }) + if err != nil { + reqLogger.Error(err, "Failed to reconcile API server network policy") + return r.ManageError(err, common.StatusConditionTypeReconciled, instance) + } + networkPolicy := &networkingv1.NetworkPolicy{ObjectMeta: defaultMeta} if np := instance.Spec.NetworkPolicy; np == nil || np != nil && !np.IsDisabled() { err = r.CreateOrUpdate(networkPolicy, instance, func() error { @@ -850,3 +916,38 @@ func (r *ReconcileOpenLiberty) deletePVC(reqLogger logr.Logger, pvcName string, } } } + +func (r *ReconcileOpenLiberty) getEndpoints(serviceName string, namespace string) (*corev1.Endpoints, error) { + endpoints := &corev1.Endpoints{} + if err := r.GetClient().Get(context.TODO(), types.NamespacedName{Name: serviceName, Namespace: namespace}, endpoints); err != nil { + return nil, err + } else { + return endpoints, nil + } +} + +func (r *ReconcileOpenLiberty) getDNSEgressRule(reqLogger logr.Logger, endpointsName string, endpointsNamespace string) (bool, networkingv1.NetworkPolicyEgressRule) { + dnsRule := networkingv1.NetworkPolicyEgressRule{} + if dnsEndpoints, err := r.getEndpoints(endpointsName, endpointsNamespace); err == nil { + if endpointPort := lutils.GetEndpointPortByName(&dnsEndpoints.Subsets[0].Ports, "dns"); endpointPort != nil { + dnsRule.Ports = append(dnsRule.Ports, lutils.CreateNetworkPolicyPortFromEndpointPort(endpointPort)) + } + if endpointPort := lutils.GetEndpointPortByName(&dnsEndpoints.Subsets[0].Ports, "dns-tcp"); endpointPort != nil { + dnsRule.Ports = append(dnsRule.Ports, lutils.CreateNetworkPolicyPortFromEndpointPort(endpointPort)) + } + peer := networkingv1.NetworkPolicyPeer{} + peer.NamespaceSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": endpointsNamespace, + }, + } + dnsRule.To = append(dnsRule.To, peer) + reqLogger.Info("Found endpoints for " + endpointsName + " service in the " + endpointsNamespace + " namespace") + return false, dnsRule + } + // use permissive rule + // egress: + // - {} + reqLogger.Info("Failed to retrieve endpoints for " + endpointsName + " service in the " + endpointsNamespace + " namespace. Using more permissive rule.") + return true, dnsRule +} diff --git a/utils/utils.go b/utils/utils.go index 0cffc5d63..d5dcd630b 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -9,6 +9,8 @@ import ( "strconv" "strings" + networkingv1 "k8s.io/api/networking/v1" + olv1 "github.com/OpenLiberty/open-liberty-operator/api/v1" rcoutils "github.com/application-stacks/runtime-component-operator/utils" routev1 "github.com/openshift/api/route/v1" @@ -18,6 +20,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -609,9 +612,11 @@ func IsLTPAJobConfigurationOutdated(job *v1.Job, appLeaderInstance *olv1.OpenLib return false } -func CustomizeLTPAJob(job *v1.Job, la *olv1.OpenLibertyApplication, ltpaSecretName string, serviceAccountName string, ltpaScriptName string) { +func CustomizeLTPAJob(job *v1.Job, la *olv1.OpenLibertyApplication, ltpaSecretName string, serviceAccountName string, ltpaScriptName string, allowAPIServerAccessLabel string) { encodingType := "aes" // the password encoding type for securityUtility (one of "xor", "aes", or "hash") job.Spec.Template.ObjectMeta.Name = "liberty" + // Enable NetworkPolicy Egress access to Kube API Server + job.Spec.Template.Labels = rcoutils.MergeMaps(job.Spec.Template.Labels, map[string]string{allowAPIServerAccessLabel: "true"}) job.Spec.Template.Spec.Containers = []corev1.Container{ { Name: job.Spec.Template.ObjectMeta.Name, @@ -678,3 +683,20 @@ func GetRequiredLabels(name string, instance string) map[string]string { requiredLabels["app.kubernetes.io/managed-by"] = "open-liberty-operator" return requiredLabels } + +func GetEndpointPortByName(endpointPorts *[]corev1.EndpointPort, name string) *corev1.EndpointPort { + for _, endpointPort := range *endpointPorts { + if endpointPort.Name == name { + return &endpointPort + } + } + return nil +} + +func CreateNetworkPolicyPortFromEndpointPort(endpointPort *corev1.EndpointPort) networkingv1.NetworkPolicyPort { + port := networkingv1.NetworkPolicyPort{} + port.Protocol = &endpointPort.Protocol + var portNumber intstr.IntOrString = intstr.FromInt((int)(endpointPort.Port)) + port.Port = &portNumber + return port +} From a68cb6a5c9d0b37c3a201f1b73c0c344acd3ce78 Mon Sep 17 00:00:00 2001 From: Kirby Chin <37311900+kabicin@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:07:10 -0400 Subject: [PATCH 2/6] Update apiserver patch to use strategic patch --- config/manifests/apiServerAccessPatch.yaml | 4 ---- config/manifests/kustomization.yaml | 6 +----- config/manifests/patches/apiServerAccessPatch.yaml | 14 ++++++++++++++ 3 files changed, 15 insertions(+), 9 deletions(-) delete mode 100644 config/manifests/apiServerAccessPatch.yaml create mode 100644 config/manifests/patches/apiServerAccessPatch.yaml diff --git a/config/manifests/apiServerAccessPatch.yaml b/config/manifests/apiServerAccessPatch.yaml deleted file mode 100644 index e4bd24511..000000000 --- a/config/manifests/apiServerAccessPatch.yaml +++ /dev/null @@ -1,4 +0,0 @@ -- op: add - path: /spec/install/spec/deployments/0/spec/template/metadata/labels- - value: - openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true" \ No newline at end of file diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml index 9338c4476..7721ca90a 100644 --- a/config/manifests/kustomization.yaml +++ b/config/manifests/kustomization.yaml @@ -7,6 +7,7 @@ resources: - ../scorecard patchesStrategicMerge: - patches/csvAnnotations.yaml + - patches/apiServerAccessPatch.yaml patches: - path: serviceBindingPatch.yaml target: @@ -23,11 +24,6 @@ patches: kind: ClusterServiceVersion name: open-liberty.v0.0.0 namespace: placeholder -- path: apiServerAccessPatch.yaml - target: - kind: ClusterServiceVersion - name: open-liberty.v0.0.0 - namespace: placeholder # [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. # Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. # These patches remove the unnecessary "cert" volume and its manager container volumeMount. diff --git a/config/manifests/patches/apiServerAccessPatch.yaml b/config/manifests/patches/apiServerAccessPatch.yaml new file mode 100644 index 000000000..6177fb931 --- /dev/null +++ b/config/manifests/patches/apiServerAccessPatch.yaml @@ -0,0 +1,14 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: open-liberty.v0.0.0 + namespace: placeholder +spec: + install: + spec: + deployments: + - spec: + template: + metadata: + labels: + openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true" \ No newline at end of file From bef850fdb96a9e0f6fdfb29bcaac22b68c9b3ef2 Mon Sep 17 00:00:00 2001 From: Kirby Chin <37311900+kabicin@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:08:00 -0400 Subject: [PATCH 3/6] Add endpoint to roles --- bundle/manifests/open-liberty.clusterserviceversion.yaml | 9 ++++++++- internal/deploy/kubectl/openliberty-app-operator.yaml | 7 +++++++ .../deploy/kubectl/openliberty-app-rbac-watch-all.yaml | 7 +++++++ .../kubectl/openliberty-app-rbac-watch-another.yaml | 7 +++++++ .../deploy/kustomize/daily/base/open-liberty-roles.yaml | 7 +++++++ .../overlays/watch-all-namespaces/cluster-roles.yaml | 7 +++++++ .../olo-watched-ns/watched-roles.yaml | 7 +++++++ 7 files changed, 50 insertions(+), 1 deletion(-) diff --git a/bundle/manifests/open-liberty.clusterserviceversion.yaml b/bundle/manifests/open-liberty.clusterserviceversion.yaml index 3357b2d49..e830d5f73 100644 --- a/bundle/manifests/open-liberty.clusterserviceversion.yaml +++ b/bundle/manifests/open-liberty.clusterserviceversion.yaml @@ -92,7 +92,7 @@ metadata: categories: Application Runtime certified: "true" containerImage: icr.io/appcafe/open-liberty-operator:daily - createdAt: "2024-04-11T14:14:26Z" + createdAt: "2024-04-23T19:07:22Z" description: Deploy and manage containerized Liberty applications olm.skipRange: '>=0.8.0 <1.3.1' operators.openshift.io/infrastructure-features: '["disconnected"]' @@ -1297,6 +1297,13 @@ spec: - list - update - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list - apiGroups: - "" resources: diff --git a/internal/deploy/kubectl/openliberty-app-operator.yaml b/internal/deploy/kubectl/openliberty-app-operator.yaml index 20f39f80a..c96c8c676 100644 --- a/internal/deploy/kubectl/openliberty-app-operator.yaml +++ b/internal/deploy/kubectl/openliberty-app-operator.yaml @@ -155,6 +155,13 @@ rules: - list - update - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list - apiGroups: - "" resources: diff --git a/internal/deploy/kubectl/openliberty-app-rbac-watch-all.yaml b/internal/deploy/kubectl/openliberty-app-rbac-watch-all.yaml index 1d7f48989..3032884b1 100644 --- a/internal/deploy/kubectl/openliberty-app-rbac-watch-all.yaml +++ b/internal/deploy/kubectl/openliberty-app-rbac-watch-all.yaml @@ -147,6 +147,13 @@ rules: - list - update - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list - apiGroups: - "" resources: diff --git a/internal/deploy/kubectl/openliberty-app-rbac-watch-another.yaml b/internal/deploy/kubectl/openliberty-app-rbac-watch-another.yaml index 1ae07af2f..553923cac 100644 --- a/internal/deploy/kubectl/openliberty-app-rbac-watch-another.yaml +++ b/internal/deploy/kubectl/openliberty-app-rbac-watch-another.yaml @@ -149,6 +149,13 @@ rules: - list - update - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list - apiGroups: - "" resources: diff --git a/internal/deploy/kustomize/daily/base/open-liberty-roles.yaml b/internal/deploy/kustomize/daily/base/open-liberty-roles.yaml index 8c12cf99d..3710164b7 100644 --- a/internal/deploy/kustomize/daily/base/open-liberty-roles.yaml +++ b/internal/deploy/kustomize/daily/base/open-liberty-roles.yaml @@ -158,6 +158,13 @@ rules: - list - update - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list - apiGroups: - "" resources: diff --git a/internal/deploy/kustomize/daily/overlays/watch-all-namespaces/cluster-roles.yaml b/internal/deploy/kustomize/daily/overlays/watch-all-namespaces/cluster-roles.yaml index a6c07a6e5..a28726101 100644 --- a/internal/deploy/kustomize/daily/overlays/watch-all-namespaces/cluster-roles.yaml +++ b/internal/deploy/kustomize/daily/overlays/watch-all-namespaces/cluster-roles.yaml @@ -147,6 +147,13 @@ rules: - list - update - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list - apiGroups: - "" resources: diff --git a/internal/deploy/kustomize/daily/overlays/watch-another-namespace/olo-watched-ns/watched-roles.yaml b/internal/deploy/kustomize/daily/overlays/watch-another-namespace/olo-watched-ns/watched-roles.yaml index 32f451cea..7ed50c710 100644 --- a/internal/deploy/kustomize/daily/overlays/watch-another-namespace/olo-watched-ns/watched-roles.yaml +++ b/internal/deploy/kustomize/daily/overlays/watch-another-namespace/olo-watched-ns/watched-roles.yaml @@ -149,6 +149,13 @@ rules: - list - update - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list - apiGroups: - "" resources: From acc195ccdb2799b6d2e7b651de72fabd2f84656f Mon Sep 17 00:00:00 2001 From: Kirby Chin <37311900+kabicin@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:11:02 -0400 Subject: [PATCH 4/6] Add apiserver access to controller-manager --- bundle/manifests/open-liberty.clusterserviceversion.yaml | 3 ++- config/manager/manager.yaml | 1 + internal/deploy/kubectl/openliberty-app-operator.yaml | 1 + .../deploy/kustomize/daily/base/open-liberty-operator.yaml | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bundle/manifests/open-liberty.clusterserviceversion.yaml b/bundle/manifests/open-liberty.clusterserviceversion.yaml index e830d5f73..d79a2b10c 100644 --- a/bundle/manifests/open-liberty.clusterserviceversion.yaml +++ b/bundle/manifests/open-liberty.clusterserviceversion.yaml @@ -92,7 +92,7 @@ metadata: categories: Application Runtime certified: "true" containerImage: icr.io/appcafe/open-liberty-operator:daily - createdAt: "2024-04-23T19:07:22Z" + createdAt: "2024-04-23T19:10:43Z" description: Deploy and manage containerized Liberty applications olm.skipRange: '>=0.8.0 <1.3.1' operators.openshift.io/infrastructure-features: '["disconnected"]' @@ -1092,6 +1092,7 @@ spec: app.kubernetes.io/managed-by: olm app.kubernetes.io/name: open-liberty-operator control-plane: controller-manager + openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true" spec: affinity: nodeAffinity: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 6fd62b776..a56783c4f 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -21,6 +21,7 @@ spec: metadata: labels: control-plane: controller-manager + openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true" annotations: kubectl.kubernetes.io/default-container: manager diff --git a/internal/deploy/kubectl/openliberty-app-operator.yaml b/internal/deploy/kubectl/openliberty-app-operator.yaml index c96c8c676..62ae99c50 100644 --- a/internal/deploy/kubectl/openliberty-app-operator.yaml +++ b/internal/deploy/kubectl/openliberty-app-operator.yaml @@ -304,6 +304,7 @@ spec: app.kubernetes.io/instance: open-liberty-operator app.kubernetes.io/name: open-liberty-operator control-plane: controller-manager + openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true" spec: affinity: nodeAffinity: diff --git a/internal/deploy/kustomize/daily/base/open-liberty-operator.yaml b/internal/deploy/kustomize/daily/base/open-liberty-operator.yaml index af7c9a6fb..e8e1ab034 100644 --- a/internal/deploy/kustomize/daily/base/open-liberty-operator.yaml +++ b/internal/deploy/kustomize/daily/base/open-liberty-operator.yaml @@ -22,6 +22,7 @@ spec: app.kubernetes.io/instance: open-liberty-operator app.kubernetes.io/name: open-liberty-operator control-plane: controller-manager + openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true" spec: affinity: nodeAffinity: From 675aca96923b4141d025f1bd95625d46d0003e7d Mon Sep 17 00:00:00 2001 From: Kirby Chin <37311900+kabicin@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:12:10 -0400 Subject: [PATCH 5/6] Update ltpa_keys_sharing.go --- controllers/ltpa_keys_sharing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/ltpa_keys_sharing.go b/controllers/ltpa_keys_sharing.go index 66732016c..21e0e8ddc 100644 --- a/controllers/ltpa_keys_sharing.go +++ b/controllers/ltpa_keys_sharing.go @@ -231,7 +231,7 @@ func (r *ReconcileOpenLiberty) generateLTPAKeys(instance *olv1.OpenLibertyApplic err = r.GetClient().Get(context.TODO(), types.NamespacedName{Name: generateLTPAKeysJob.Name, Namespace: generateLTPAKeysJob.Namespace}, generateLTPAKeysJob) if err != nil && kerrors.IsNotFound(err) { err = r.CreateOrUpdate(generateLTPAKeysJob, instance, func() error { - lutils.CustomizeLTPAJob(generateLTPAKeysJob, instance, ltpaSecret.Name, ltpaServiceAccountName, ltpaKeysCreationScriptConfigMap.Name) + lutils.CustomizeLTPAJob(generateLTPAKeysJob, instance, ltpaSecret.Name, ltpaServiceAccountName, ltpaKeysCreationScriptConfigMap.Name, OperatorAllowAPIServerAccessLabel) return nil }) if err != nil { From 99f6414bfce86d08e140e367113927958af86847 Mon Sep 17 00:00:00 2001 From: Kirby Chin <37311900+kabicin@users.noreply.github.com> Date: Wed, 24 Apr 2024 11:29:19 -0400 Subject: [PATCH 6/6] Remove additional endpoint checks when DNS is unreachable also, add nil check to dnsEndpoints.Subsets and GetEndpointPortByName --- .../openlibertyapplication_controller.go | 69 ++++++++++--------- utils/utils.go | 8 ++- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/controllers/openlibertyapplication_controller.go b/controllers/openlibertyapplication_controller.go index 08d1226d2..6bf720635 100644 --- a/controllers/openlibertyapplication_controller.go +++ b/controllers/openlibertyapplication_controller.go @@ -403,36 +403,39 @@ func (r *ReconcileOpenLiberty) Reconcile(ctx context.Context, request ctrl.Reque } apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, dnsRule) - // If allowed, add an Egress rule to access the API server. - // Otherwise, if the OpenShift DNS or K8s CoreDNS Egress rule does not provide permissive cluster-wide access - // and the K8s API server could not be found, use a permissive cluster-wide Egress rule. - if apiServerEndpoints, err := r.getEndpoints("kubernetes", "default"); err == nil { - rule := networkingv1.NetworkPolicyEgressRule{} - // Define the port - port := networkingv1.NetworkPolicyPort{} - port.Protocol = &apiServerEndpoints.Subsets[0].Ports[0].Protocol - var portNumber intstr.IntOrString = intstr.FromInt((int)(apiServerEndpoints.Subsets[0].Ports[0].Port)) - port.Port = &portNumber - rule.Ports = append(rule.Ports, port) - - // Add the endpoint address as ipBlock entries - for _, endpoint := range apiServerEndpoints.Subsets { - for _, address := range endpoint.Addresses { - peer := networkingv1.NetworkPolicyPeer{} - ipBlock := networkingv1.IPBlock{} - ipBlock.CIDR = address.IP + "/32" - - peer.IPBlock = &ipBlock - rule.To = append(rule.To, peer) + // If the DNS rule is a specific Egress rule also check if another Egress rule can be created for the API server. + // Otherwise, fallback to a permissive cluster-wide Egress rule. + if !usingPermissiveRule { + if apiServerEndpoints, err := r.getEndpoints("kubernetes", "default"); err == nil { + rule := networkingv1.NetworkPolicyEgressRule{} + // Define the port + port := networkingv1.NetworkPolicyPort{} + port.Protocol = &apiServerEndpoints.Subsets[0].Ports[0].Protocol + var portNumber intstr.IntOrString = intstr.FromInt((int)(apiServerEndpoints.Subsets[0].Ports[0].Port)) + port.Port = &portNumber + rule.Ports = append(rule.Ports, port) + + // Add the endpoint address as ipBlock entries + for _, endpoint := range apiServerEndpoints.Subsets { + for _, address := range endpoint.Addresses { + peer := networkingv1.NetworkPolicyPeer{} + ipBlock := networkingv1.IPBlock{} + ipBlock.CIDR = address.IP + "/32" + + peer.IPBlock = &ipBlock + rule.To = append(rule.To, peer) + } } + apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, rule) + reqLogger.Info("Found endpoints for kubernetes service in the default namespace") + } else { + // The operator couldn't create a rule for the K8s API server so add a permissive Egress rule + rule := networkingv1.NetworkPolicyEgressRule{} + apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, rule) + reqLogger.Info("Found endpoints for kubernetes service in the default namespace") } - apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, rule) - reqLogger.Info("Found endpoints for kubernetes service in the default namespace") - } else if !usingPermissiveRule { - rule := networkingv1.NetworkPolicyEgressRule{} - apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, rule) - reqLogger.Info("Found endpoints for kubernetes service in the default namespace") } + apiServerNetworkPolicy.Labels = ba.GetLabels() apiServerNetworkPolicy.Annotations = oputils.MergeMaps(apiServerNetworkPolicy.Annotations, ba.GetAnnotations()) apiServerNetworkPolicy.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeEgress} @@ -929,11 +932,13 @@ func (r *ReconcileOpenLiberty) getEndpoints(serviceName string, namespace string func (r *ReconcileOpenLiberty) getDNSEgressRule(reqLogger logr.Logger, endpointsName string, endpointsNamespace string) (bool, networkingv1.NetworkPolicyEgressRule) { dnsRule := networkingv1.NetworkPolicyEgressRule{} if dnsEndpoints, err := r.getEndpoints(endpointsName, endpointsNamespace); err == nil { - if endpointPort := lutils.GetEndpointPortByName(&dnsEndpoints.Subsets[0].Ports, "dns"); endpointPort != nil { - dnsRule.Ports = append(dnsRule.Ports, lutils.CreateNetworkPolicyPortFromEndpointPort(endpointPort)) - } - if endpointPort := lutils.GetEndpointPortByName(&dnsEndpoints.Subsets[0].Ports, "dns-tcp"); endpointPort != nil { - dnsRule.Ports = append(dnsRule.Ports, lutils.CreateNetworkPolicyPortFromEndpointPort(endpointPort)) + if len(dnsEndpoints.Subsets) > 0 { + if endpointPort := lutils.GetEndpointPortByName(&dnsEndpoints.Subsets[0].Ports, "dns"); endpointPort != nil { + dnsRule.Ports = append(dnsRule.Ports, lutils.CreateNetworkPolicyPortFromEndpointPort(endpointPort)) + } + if endpointPort := lutils.GetEndpointPortByName(&dnsEndpoints.Subsets[0].Ports, "dns-tcp"); endpointPort != nil { + dnsRule.Ports = append(dnsRule.Ports, lutils.CreateNetworkPolicyPortFromEndpointPort(endpointPort)) + } } peer := networkingv1.NetworkPolicyPeer{} peer.NamespaceSelector = &metav1.LabelSelector{ diff --git a/utils/utils.go b/utils/utils.go index d5dcd630b..7e8cd4abd 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -685,9 +685,11 @@ func GetRequiredLabels(name string, instance string) map[string]string { } func GetEndpointPortByName(endpointPorts *[]corev1.EndpointPort, name string) *corev1.EndpointPort { - for _, endpointPort := range *endpointPorts { - if endpointPort.Name == name { - return &endpointPort + if endpointPorts != nil { + for _, endpointPort := range *endpointPorts { + if endpointPort.Name == name { + return &endpointPort + } } } return nil