Add API server NetworkPolicy to support a potential deny all egress

bundle/manifests/open-liberty.clusterserviceversion.yaml

@@ -92,7 +92,7 @@ metadata:
     categories: Application Runtime
     certified: "true"
     containerImage: icr.io/appcafe/open-liberty-operator:daily
-    createdAt: "2024-04-11T14:14:26Z"
+    createdAt: "2024-04-23T19:10:43Z"
     description: Deploy and manage containerized Liberty applications
     olm.skipRange: '>=0.8.0 <1.3.1'
     operators.openshift.io/infrastructure-features: '["disconnected"]'
@@ -1092,6 +1092,7 @@ spec:
                 app.kubernetes.io/managed-by: olm
                 app.kubernetes.io/name: open-liberty-operator
                 control-plane: controller-manager
+                openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true"
             spec:
               affinity:
                 nodeAffinity:
@@ -1297,6 +1298,13 @@ spec:
           - list
           - update
           - watch
+        - apiGroups:
+          - ""
+          resources:
+          - endpoints
+          verbs:
+          - get
+          - list
         - apiGroups:
           - ""
           resources:

config/manager/manager.yaml

@@ -21,6 +21,7 @@ spec:
     metadata:
       labels:
         control-plane: controller-manager
+        openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true"
       annotations:
         kubectl.kubernetes.io/default-container: manager
 

config/manifests/kustomization.yaml

@@ -7,6 +7,7 @@ resources:
 - ../scorecard
 patchesStrategicMerge:
   - patches/csvAnnotations.yaml
+  - patches/apiServerAccessPatch.yaml
 patches:
 - path: serviceBindingPatch.yaml
   target:

config/manifests/patches/apiServerAccessPatch.yaml

@@ -0,0 +1,14 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: ClusterServiceVersion
+metadata:
+  name: open-liberty.v0.0.0
+  namespace: placeholder
+spec:
+  install:
+    spec:
+      deployments:
+        - spec:
+            template:
+              metadata:
+                labels:
+                  openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true"

config/rbac/role.yaml

@@ -116,6 +116,13 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

controllers/ltpa_keys_sharing.go

@@ -231,7 +231,7 @@ func (r *ReconcileOpenLiberty) generateLTPAKeys(instance *olv1.OpenLibertyApplic
 				err = r.GetClient().Get(context.TODO(), types.NamespacedName{Name: generateLTPAKeysJob.Name, Namespace: generateLTPAKeysJob.Namespace}, generateLTPAKeysJob)
 				if err != nil && kerrors.IsNotFound(err) {
 					err = r.CreateOrUpdate(generateLTPAKeysJob, instance, func() error {
-						lutils.CustomizeLTPAJob(generateLTPAKeysJob, instance, ltpaSecret.Name, ltpaServiceAccountName, ltpaKeysCreationScriptConfigMap.Name)
+						lutils.CustomizeLTPAJob(generateLTPAKeysJob, instance, ltpaSecret.Name, ltpaServiceAccountName, ltpaKeysCreationScriptConfigMap.Name, OperatorAllowAPIServerAccessLabel)
 						return nil
 					})
 					if err != nil {

controllers/openlibertyapplication_controller.go

@@ -27,6 +27,7 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	servingv1 "knative.dev/serving/pkg/apis/serving/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -38,8 +39,9 @@ import (
 )
 
 const (
-	OperatorName      = "open-liberty-operator"
-	OperatorShortName = "olo"
+	OperatorName                      = "open-liberty-operator"
+	OperatorShortName                 = "olo"
+	OperatorAllowAPIServerAccessLabel = "openlibertyapplications.apps.openliberty.io/allow-apiserver-access"
 )
 
 // ReconcileOpenLiberty reconciles an OpenLibertyApplication object
@@ -58,6 +60,7 @@ const applicationFinalizer = "finalizer.openlibertyapplications.apps.openliberty
 // +kubebuilder:rbac:groups=apps,resources=deployments;statefulsets,verbs=get;list;watch;create;update;delete,namespace=open-liberty-operator
 // +kubebuilder:rbac:groups=apps,resources=deployments/finalizers;statefulsets,verbs=update,namespace=open-liberty-operator
 // +kubebuilder:rbac:groups=core,resources=services;secrets;serviceaccounts;configmaps;persistentvolumeclaims,verbs=get;list;watch;create;update;delete,namespace=open-liberty-operator
+// +kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list,namespace=open-liberty-operator
 // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;delete,namespace=open-liberty-operator
 // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;delete,namespace=open-liberty-operator
 // +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;delete,namespace=open-liberty-operator
@@ -377,6 +380,69 @@ func (r *ReconcileOpenLiberty) Reconcile(ctx context.Context, request ctrl.Reque
 			common.StatusConditionTypeReconciled, instance)
 	}
 
+	// Kube API Server NetworkPolicy (based upon impl. by Martin Smithson)
+	apiServerNetworkPolicy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{
+		Name:      instance.Name + "-egress-dns-and-apiserver-access",
+		Namespace: instance.Namespace,
+	}}
+	err = r.CreateOrUpdate(apiServerNetworkPolicy, instance, func() error {
+		apiServerNetworkPolicy.Spec.PodSelector = metav1.LabelSelector{
+			MatchLabels: map[string]string{
+				OperatorAllowAPIServerAccessLabel: "true",
+			},
+		}
+		apiServerNetworkPolicy.Spec.Egress = make([]networkingv1.NetworkPolicyEgressRule, 0)
+
+		var dnsRule networkingv1.NetworkPolicyEgressRule
+		var usingPermissiveRule bool
+		// If allowed, add an Egress rule to access the OpenShift DNS or K8s CoreDNS. Otherwise, use a permissive cluster-wide Egress rule.
+		if r.IsOpenShift() {
+			usingPermissiveRule, dnsRule = r.getDNSEgressRule(reqLogger, "dns-default", "openshift-dns")
+		} else {
+			usingPermissiveRule, dnsRule = r.getDNSEgressRule(reqLogger, "kube-dns", "kube-system")
+		}
+		apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, dnsRule)
+
+		// If allowed, add an Egress rule to access the API server.
+		// Otherwise, if the OpenShift DNS or K8s CoreDNS Egress rule does not provide permissive cluster-wide access
+		// and the K8s API server could not be found, use a permissive cluster-wide Egress rule.
+		if apiServerEndpoints, err := r.getEndpoints("kubernetes", "default"); err == nil {
+			rule := networkingv1.NetworkPolicyEgressRule{}
+			// Define the port
+			port := networkingv1.NetworkPolicyPort{}
+			port.Protocol = &apiServerEndpoints.Subsets[0].Ports[0].Protocol
+			var portNumber intstr.IntOrString = intstr.FromInt((int)(apiServerEndpoints.Subsets[0].Ports[0].Port))
+			port.Port = &portNumber
+			rule.Ports = append(rule.Ports, port)
+
+			// Add the endpoint address as ipBlock entries
+			for _, endpoint := range apiServerEndpoints.Subsets {
+				for _, address := range endpoint.Addresses {
+					peer := networkingv1.NetworkPolicyPeer{}
+					ipBlock := networkingv1.IPBlock{}
+					ipBlock.CIDR = address.IP + "/32"
+
+					peer.IPBlock = &ipBlock
+					rule.To = append(rule.To, peer)
+				}
+			}
+			apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, rule)
+			reqLogger.Info("Found endpoints for kubernetes service in the default namespace")
+		} else if !usingPermissiveRule {
+			rule := networkingv1.NetworkPolicyEgressRule{}
+			apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, rule)
+			reqLogger.Info("Found endpoints for kubernetes service in the default namespace")
+		}
+		apiServerNetworkPolicy.Labels = ba.GetLabels()
+		apiServerNetworkPolicy.Annotations = oputils.MergeMaps(apiServerNetworkPolicy.Annotations, ba.GetAnnotations())
+		apiServerNetworkPolicy.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}
+		return nil
+	})
+	if err != nil {
+		reqLogger.Error(err, "Failed to reconcile API server network policy")
+		return r.ManageError(err, common.StatusConditionTypeReconciled, instance)
+	}
+
 	networkPolicy := &networkingv1.NetworkPolicy{ObjectMeta: defaultMeta}
 	if np := instance.Spec.NetworkPolicy; np == nil || np != nil && !np.IsDisabled() {
 		err = r.CreateOrUpdate(networkPolicy, instance, func() error {
@@ -850,3 +916,38 @@ func (r *ReconcileOpenLiberty) deletePVC(reqLogger logr.Logger, pvcName string,
 		}
 	}
 }
+
+func (r *ReconcileOpenLiberty) getEndpoints(serviceName string, namespace string) (*corev1.Endpoints, error) {
+	endpoints := &corev1.Endpoints{}
+	if err := r.GetClient().Get(context.TODO(), types.NamespacedName{Name: serviceName, Namespace: namespace}, endpoints); err != nil {
+		return nil, err
+	} else {
+		return endpoints, nil
+	}
+}
+
+func (r *ReconcileOpenLiberty) getDNSEgressRule(reqLogger logr.Logger, endpointsName string, endpointsNamespace string) (bool, networkingv1.NetworkPolicyEgressRule) {
+	dnsRule := networkingv1.NetworkPolicyEgressRule{}
+	if dnsEndpoints, err := r.getEndpoints(endpointsName, endpointsNamespace); err == nil {
+		if endpointPort := lutils.GetEndpointPortByName(&dnsEndpoints.Subsets[0].Ports, "dns"); endpointPort != nil {
+			dnsRule.Ports = append(dnsRule.Ports, lutils.CreateNetworkPolicyPortFromEndpointPort(endpointPort))
+		}
+		if endpointPort := lutils.GetEndpointPortByName(&dnsEndpoints.Subsets[0].Ports, "dns-tcp"); endpointPort != nil {
+			dnsRule.Ports = append(dnsRule.Ports, lutils.CreateNetworkPolicyPortFromEndpointPort(endpointPort))
+		}
+		peer := networkingv1.NetworkPolicyPeer{}
+		peer.NamespaceSelector = &metav1.LabelSelector{
+			MatchLabels: map[string]string{
+				"kubernetes.io/metadata.name": endpointsNamespace,
+			},
+		}
+		dnsRule.To = append(dnsRule.To, peer)
+		reqLogger.Info("Found endpoints for " + endpointsName + " service in the " + endpointsNamespace + " namespace")
+		return false, dnsRule
+	}
+	// use permissive rule
+	// egress:
+	//   - {}
+	reqLogger.Info("Failed to retrieve endpoints for " + endpointsName + " service in the " + endpointsNamespace + "  namespace. Using more permissive rule.")
+	return true, dnsRule
+}

internal/deploy/kubectl/openliberty-app-operator.yaml

@@ -155,6 +155,13 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:
@@ -297,6 +304,7 @@ spec:
         app.kubernetes.io/instance: open-liberty-operator
         app.kubernetes.io/name: open-liberty-operator
         control-plane: controller-manager
+        openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true"
     spec:
       affinity:
         nodeAffinity:

internal/deploy/kubectl/openliberty-app-rbac-watch-all.yaml

@@ -147,6 +147,13 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

internal/deploy/kubectl/openliberty-app-rbac-watch-another.yaml

@@ -149,6 +149,13 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

internal/deploy/kustomize/daily/base/open-liberty-operator.yaml

@@ -22,6 +22,7 @@ spec:
         app.kubernetes.io/instance: open-liberty-operator
         app.kubernetes.io/name: open-liberty-operator
         control-plane: controller-manager
+        openlibertyapplications.apps.openliberty.io/allow-apiserver-access: "true"
     spec:
       affinity:
         nodeAffinity:

internal/deploy/kustomize/daily/base/open-liberty-roles.yaml

@@ -158,6 +158,13 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

internal/deploy/kustomize/daily/overlays/watch-all-namespaces/cluster-roles.yaml

@@ -147,6 +147,13 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

internal/deploy/kustomize/daily/overlays/watch-another-namespace/olo-watched-ns/watched-roles.yaml

@@ -149,6 +149,13 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

utils/utils.go

@@ -9,6 +9,8 @@ import (
 	"strconv"
 	"strings"
 
+	networkingv1 "k8s.io/api/networking/v1"
+
 	olv1 "github.com/OpenLiberty/open-liberty-operator/api/v1"
 	rcoutils "github.com/application-stacks/runtime-component-operator/utils"
 	routev1 "github.com/openshift/api/route/v1"
@@ -18,6 +20,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -609,9 +612,11 @@ func IsLTPAJobConfigurationOutdated(job *v1.Job, appLeaderInstance *olv1.OpenLib
 	return false
 }
 
-func CustomizeLTPAJob(job *v1.Job, la *olv1.OpenLibertyApplication, ltpaSecretName string, serviceAccountName string, ltpaScriptName string) {
+func CustomizeLTPAJob(job *v1.Job, la *olv1.OpenLibertyApplication, ltpaSecretName string, serviceAccountName string, ltpaScriptName string, allowAPIServerAccessLabel string) {
 	encodingType := "aes" // the password encoding type for securityUtility (one of "xor", "aes", or "hash")
 	job.Spec.Template.ObjectMeta.Name = "liberty"
+	// Enable NetworkPolicy Egress access to Kube API Server
+	job.Spec.Template.Labels = rcoutils.MergeMaps(job.Spec.Template.Labels, map[string]string{allowAPIServerAccessLabel: "true"})
 	job.Spec.Template.Spec.Containers = []corev1.Container{
 		{
 			Name:            job.Spec.Template.ObjectMeta.Name,
@@ -678,3 +683,20 @@ func GetRequiredLabels(name string, instance string) map[string]string {
 	requiredLabels["app.kubernetes.io/managed-by"] = "open-liberty-operator"
 	return requiredLabels
 }
+
+func GetEndpointPortByName(endpointPorts *[]corev1.EndpointPort, name string) *corev1.EndpointPort {
+	for _, endpointPort := range *endpointPorts {
+		if endpointPort.Name == name {
+			return &endpointPort
+		}
+	}
+	return nil
+}
+
+func CreateNetworkPolicyPortFromEndpointPort(endpointPort *corev1.EndpointPort) networkingv1.NetworkPolicyPort {
+	port := networkingv1.NetworkPolicyPort{}
+	port.Protocol = &endpointPort.Protocol
+	var portNumber intstr.IntOrString = intstr.FromInt((int)(endpointPort.Port))
+	port.Port = &portNumber
+	return port
+}