fix(log): add custom error handler for Kubernetes API errors

pkg/controllers/operator/cilium-crds/endpoint/endpoint_controller.go

@@ -533,7 +533,7 @@ func (r *endpointReconciler) handlePodUpsert(ctx context.Context, newPEP *PodEnd
 		// May end up getting another endpoint ID below if we try to create the CEP below.
 		// No downside to this.
 
-		if !k8serrors.IsNotFound(err) && err != nil {
+		if !k8serrors.IsNotFound(err) {
 			r.l.WithError(err).WithFields(logrus.Fields{
 				"podKey": newPEP.key.String(),
 				"pep":    newPEP,

pkg/k8s/watcher_linux.go

@@ -2,9 +2,12 @@ package k8s
 
 import (
 	"context"
+	"strings"
 	"sync"
 	"time"
 
+	"k8s.io/apimachinery/pkg/util/runtime"
+
 	agentK8s "github.com/cilium/cilium/daemon/k8s"
 	"github.com/cilium/cilium/pkg/hive/cell"
 	"github.com/cilium/cilium/pkg/ipcache"
@@ -17,6 +20,13 @@ import (
 	"github.com/cilium/cilium/pkg/option"
 )
 
+func init() {
+	// Register custom error handler for the watcher
+	runtime.ErrorHandlers = []func(error){
+		retinaK8sErrorHandler,
+	}
+}
+
 const (
 	K8sAPIGroupCiliumEndpointV2 = "cilium/v2::CiliumEndpoint"
 	K8sAPIGroupServiceV1Core    = "core/v1::Service"
@@ -92,3 +102,22 @@ func Start(ctx context.Context, k *watchers.K8sWatcher) {
 	<-syncdCache
 	logger.Info("Kubernetes watcher synced")
 }
+
+// retinaK8sErrorHandler is a custom error handler for the watcher
+// that logs the error and tags the error to easily identify
+func retinaK8sErrorHandler(e error) {
+	errStr := e.Error()
+	switch {
+	case strings.Contains(errStr, "Failed to watch *v1.Node"):
+		logger.WithField("actualError", errStr).Error("Potentially Network Error coming from K8s API Server failing to watch Nodes")
+	case strings.Contains(errStr, "Failed to watch *v2.CiliumEndpoint"):
+		logger.WithField("actualError", errStr).Error("Potentially Network Error coming from K8s API Server failing to watch CiliumEndpoints")
+	case strings.Contains(errStr, "Failed to watch *v1.Service"):
+		logger.WithField("actualError", errStr).Error("Potentially Network Error coming from K8s API Server failing to watch Services")
+	case strings.Contains(errStr, "Failed to watch *v2.CiliumNode"):
+		logger.WithField("actualError", errStr).Error("Potentially Network Error coming from K8s API Server failing to watch CiliumNodes")
+
+	default:
+		k8s.K8sErrorHandler(e)
+	}
+}