Backport helm updates

Author: jotak

.github/workflows/pull_request.yml

@@ -33,7 +33,7 @@ jobs:
         files: ./cover.out
         flags: unittests
         fail_ci_if_error: false
-        verbose: true   
+        verbose: true
 
   bundle-check:
     runs-on: ubuntu-latest
@@ -47,3 +47,27 @@ jobs:
       run: make update-bundle
     - name: check bundle clean state
       run: git add -A && git diff HEAD -I "operator-sdk-v" --exit-code
+
+  helm-check:
+    runs-on: ubuntu-latest
+    name: Checking helm up-to-date and valid
+    steps:
+    - name: install make
+      run: sudo apt-get install make
+
+    - name: checkout
+      uses: actions/checkout@v3
+
+    - name: generate helm
+      run: make helm-update
+
+    - name: check helm clean state
+      run: git add -A && git diff HEAD --exit-code
+
+    - name: set up Helm
+      uses: azure/[email protected]
+      with:
+        version: v3.17.0
+
+    - name: run helm lint
+      run: helm lint helm/

.mk/development.mk

@@ -31,7 +31,7 @@ undeploy-loki-tls:
 .PHONY: deploy-loki-tls
 deploy-loki-tls:
 	@echo -e "\n==> Deploy tls loki"
-	kubectl create namespace $(NAMESPACE)  --dry-run=client -o yaml | kubectl apply -f -
+	kubectl create namespace $(NAMESPACE) --dry-run=client -o yaml | kubectl apply -f -
 	kubectl config set-context --current --namespace=$(NAMESPACE)
 	curl -S -L https://raw.githubusercontent.com/netobserv/documents/main/examples/zero-click-loki/1-storage.yaml | kubectl create -f - || true
 	curl -S -L https://raw.githubusercontent.com/netobserv/documents/main/examples/zero-click-loki/2-loki-tls.yaml	 | kubectl create -f - || true
@@ -119,6 +119,8 @@ undeploy-all: undeploy-infra undeploy-sample-cr undeploy-sample-workload
 .PHONY: deploy-prometheus
 deploy-prometheus: ## Deploy prometheus.
 	@echo -e "\n==> Deploy prometheus"
+	kubectl create namespace $(NAMESPACE) || true
+	kubectl config set-context --current --namespace=$(NAMESPACE)
 	kubectl apply -f config/kind/deployment-prometheus.yaml
 	kubectl rollout status "deploy/prometheus" --timeout=600s
 	-pkill --oldest --full "9090:9090"

Makefile

@@ -526,6 +526,14 @@ endif
 	@echo ""
 	@echo "Everything is ready to be pushed. Before that, you should compare the content of $(BUNDLE_VERSION) with $(PREVIOUS_VERSION) to make sure it looks correct."
 
+# Update helm templates
+.PHONY: helm-update
+helm-update: YQ ## Update helm template
+	sed -i -r 's/appVersion:.*/appVersion: $(BUNDLE_VERSION)/g' helm/Chart.yaml
+	sed -i -r 's/version:.*/version: $(BUNDLE_VERSION:%-community=%)/g' helm/Chart.yaml
+	hack/helm-update.sh
+	cp LICENSE helm/
+
 include .mk/sample.mk
 include .mk/development.mk
 include .mk/local.mk

README.md

@@ -16,24 +16,73 @@ Flow data is then available in multiple ways, each optional:
 
 ## Getting Started
 
-You can install NetObserv Operator using [OLM](https://olm.operatorframework.io/) if it is available in your cluster, or directly from its repository.
+You can install the NetObserv Operator using [Helm](https://helm.sh/), or directly from sources.
 
-### Install with OLM
+In OpenShift, NetObserv is named Network Observability operator and can be found in OperatorHub as an OLM operator. This section does not apply to it: please refer to the [OpenShift documentation](https://docs.openshift.com/container-platform/latest/observability/network_observability/installing-operators.html) in that case.
 
 > [!IMPORTANT]
-> There hasn't been recent releases pushed to the community OperatorHub. This is mostly due to the lack of demand. Unless there is demand, going forward we only release the downstream NetObserv aka [Network Observability operator](https://docs.openshift.com/container-platform/latest/observability/network_observability/network-observability-operator-release-notes.html) for OpenShift. But there's nothing written in stone. [Let us know](https://github.com/netobserv/network-observability-operator/discussions) if you would like that to change.
+> NetObserv community was previously distributed via [OperatorHub](https://operatorhub.io/operator/netobserv-operator). This installation method is replaced with a helm chart. If you previously installed NetObserv community from OperatorHub, we recommend that you uninstall it, and re-install using the helm chart. The operation should not cause any data loss.
 
-NetObserv Operator is available in [OperatorHub](https://operatorhub.io/operator/netobserv-operator) with guided steps on how to install this. It is also available in the OperatorHub catalog directly in the OpenShift Console.
+### Pre-requisite
 
-![OpenShift OperatorHub search](./docs/assets/operatorhub-search.png)
+NetObserv has a couple of dependencies that must be installed on your cluster:
 
-Please read the [operator description in OLM](./config/descriptions/upstream.md).
+- Cert-manager
+- Prometheus
+- Loki
 
-After the operator is installed, create a `FlowCollector` resource:
+Loki is not mandatory but improves the overall experience with NetObserv.
+If you don't have these dependencies already, some convenience scripts are available from the repository, provided for demo purpose:
 
-![OpenShift OperatorHub FlowCollector](./docs/assets/operatorhub-flowcollector.png)
+```bash
+git clone https://github.com/netobserv/network-observability-operator.git && cd network-observability-operator
+PORT_FWD=false make deploy-prometheus deploy-loki install-cert-manager
+# (it is expected to see errors while running this script, since it runs several attempts creating a certificate for testing, before eventually succeeding)
+```
 
-Refer to the [Configuration section](#configuration) of this document.
+### Install with Helm
+
+Check it out on [ArtifactHub](https://artifacthub.io/packages/helm/netobserv/netobserv-operator).
+
+```bash
+helm repo add netobserv https://netobserv.io/static/helm/ --force-update
+helm install my-netobserv --set standaloneConsole.enable=true netobserv/netobserv-operator
+# If you're in OpenShift, you can omit "--set standaloneConsole.enable=true" to use the Console plugin instead.
+```
+
+You can now create a `FlowCollector` resource. Refer to the [Configuration section](#configuration) of this document. A very short `FlowCollector` should work, using default values, plus with the standalone console enabled:
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: flows.netobserv.io/v1beta2
+kind: FlowCollector
+metadata:
+  name: cluster
+spec:
+  namespace: netobserv
+  consolePlugin:
+    advanced:
+      env:
+        TEST_CONSOLE: "true"
+  prometheus:
+    querier:
+      manual:
+        url: http://prometheus:9090
+EOF
+```
+
+A few remarks:
+- While the [web console](https://github.com/netobserv/network-observability-console-plugin) is primarily designed as a plugin for the OpenShift Console, it is still possible to deploy it as a standalone, which the dev team sometimes use for testing. This is why it is mentioned as "TEST_CONSOLE" here.
+- If you're in OpenShift, you should omit "TEST_CONSOLE: true" to use the Console plugin instead, which offers a better / more integrated experience.
+- You can change the Prometheus URL depending on your installation. This example URL works if you use the `make deploy-prometheus` script from the repository. Prometheus configuration options are documented [here](./docs/FlowCollector.md#flowcollectorspecprometheus-1).
+
+To view the test console, you can port-forward 9001:
+
+```bash
+kubectl port-forward svc/netobserv-plugin 9001:9001 -n netobserv
+```
+
+Then open http://localhost:9001/ in your browser.
 
 ### Install from repository
 

RELEASE.md

@@ -32,8 +32,7 @@ Once all sub-components are released (or have a release candidate), we can proce
 Edit the [Makefile](./Makefile) to update `PREVIOUS_VERSION`, `BUNDLE_VERSION`, `PLG_VERSION`, `FLP_VERSION` and `BPF_VERSION`.
 
 ```bash
-
-make update-bundle
+make update-bundle helm-update
 
 # Set desired operator version - CAREFUL, no leading "v" here
 version="1.8.1-community"

config/kind/deployment-prometheus.yaml

@@ -1,3 +1,35 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-scraper
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - services
+  verbs:
+  - list
+  - get
+  - watch
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus-sa
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus-scraper
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-scraper
+subjects:
+- kind: ServiceAccount
+  name: prometheus-sa
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -35,7 +67,7 @@ spec:
               mountPath: /etc/prometheus/
             - name: prometheus-storage-volume
               mountPath: /prometheus/
-      serviceAccountName: flowlogs-pipeline
+      serviceAccountName: prometheus-sa
       volumes:
         - name: prometheus-config-volume
           configMap:
@@ -75,6 +107,8 @@ data:
       - job_name: 'flowlogs-pipeline'
         kubernetes_sd_configs:
           - role: pod
+            namespaces:
+              own_namespace: true
 
         relabel_configs:
         # Scrape only pods that have "prometheus.io/scrape = true" annotation.

docs/assets/operatorhub-flowcollector.png

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+mkdir -p _tmp
+
+# Copy and edit CRDs
+for crd in "flows.netobserv.io_flowcollectors.yaml" "flows.netobserv.io_flowmetrics.yaml"; do
+  cp "bundle/manifests/$crd" helm/templates
+  sed -i -r 's/(`[^`]*\{\{[^`]*`)/{{\1}}/g' helm/templates/$crd # escape "{{" for helm
+  yq -i '.spec.conversion.webhook.clientConfig.service.namespace="{{ .Release.Namespace }}"' helm/templates/$crd
+done
+
+# Copy unchanged files
+for file in "netobserv-manager-config_v1_configmap.yaml" "netobserv-metrics-service_v1_service.yaml" "netobserv-webhook-service_v1_service.yaml" ; do
+  cp "bundle/manifests/$file" helm/templates
+done
+
+# Services: remove openshift annotations for certificates (and some kubeconfig labels)
+yq -i 'del(.metadata.annotations)' helm/templates/netobserv-metrics-service_v1_service.yaml
+yq -i 'del(.metadata.annotations)' helm/templates/netobserv-webhook-service_v1_service.yaml
+yq -i 'del(.metadata.labels)' helm/templates/netobserv-webhook-service_v1_service.yaml
+
+# Extract data from clusterserviceversion
+yq '.spec.install.spec.deployments[0].spec' bundle/manifests/netobserv-operator.clusterserviceversion.yaml > _tmp/csv-deployment.yaml
+yq '.spec.install.spec.clusterPermissions[0]' bundle/manifests/netobserv-operator.clusterserviceversion.yaml > _tmp/csv-clusterrole.yaml
+yq '.spec.install.spec.permissions[0]' bundle/manifests/netobserv-operator.clusterserviceversion.yaml > _tmp/csv-role.yaml
+ 
+# Create deployment
+yq '{"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": "netobserv-controller-manager", "labels": {"app": "netobserv-operator", "control-plane": "controller-manager"}}, "spec": .}' _tmp/csv-deployment.yaml > helm/templates/deployment.yaml
+
+# Inject paramterized standalone console in deployment
+PLUGIN_IMAGE=$(yq '(.spec.template.spec.containers[0].env[] | select(.name=="RELATED_IMAGE_CONSOLE_PLUGIN") | .value)' helm/templates/deployment.yaml)
+STANDALONE_IMAGE=$(echo $PLUGIN_IMAGE | sed 's/network-observability-console-plugin/network-observability-standalone-frontend/')
+yq -i "(.spec.template.spec.containers[0].env[] | select(.name==\"RELATED_IMAGE_CONSOLE_PLUGIN\") | .value) = \"{{ if .Values.standaloneConsole.enable }}$STANDALONE_IMAGE{{ else }}$PLUGIN_IMAGE{{ end }}\"" helm/templates/deployment.yaml
+
+# Create roles
+yq '{"apiVersion": "v1", "kind": "ServiceAccount", "metadata": {"name": .serviceAccountName}}' _tmp/csv-clusterrole.yaml > helm/templates/serviceaccount.yaml
+yq '{"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", "metadata": {"name": "netobserv-manager-role"}, "rules": .rules}' _tmp/csv-clusterrole.yaml > helm/templates/clusterrole.yaml
+yq '{"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRoleBinding", "metadata": {"name": "netobserv-manager-rolebinding"}, "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": "netobserv-manager-role"}, "subjects": [{"kind": "ServiceAccount", "name": .serviceAccountName, "namespace": "{{ .Release.Namespace }}"}]}' _tmp/csv-clusterrole.yaml > helm/templates/clusterrolebinding.yaml
+yq '{"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "Role", "metadata": {"name": "netobserv-leader-election-role"}, "rules": .rules}' _tmp/csv-role.yaml > helm/templates/role.yaml
+yq '{"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", "metadata": {"name": "netobserv-leader-election-rolebinding"}, "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": "netobserv-leader-election-role"}, "subjects": [{"kind": "ServiceAccount", "name": .serviceAccountName, "namespace": "{{ .Release.Namespace }}"}]}' _tmp/csv-role.yaml > helm/templates/rolebinding.yaml

docs/assets/operatorhub-search.png

@@ -1,6 +1,6 @@
 apiVersion: v2
-name: Netobserv
-description: A Helm chart to install Network Observability Operator on k8s
+name: netobserv-operator
+description: Network Observability in Kubernetes based on eBPF.
 
 # A chart can be either an 'application' or a 'library' chart.
 #
@@ -15,10 +15,43 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.0.1
+version: 1.8.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.6.2-community"
+appVersion: 1.8.1-community
+
+keywords:
+- network observability
+- ebpf
+- ipfix
+- flow tracing
+- flows
+- topology
+- network
+- observability
+
+home: https://netobserv.io/
+
+sources:
+- https://github.com/netobserv/network-observability-operator
+- https://github.com/netobserv/flowlogs-pipeline
+- https://github.com/netobserv/netobserv-ebpf-agent
+- https://github.com/netobserv/network-observability-console-plugin
+- https://github.com/netobserv/network-observability-cli
+
+maintainers:
+- email: [email protected]
+  name: Julien Pinsonneau
+- email: [email protected]
+  name: Joel Takvorian
+- email: [email protected]
+  name: Mohamed S. Mahmoud
+- email: [email protected]
+  name: Olivier Cazade
+- email: [email protected]
+  name: Steven Lee
+
+icon: https://netobserv.io/static/assets/images/netobserv.svg