Merge pull request #11 from datainfrahq/deepstorage

Deepstorage and Fixes

Author: AdheipSingh

Makefile

@@ -162,35 +162,55 @@ $(ENVTEST): $(LOCALBIN)
 
 ZK_OPERATOR_VERSION = 0.2.15
 ZK_VERSION = 0.2.15
+PINOT_OPERATOR_NAMESPACE=pinot-operator
+MINIO_OPERATOR_NAMESPACE=minio-operator
+ZOOKEEPER_OPERATOR_NAMESPACE=zookeeper-operator
+PINOT_CLUSTER_NAMESPACE=pinot
 
 .PHONY: helm-install-pinot-operator
 helm-install-pinot-operator: ## helm upgrade/install
 	helm upgrade --install \
-	--namespace pinot-operator \
+	--namespace ${PINOT_OPERATOR_NAMESPACE} \
 	--create-namespace \
-	pinot-operator helm/pinot-operator
+	${PINOT_OPERATOR_NAMESPACE} helm/pinot-operator
 
 .PHONY: helm-install-zk-operator
 helm-install-zk-operator: ## helm upgrade/install
 	helm repo add pravega https://charts.pravega.io
 	helm repo update pravega
 	helm upgrade --install \
-	--namespace zookeeper-operator \
+	--namespace ${ZOOKEEPER_OPERATOR_NAMESPACE} \
 	--create-namespace \
-	zk-operator pravega/zookeeper-operator \
+	${ZOOKEEPER_OPERATOR_NAMESPACE} pravega/zookeeper-operator \
 	--version=${ZK_OPERATOR_VERSION}
 	helm upgrade --install zk-pinot \
 	pravega/zookeeper \
-	--namespace pinot \
+	--namespace ${PINOT_CLUSTER_NAMESPACE} \
 	--create-namespace \
 	--version=${ZK_VERSION} \
 	--set replicas=1 \
 	--set persistence.storageClassName=${STORAGE_CLASS_NAME}
 
 .PHONY: clean
 clean: ## clean up getting started
-	kubectl delete -f examples/pinot-simple.yaml -n pinot
-	helm delete zk-pinot -n pinot
-	helm delete pinot-operator -n pinot-operator
-	helm delete zk-operator -n zookeeper-operator
-	kubectl delete ns pinot-operator zookeeper-operator pinot
+	kubectl delete -f examples/pinot-simple.yaml -n ${PINOT_CLUSTER_NAMESPACE}
+	helm delete zk-pinot -n ${PINOT_CLUSTER_NAMESPACE}
+	helm delete pinot-operator -n ${PINOT_OPERATOR_NAMESPACE}
+	helm delete zk-operator -n ${ZOOKEEPER_OPERATOR_NAMESPACE}
+	kubectl delete ns ${PINOT_OPERATOR_NAMESPACE} ${ZOOKEEPER_OPERATOR_NAMESPACE} ${PINOT_CLUSTER_NAMESPACE}
+
+## Helm deploy minio operator and minio
+.PHONY: helm-minio-install
+helm-minio-install:
+	helm repo add minio https://operator.min.io/
+	helm repo update minio
+	helm upgrade --install \
+	--namespace ${MINIO_OPERATOR_NAMESPACE} \
+	--create-namespace \
+	 ${MINIO_OPERATOR_NAMESPACE} minio/operator \
+	-f e2e/configs/minio-operator-override.yaml
+	helm upgrade --install \
+	--namespace ${PINOT_CLUSTER_NAMESPACE} \
+	--create-namespace \
+  	${PINOT_CLUSTER_NAMESPACE}-minio minio/tenant \
+	-f e2e/configs/minio-tenant-override.yaml

README.md

@@ -19,7 +19,7 @@
 
 Pinot Kubernetes Operator for deploying and managing heterogenous apache pinot kubernetes clusters. This operator is based on [Dsoi-Spec](https://github.com/datainfrahq/dsoi-spec) and is built using [operator-runtime](https://github.com/datainfrahq/operator-runtime). This is a radical new approach that brings ease of use and decouples application and kubernetes in a way that it becomes easier for day 2 operations. The goal of this project is to build a control plane for managing apache pinot clusters. The underlying controllers are built on conditions ie orthogonal concepts and not state machines.
 
-### Getting Started
+### Getting Started 
 
 ```
 export STORAGE_CLASS_NAME=civo-volume

api/v1beta1/pinot_types.go

@@ -23,57 +23,109 @@ import (
 
 // PinotSpec defines the desired state of Pinot
 type PinotSpec struct {
-	DeploymentOrder []PinotNodeType   `json:"deploymentOrder"`
-	External        ExternalSpec      `json:"external,omitempty"`
-	K8sConfig       []K8sConfig       `json:"k8sConfig"`
+	// +optional
+	Plugins []string `json:"plugins"`
+	// +required
+	DeploymentOrder []PinotNodeType `json:"deploymentOrder"`
+	// +required
+	External ExternalSpec `json:"external,omitempty"`
+	// +required
+	K8sConfig []K8sConfig `json:"k8sConfig"`
+	// +required
 	PinotNodeConfig []PinotNodeConfig `json:"pinotNodeConfig"`
-	Nodes           []NodeSpec        `json:"nodes"`
+	// +required
+	Nodes []NodeSpec `json:"nodes"`
 }
 
 type ExternalSpec struct {
+	// +required
 	Zookeeper ZookeeperSpec `json:"zookeeper"`
+	// +optional
+	DeepStorage DeepStorageSpec `json:"deepStorage"`
 }
 
 type ZookeeperSpec struct {
+	// +required
 	Spec ZookeeperConfig `json:"spec"`
 }
 
 type ZookeeperConfig struct {
+	// +required
 	ZkAddress string `json:"zkAddress"`
 }
 
+type DeepStorageSpec struct {
+	// +optional
+	Spec []DeepStorageConfig `json:"spec"`
+}
+
+type DeepStorageConfig struct {
+	// +optional
+	NodeType PinotNodeType `json:"nodeType"`
+	// +optional
+	Data string `json:"data"`
+}
+
 type K8sConfig struct {
-	Name               string                  `json:"name"`
-	Volumes            []v1.Volume             `json:"volumes,omitempty"`
-	Port               []v1.ContainerPort      `json:"port"`
-	VolumeMount        []v1.VolumeMount        `json:"volumeMount,omitempty"`
-	Image              string                  `json:"image"`
-	ImagePullPolicy    v1.PullPolicy           `json:"imagePullPolicy,omitempty"`
-	ServiceAccountName string                  `json:"serviceAccountName,omitempty"`
-	Env                []v1.EnvVar             `json:"env,omitempty"`
-	Tolerations        []v1.Toleration         `json:"tolerations,omitempty"`
-	PodMetadata        Metadata                `json:"podMetadata,omitempty"`
-	StorageConfig      []StorageConfig         `json:"storageConfig,omitempty"`
-	NodeSelector       map[string]string       `json:"nodeSelector,omitempty"`
-	Service            *v1.ServiceSpec         `json:"service,omitempty"`
-	Resources          v1.ResourceRequirements `json:"resources,omitempty"`
+	// +required
+	Name string `json:"name"`
+	// +optional
+	Volumes []v1.Volume `json:"volumes,omitempty"`
+	// +required
+	Port []v1.ContainerPort `json:"port"`
+	// +optional
+	VolumeMount []v1.VolumeMount `json:"volumeMount,omitempty"`
+	// +required
+	Image string `json:"image"`
+	// +optional
+	ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"`
+	// +optional
+	ServiceAccountName string `json:"serviceAccountName,omitempty"`
+	// +optional
+	Env []v1.EnvVar `json:"env,omitempty"`
+	// +optional
+	Tolerations []v1.Toleration `json:"tolerations,omitempty"`
+	// +optional
+	PodMetadata Metadata `json:"podMetadata,omitempty"`
+	// +optional
+	StorageConfig []StorageConfig `json:"storageConfig,omitempty"`
+	// +optional
+	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+	// +optional
+	Service *v1.ServiceSpec `json:"service,omitempty"`
+	// +optional
+	LivenessProbe *v1.Probe `json:"livenessProbe,omitempty"`
+	// +optional
+	ReadinessProbe *v1.Probe `json:"readinessProbe,omitempty"`
+	// +optional
+	StartUpProbe *v1.Probe `json:"startUpProbe,omitempty"`
+	// +optional
+	Resources v1.ResourceRequirements `json:"resources,omitempty"`
 }
 
 type Metadata struct {
+	// +optional
 	Annotations map[string]string `json:"annotations,omitempty"`
-	Labels      map[string]string `json:"labels,omitempty"`
+	// +optional
+	Labels map[string]string `json:"labels,omitempty"`
 }
 
 type StorageConfig struct {
-	Name      string                       `json:"name"`
-	MountPath string                       `json:"mountPath"`
-	PvcSpec   v1.PersistentVolumeClaimSpec `json:"spec"`
+	// +required
+	Name string `json:"name"`
+	// +required
+	MountPath string `json:"mountPath"`
+	// +required
+	PvcSpec v1.PersistentVolumeClaimSpec `json:"spec"`
 }
 
 type PinotNodeConfig struct {
-	Name     string `json:"name"`
+	// +required
+	Name string `json:"name"`
+	// +required
 	JavaOpts string `json:"java_opts"`
-	Data     string `json:"data"`
+	// +required
+	Data string `json:"data"`
 }
 
 type PinotNodeType string
@@ -86,12 +138,18 @@ const (
 )
 
 type NodeSpec struct {
-	Name            string        `json:"name"`
-	Kind            string        `json:"kind"`
-	NodeType        PinotNodeType `json:"nodeType"`
-	Replicas        int           `json:"replicas"`
-	K8sConfig       string        `json:"k8sConfig"`
-	PinotNodeConfig string        `json:"pinotNodeConfig"`
+	// +required
+	Name string `json:"name"`
+	// +required
+	Kind string `json:"kind"`
+	// +required
+	NodeType PinotNodeType `json:"nodeType"`
+	// +required
+	Replicas int `json:"replicas"`
+	// +required
+	K8sConfig string `json:"k8sConfig"`
+	// +required
+	PinotNodeConfig string `json:"pinotNodeConfig"`
 }
 
 // PinotStatus defines the observed state of Pinot