Skip to content

Commit bb03624

Browse files
jmcwilliams-tejmcwilliams-te
authored
[Ingestion Controller ] Add compaction, rules, dynamic configurations, nativeSpec (#197)
* Add compaction, rules, dynamic configurations, nativeSpec * update strings in EmitEventGeneric() calls within dynamic_config.go to use constants, set MinIO version to avoid unexpected breaking upgrades
1 parent 1430903 commit bb03624

24 files changed

+11962
-10778
lines changed

‎Makefile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ NAMESPACE_DRUID_OPERATOR ?= "druid-operator-system"
1111
NAMESPACE_ZK_OPERATOR ?= "zk-operator"
1212
# NAMESPACE for zk operator e2e
1313
NAMESPACE_MINIO_OPERATOR ?= "minio-operator"
14+
# MinIO version to install. Specific version is set to avoid breaking changes
15+
# using latest version. This version is tested with the operator.
16+
MINIO_VERSION ?= "6.0.4"
1417
# NAMESPACE for druid app e2e
1518
NAMESPACE_DRUID ?= "druid"
1619

@@ -116,9 +119,11 @@ helm-minio-install: ## Helm deploy minio operator and minio
116119
--namespace ${NAMESPACE_MINIO_OPERATOR} \
117120
--create-namespace \
118121
${NAMESPACE_MINIO_OPERATOR} minio/operator \
122+
--version ${MINIO_VERSION} \
119123
-f e2e/configs/minio-operator-override.yaml
120124
helm upgrade --install \
121125
--namespace ${NAMESPACE_DRUID} \
126+
--version ${MINIO_VERSION} \
122127
--create-namespace \
123128
${NAMESPACE_DRUID}-minio minio/tenant \
124129
-f e2e/configs/minio-tenant-override.yaml

‎apis/druid/v1alpha1/druid_types.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@ package v1alpha1
33
import (
44
"encoding/json"
55

6+
druidapi "github.com/datainfrahq/druid-operator/pkg/druidapi"
67
appsv1 "k8s.io/api/apps/v1"
78
autoscalev2 "k8s.io/api/autoscaling/v2"
89
v1 "k8s.io/api/core/v1"
910
networkingv1 "k8s.io/api/networking/v1"
1011
policyv1 "k8s.io/api/policy/v1"
1112
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/apimachinery/pkg/runtime"
1214
)
1315

1416
// druid-operator deploys a druid cluster from given spec below, based on the spec it would create following
@@ -281,6 +283,13 @@ type DruidSpec struct {
281283
// CoreSite Contents of `core-site.xml`.
282284
// +optional
283285
CoreSite string `json:"core-site.xml,omitempty"`
286+
287+
// Dynamic Configurations for Druid. Applied through the dynamic configuration API.
288+
// +optional
289+
DynamicConfig runtime.RawExtension `json:"dynamicConfig,omitempty"`
290+
291+
// +optional
292+
Auth druidapi.Auth `json:"auth,omitempty"`
284293
}
285294

286295
// DruidNodeSpec Specification of `Druid` Node type and its configurations.
@@ -481,6 +490,10 @@ type DruidNodeSpec struct {
481490
// ServiceAccountName Kubernetes native `serviceAccountName` specification.
482491
// +optional
483492
ServiceAccountName string `json:"serviceAccountName,omitempty"`
493+
494+
// Dynamic Configurations for Druid. Applied through the dynamic configuration API.
495+
// +optional
496+
DynamicConfig runtime.RawExtension `json:"dynamicConfig,omitempty"`
484497
}
485498

486499
// ZookeeperSpec IGNORED (Future API): In order to make Druid dependency setup extensible from within Druid operator.

‎apis/druid/v1alpha1/druidingestion_types.go

Lines changed: 26 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@ limitations under the License.
1717
package v1alpha1
1818

1919
import (
20+
druidapi "github.com/datainfrahq/druid-operator/pkg/druidapi"
2021
v1 "k8s.io/api/core/v1"
2122
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
23+
"k8s.io/apimachinery/pkg/runtime"
2224
)
2325

2426
type DruidIngestionMethod string
@@ -39,37 +41,39 @@ type DruidIngestionSpec struct {
3941
// +required
4042
Ingestion IngestionSpec `json:"ingestion"`
4143
// +optional
42-
Auth Auth `json:"auth"`
44+
Auth druidapi.Auth `json:"auth"`
4345
}
4446

4547
type IngestionSpec struct {
4648
// +required
4749
Type DruidIngestionMethod `json:"type"`
48-
// +required
50+
// +optional
51+
// Spec should be passed in as a JSON string.
52+
// Note: This field is planned for deprecation in favor of nativeSpec.
4953
Spec string `json:"spec,omitempty"`
54+
// +optional
55+
// nativeSpec allows the ingestion specification to be defined in a native Kubernetes format.
56+
// This is particularly useful for environment-specific configurations and will eventually
57+
// replace the JSON-based Spec field.
58+
// Note: Spec will be ignored if nativeSpec is provided.
59+
NativeSpec runtime.RawExtension `json:"nativeSpec,omitempty"`
60+
// +optional
61+
Compaction runtime.RawExtension `json:"compaction,omitempty"`
62+
// +optional
63+
Rules []runtime.RawExtension `json:"rules,omitempty"`
5064
}
5165

5266
type DruidIngestionStatus struct {
53-
TaskId string `json:"taskId"`
54-
Type string `json:"type,omitempty"`
55-
Status v1.ConditionStatus `json:"status,omitempty"`
56-
Reason string `json:"reason,omitempty"`
57-
Message string `json:"message,omitempty"`
58-
LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
59-
CurrentIngestionSpec string `json:"currentIngestionSpec.json"`
60-
}
61-
62-
type AuthType string
63-
64-
const (
65-
BasicAuth AuthType = "basic-auth"
66-
)
67-
68-
type Auth struct {
69-
// +required
70-
Type AuthType `json:"type"`
71-
// +required
72-
SecretRef v1.SecretReference `json:"secretRef"`
67+
TaskId string `json:"taskId"`
68+
Type string `json:"type,omitempty"`
69+
Status v1.ConditionStatus `json:"status,omitempty"`
70+
Reason string `json:"reason,omitempty"`
71+
Message string `json:"message,omitempty"`
72+
LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
73+
// CurrentIngestionSpec is a string instead of RawExtension to maintain compatibility with existing
74+
// IngestionSpecs that are stored as JSON strings.
75+
CurrentIngestionSpec string `json:"currentIngestionSpec.json"`
76+
CurrentRules []runtime.RawExtension `json:"rules,omitempty"`
7377
}
7478

7579
// +kubebuilder:object:root=true

‎apis/druid/v1alpha1/zz_generated.deepcopy.go

Lines changed: 22 additions & 20 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎chart/crds/druid.apache.org_druidingestions.yaml

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
33
kind: CustomResourceDefinition
44
metadata:
55
annotations:
6-
controller-gen.kubebuilder.io/version: v0.11.2
7-
creationTimestamp: null
6+
controller-gen.kubebuilder.io/version: v0.14.0
87
name: druidingestions.druid.apache.org
98
spec:
109
group: druid.apache.org
@@ -28,14 +27,19 @@ spec:
2827
description: Ingestion is the Schema for the Ingestion API
2928
properties:
3029
apiVersion:
31-
description: 'APIVersion defines the versioned schema of this representation
32-
of an object. Servers should convert recognized schemas to the latest
33-
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
30+
description: |-
31+
APIVersion defines the versioned schema of this representation of an object.
32+
Servers should convert recognized schemas to the latest internal value, and
33+
may reject unrecognized values.
34+
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
3435
type: string
3536
kind:
36-
description: 'Kind is a string value representing the REST resource this
37-
object represents. Servers may infer this from the endpoint the client
38-
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
37+
description: |-
38+
Kind is a string value representing the REST resource this object represents.
39+
Servers may infer this from the endpoint the client submits requests to.
40+
Cannot be updated.
41+
In CamelCase.
42+
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
3943
type: string
4044
metadata:
4145
type: object
@@ -44,8 +48,9 @@ spec:
4448
auth:
4549
properties:
4650
secretRef:
47-
description: SecretReference represents a Secret Reference. It
48-
has enough information to retrieve secret in any namespace
51+
description: |-
52+
SecretReference represents a Secret Reference. It has enough information to retrieve secret
53+
in any namespace
4954
properties:
5055
name:
5156
description: name is unique within a namespace to reference
@@ -67,7 +72,26 @@ spec:
6772
type: string
6873
ingestion:
6974
properties:
75+
compaction:
76+
type: object
77+
x-kubernetes-preserve-unknown-fields: true
78+
nativeSpec:
79+
description: |-
80+
nativeSpec allows the ingestion specification to be defined in a native Kubernetes format.
81+
This is particularly useful for environment-specific configurations and will eventually
82+
replace the JSON-based Spec field.
83+
Note: Spec will be ignored if nativeSpec is provided.
84+
type: object
85+
x-kubernetes-preserve-unknown-fields: true
86+
rules:
87+
items:
88+
type: object
89+
x-kubernetes-preserve-unknown-fields: true
90+
type: array
7091
spec:
92+
description: |-
93+
Spec should be passed in as a JSON string.
94+
Note: This field is planned for deprecation in favor of nativeSpec.
7195
type: string
7296
type:
7397
type: string
@@ -83,6 +107,9 @@ spec:
83107
status:
84108
properties:
85109
currentIngestionSpec.json:
110+
description: |-
111+
CurrentIngestionSpec is a string instead of RawExtension to maintain compatibility with existing
112+
IngestionSpecs that are stored as JSON strings.
86113
type: string
87114
lastUpdateTime:
88115
format: date-time
@@ -91,6 +118,11 @@ spec:
91118
type: string
92119
reason:
93120
type: string
121+
rules:
122+
items:
123+
type: object
124+
x-kubernetes-preserve-unknown-fields: true
125+
type: array
94126
status:
95127
type: string
96128
taskId:

0 commit comments

Comments
 (0)