Skip to content

Commit 6155321

Browse files
author
Chance Zibolski
committed
images,charts,hack: Remove hive image build, customize hive in chart
Removes the metering-hive image and uses the quay.io/coreos/hive image directly, and instead customizes configuration and entrypoint scripts using configmaps/secrets and helm templates.
1 parent a932640 commit 6155321

25 files changed

+269
-618
lines changed

‎Makefile

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ endif
4949
DOCKER_COMMON_NAMES := \
5050
reporting-operator \
5151
metering-operator \
52-
hive \
5352
metering-e2e
5453

5554
DOCKER_BUILD_NAMES = $(DOCKER_COMMON_NAMES)
@@ -60,7 +59,6 @@ DOCKER_BASE_URL := quay.io/coreos
6059

6160
METERING_OPERATOR_IMAGE := $(DOCKER_BASE_URL)/metering-helm-operator
6261
REPORTING_OPERATOR_IMAGE := $(DOCKER_BASE_URL)/metering-reporting-operator
63-
HIVE_IMAGE := $(DOCKER_BASE_URL)/metering-hive
6462
METERING_E2E_IMAGE := $(DOCKER_BASE_URL)/metering-e2e
6563

6664
GIT_SHA := $(shell git rev-parse HEAD)
@@ -187,9 +185,6 @@ metering-e2e-docker-build: Dockerfile.e2e
187185
metering-operator-docker-build: Dockerfile.metering-operator
188186
$(MAKE) docker-build DOCKERFILE=$< IMAGE_NAME=$(METERING_OPERATOR_IMAGE) DOCKER_BUILD_CONTEXT=$(ROOT_DIR)
189187

190-
hive-docker-build: images/hive/Dockerfile
191-
$(MAKE) docker-build DOCKERFILE=$< IMAGE_NAME=$(HIVE_IMAGE)
192-
193188
# Update dependencies
194189
vendor: Gopkg.toml
195190
dep ensure -v

‎charts/presto/templates/_helpers.tpl

Lines changed: 12 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -51,66 +51,6 @@ connector.name=jmx
5151
{{- end }}
5252

5353
{{- define "hive-env" }}
54-
- name: CORE_CONF_fs_s3a_access_key
55-
valueFrom:
56-
secretKeyRef:
57-
name: hive-common-secrets
58-
key: aws-access-key-id
59-
optional: true
60-
- name: CORE_CONF_fs_s3a_secret_key
61-
valueFrom:
62-
secretKeyRef:
63-
name: hive-common-secrets
64-
key: aws-secret-access-key
65-
optional: true
66-
- name: CORE_CONF_fs_defaultFS
67-
valueFrom:
68-
configMapKeyRef:
69-
name: hive-common-config
70-
key: default-fs
71-
optional: true
72-
- name: HIVE_SITE_CONF_hive_metastore_uris
73-
valueFrom:
74-
configMapKeyRef:
75-
name: hive-common-config
76-
key: metastore-uris
77-
- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionURL
78-
valueFrom:
79-
configMapKeyRef:
80-
name: hive-common-config
81-
key: db-connection-url
82-
- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName
83-
valueFrom:
84-
configMapKeyRef:
85-
name: hive-common-config
86-
key: db-connection-driver
87-
- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName
88-
valueFrom:
89-
secretKeyRef:
90-
name: hive-common-secrets
91-
key: db-connection-username
92-
optional: true
93-
- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword
94-
valueFrom:
95-
secretKeyRef:
96-
name: hive-common-secrets
97-
key: db-connection-password
98-
optional: true
99-
- name: HIVE_SITE_CONF_hive_metastore_schema_verification
100-
valueFrom:
101-
configMapKeyRef:
102-
name: hive-common-config
103-
key: enable-metastore-schema-verification
104-
- name: HIVE_SITE_CONF_datanucleus_schema_autoCreateAll
105-
valueFrom:
106-
configMapKeyRef:
107-
name: hive-common-config
108-
key: auto-create-metastore-schema
109-
- name: HIVE_SITE_CONF_hive_default_fileformat
110-
valueFrom:
111-
configMapKeyRef:
112-
name: hive-common-config
113-
key: default-file-format
11454
- name: MY_NODE_NAME
11555
valueFrom:
11656
fieldRef:
@@ -125,4 +65,16 @@ connector.name=jmx
12565
fieldPath: metadata.namespace
12666
- name: JAVA_MAX_MEM_RATIO
12767
value: "50"
68+
- name: AWS_ACCESS_KEY_ID
69+
valueFrom:
70+
secretKeyRef:
71+
name: hive-secrets
72+
key: aws-access-key-id
73+
optional: true
74+
- name: AWS_SECRET_ACCESS_KEY
75+
valueFrom:
76+
secretKeyRef:
77+
name: hive-secrets
78+
key: aws-secret-access-key
79+
optional: true
12880
{{- end }}

‎charts/presto/templates/hive-common-config.yaml

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎charts/presto/templates/hive-configmap.yaml

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: hive-config
5+
{{- block "extraMetadata" . }}
6+
{{- end }}
7+
data:
8+
hive-site.xml: |
9+
<configuration>
10+
<property>
11+
<name>hive.server2.enable.doAs</name>
12+
<value>false</value>
13+
</property>
14+
<property>
15+
<name>hive.server2.use.SSL</name>
16+
<value>false</value>
17+
</property>
18+
<property>
19+
<name>hive.server2.authentication</name>
20+
<value>NOSASL</value>
21+
</property>
22+
<property>
23+
<name>hive.metastore.uris</name>
24+
<value>{{ .Values.spec.hive.config.metastoreURIs }}</value>
25+
</property>
26+
<property>
27+
<name>javax.jdo.option.ConnectionURL</name>
28+
<value>{{ .Values.spec.hive.config.dbConnectionURL }}</value>
29+
</property>
30+
<property>
31+
<name>javax.jdo.option.ConnectionDriverName</name>
32+
<value>{{ .Values.spec.hive.config.dbConnectionDriver }}</value>
33+
</property>
34+
<property>
35+
<name>javax.jdo.option.ConnectionUserName</name>
36+
<value>{{ .Values.spec.hive.config.dbConnectionUsername }}</value>
37+
</property>
38+
<property>
39+
<name>javax.jdo.option.ConnectionPassword</name>
40+
<value>{{ .Values.spec.hive.config.dbConnectionPassword }}</value>
41+
</property>
42+
<property>
43+
<name>datanucleus.schema.autoCreateAll</name>
44+
<value>{{ .Values.spec.hive.config.autoCreateMetastoreSchema }}</value>
45+
</property>
46+
<property>
47+
<name>hive.metastore.schema.verification</name>
48+
<value>{{ .Values.spec.hive.config.enableMetastoreSchemaVerification }}</value>
49+
</property>
50+
<property>
51+
<name>hive.default.fileformat</name>
52+
<value>{{ .Values.spec.hive.config.defaultFileFormat }}</value>
53+
</property>
54+
</configuration>
55+
56+
57+
hive-log4j2.properties: |
58+
status = INFO
59+
name = HiveLog4j2
60+
packages = org.apache.hadoop.hive.ql.log
61+
62+
# list of properties
63+
property.hive.log.level = INFO
64+
property.hive.root.logger = console
65+
property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}
66+
property.hive.log.file = hive.log
67+
68+
# list of all appenders
69+
appenders = console
70+
71+
# console appender
72+
appender.console.type = Console
73+
appender.console.name = console
74+
appender.console.target = SYSTEM_ERR
75+
appender.console.layout.type = PatternLayout
76+
appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
77+
78+
# list of all loggers
79+
loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX
80+
81+
logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
82+
logger.NIOServerCnxn.level = WARN
83+
84+
logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
85+
logger.ClientCnxnSocketNIO.level = WARN
86+
87+
logger.DataNucleus.name = DataNucleus
88+
logger.DataNucleus.level = ERROR
89+
90+
logger.Datastore.name = Datastore
91+
logger.Datastore.level = ERROR
92+
93+
logger.JPOX.name = JPOX
94+
logger.JPOX.level = ERROR
95+
96+
# root logger
97+
rootLogger.level = ${sys:hive.log.level}
98+
rootLogger.appenderRefs = root
99+
rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}
100+
101+
hive-exec-log4j2.properties: |
102+
status = INFO
103+
name = HiveLog4j2
104+
packages = org.apache.hadoop.hive.ql.log
105+
106+
# list of properties
107+
property.hive.log.level = INFO
108+
property.hive.root.logger = console
109+
property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}
110+
property.hive.log.file = hive.log
111+
112+
# list of all appenders
113+
appenders = console
114+
115+
# console appender
116+
appender.console.type = Console
117+
appender.console.name = console
118+
appender.console.target = SYSTEM_ERR
119+
appender.console.layout.type = PatternLayout
120+
appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
121+
122+
# list of all loggers
123+
loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX
124+
125+
logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
126+
logger.NIOServerCnxn.level = WARN
127+
128+
logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
129+
logger.ClientCnxnSocketNIO.level = WARN
130+
131+
logger.DataNucleus.name = DataNucleus
132+
logger.DataNucleus.level = ERROR
133+
134+
logger.Datastore.name = Datastore
135+
logger.Datastore.level = ERROR
136+
137+
logger.JPOX.name = JPOX
138+
logger.JPOX.level = ERROR
139+
140+
# root logger
141+
rootLogger.level = ${sys:hive.log.level}
142+
rootLogger.appenderRefs = root
143+
rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}
144+

‎charts/presto/templates/hive-metastore-config.yaml

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎charts/presto/templates/hive-metastore-statefulset.yaml

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,8 @@ spec:
2828
{{ toYaml .Values.spec.hive.labels | indent 8 }}
2929
{{- end }}
3030
annotations:
31-
hive-common-configmap-hash: {{ include (print $.Template.BasePath "/hive-common-config.yaml") . | sha256sum }}
32-
hive-common-secret-hash: {{ include (print $.Template.BasePath "/hive-common-secrets.yaml") . | sha256sum }}
33-
hive-metastore-configmap-hash: {{ include (print $.Template.BasePath "/hive-metastore-config.yaml") . | sha256sum }}
31+
hive-configmap-hash: {{ include (print $.Template.BasePath "/hive-configmap.yaml") . | sha256sum }}
32+
hive-scripts-hash: {{ include (print $.Template.BasePath "/hive-scripts-configmap.yaml") . | sha256sum }}
3433
{{- if .Values.spec.hive.annotations }}
3534
{{ toYaml .Values.spec.hive.annotations | indent 8 }}
3635
{{- end }}
@@ -43,7 +42,8 @@ spec:
4342
{{- end }}
4443
containers:
4544
- name: metastore
46-
args: ["--service", "metastore"]
45+
command: ["/hive-scripts/entrypoint.sh"]
46+
args: ["/opt/hive/bin/hive", "--service", "metastore"]
4747
image: "{{ .Values.spec.hive.image.repository }}:{{ .Values.spec.hive.image.tag }}"
4848
imagePullPolicy: {{ .Values.spec.hive.image.pullPolicy }}
4949
ports:
@@ -52,10 +52,7 @@ spec:
5252
protocol: TCP
5353
env:
5454
- name: HIVE_LOGLEVEL
55-
valueFrom:
56-
configMapKeyRef:
57-
name: hive-server-config
58-
key: log-level
55+
value: {{ upper .Values.spec.hive.metastore.config.logLevel | quote}}
5956
{{ include "hive-env" . | indent 8 }}
6057
- name: MY_MEM_REQUEST
6158
valueFrom:
@@ -68,6 +65,14 @@ spec:
6865
containerName: metastore
6966
resource: limits.memory
7067
volumeMounts:
68+
- name: hive-config
69+
mountPath: /hive-config
70+
- name: hive-scripts
71+
mountPath: /hive-scripts
72+
{{- if .Values.spec.hive.config.useHdfsConfigMap }}
73+
- name: hdfs-config
74+
mountPath: /hadoop-config
75+
{{- end }}
7176
- name: hive-metastore-db-data
7277
mountPath: /var/lib/hive
7378
# openshift requires volumeMounts for VOLUMEs in a Dockerfile
@@ -90,6 +95,18 @@ spec:
9095
terminationGracePeriodSeconds: {{ .Values.spec.hive.terminationGracePeriodSeconds }}
9196
serviceAccount: hive
9297
volumes:
98+
- name: hive-config
99+
configMap:
100+
name: hive-config
101+
- name: hive-scripts
102+
configMap:
103+
name: hive-scripts
104+
defaultMode: 0555
105+
{{- if .Values.spec.hive.config.useHdfsConfigMap }}
106+
- name: hdfs-config
107+
configMap:
108+
name: {{ .Values.spec.hive.config.hdfsConfigMapName }}
109+
{{- end }}
93110
# these emptyDir volumes are necessary because Openshift requires VOLUMEs
94111
# in a Dockerfile have a corresponding volumeMount
95112
- name: hive-warehouse-empty

0 commit comments

Comments
 (0)