Merge remote-tracking branch 'origin/master' into release-0.9

Pull in hive images changes for the release
kube-reporting · Nov 5, 2018 · 2f4fa64 · 2f4fa64
2 parents a270e5c + 06297aa
commit 2f4fa64
Show file tree

Hide file tree

Showing 25 changed files with 269 additions and 618 deletions.
diff --git a/Makefile b/Makefile
@@ -49,7 +49,6 @@ endif
 DOCKER_COMMON_NAMES := \
 	reporting-operator \
 	metering-operator \
-	hive \
 	metering-e2e
 
 DOCKER_BUILD_NAMES = $(DOCKER_COMMON_NAMES)
@@ -60,7 +59,6 @@ DOCKER_BASE_URL := quay.io/coreos
 
 METERING_OPERATOR_IMAGE := $(DOCKER_BASE_URL)/metering-helm-operator
 REPORTING_OPERATOR_IMAGE := $(DOCKER_BASE_URL)/metering-reporting-operator
-HIVE_IMAGE := $(DOCKER_BASE_URL)/metering-hive
 METERING_E2E_IMAGE := $(DOCKER_BASE_URL)/metering-e2e
 
 GIT_SHA    := $(shell git rev-parse HEAD)
@@ -187,9 +185,6 @@ metering-e2e-docker-build: Dockerfile.e2e
 metering-operator-docker-build: Dockerfile.metering-operator
 	$(MAKE) docker-build DOCKERFILE=$< IMAGE_NAME=$(METERING_OPERATOR_IMAGE) DOCKER_BUILD_CONTEXT=$(ROOT_DIR)
 
-hive-docker-build: images/hive/Dockerfile
-	$(MAKE) docker-build DOCKERFILE=$< IMAGE_NAME=$(HIVE_IMAGE)
-
 # Update dependencies
 vendor: Gopkg.toml
 	dep ensure -v

diff --git a/charts/presto/templates/_helpers.tpl b/charts/presto/templates/_helpers.tpl
@@ -51,66 +51,6 @@ connector.name=jmx
 {{- end }}
 
 {{- define "hive-env" }}
-- name: CORE_CONF_fs_s3a_access_key
-  valueFrom:
-    secretKeyRef:
-      name: hive-common-secrets
-      key: aws-access-key-id
-      optional: true
-- name: CORE_CONF_fs_s3a_secret_key
-  valueFrom:
-    secretKeyRef:
-      name: hive-common-secrets
-      key: aws-secret-access-key
-      optional: true
-- name: CORE_CONF_fs_defaultFS
-  valueFrom:
-    configMapKeyRef:
-      name: hive-common-config
-      key: default-fs
-      optional: true
-- name: HIVE_SITE_CONF_hive_metastore_uris
-  valueFrom:
-    configMapKeyRef:
-      name: hive-common-config
-      key: metastore-uris
-- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionURL
-  valueFrom:
-    configMapKeyRef:
-      name: hive-common-config
-      key: db-connection-url
-- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName
-  valueFrom:
-    configMapKeyRef:
-      name: hive-common-config
-      key: db-connection-driver
-- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName
-  valueFrom:
-    secretKeyRef:
-      name: hive-common-secrets
-      key: db-connection-username
-      optional: true
-- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword
-  valueFrom:
-    secretKeyRef:
-      name: hive-common-secrets
-      key: db-connection-password
-      optional: true
-- name: HIVE_SITE_CONF_hive_metastore_schema_verification
-  valueFrom:
-    configMapKeyRef:
-      name: hive-common-config
-      key: enable-metastore-schema-verification
-- name: HIVE_SITE_CONF_datanucleus_schema_autoCreateAll
-  valueFrom:
-    configMapKeyRef:
-      name: hive-common-config
-      key: auto-create-metastore-schema
-- name: HIVE_SITE_CONF_hive_default_fileformat
-  valueFrom:
-    configMapKeyRef:
-      name: hive-common-config
-      key: default-file-format
 - name: MY_NODE_NAME
   valueFrom:
     fieldRef:
@@ -125,4 +65,16 @@ connector.name=jmx
       fieldPath: metadata.namespace
 - name: JAVA_MAX_MEM_RATIO
   value: "50"
+- name: AWS_ACCESS_KEY_ID
+  valueFrom:
+    secretKeyRef:
+      name: hive-secrets
+      key: aws-access-key-id
+      optional: true
+- name: AWS_SECRET_ACCESS_KEY
+  valueFrom:
+    secretKeyRef:
+      name: hive-secrets
+      key: aws-secret-access-key
+      optional: true
 {{- end }}
diff --git a/charts/presto/templates/hive-common-config.yaml b/charts/presto/templates/hive-common-config.yaml
diff --git a/charts/presto/templates/hive-configmap.yaml b/charts/presto/templates/hive-configmap.yaml
@@ -0,0 +1,144 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hive-config
+{{- block "extraMetadata" . }}
+{{- end }}
+data:
+  hive-site.xml: |
+    <configuration>
+      <property>
+        <name>hive.server2.enable.doAs</name>
+        <value>false</value>
+      </property>
+      <property>
+        <name>hive.server2.use.SSL</name>
+        <value>false</value>
+      </property>
+      <property>
+        <name>hive.server2.authentication</name>
+        <value>NOSASL</value>
+      </property>
+      <property>
+        <name>hive.metastore.uris</name>
+        <value>{{ .Values.spec.hive.config.metastoreURIs }}</value>
+      </property>
+      <property>
+        <name>javax.jdo.option.ConnectionURL</name>
+        <value>{{ .Values.spec.hive.config.dbConnectionURL }}</value>
+      </property>
+      <property>
+        <name>javax.jdo.option.ConnectionDriverName</name>
+        <value>{{ .Values.spec.hive.config.dbConnectionDriver }}</value>
+      </property>
+      <property>
+        <name>javax.jdo.option.ConnectionUserName</name>
+        <value>{{ .Values.spec.hive.config.dbConnectionUsername }}</value>
+      </property>
+      <property>
+        <name>javax.jdo.option.ConnectionPassword</name>
+        <value>{{ .Values.spec.hive.config.dbConnectionPassword }}</value>
+      </property>
+      <property>
+        <name>datanucleus.schema.autoCreateAll</name>
+        <value>{{ .Values.spec.hive.config.autoCreateMetastoreSchema }}</value>
+      </property>
+      <property>
+        <name>hive.metastore.schema.verification</name>
+        <value>{{ .Values.spec.hive.config.enableMetastoreSchemaVerification }}</value>
+      </property>
+      <property>
+        <name>hive.default.fileformat</name>
+        <value>{{ .Values.spec.hive.config.defaultFileFormat }}</value>
+      </property>
+    </configuration>
+
+
+  hive-log4j2.properties: |
+    status = INFO
+    name = HiveLog4j2
+    packages = org.apache.hadoop.hive.ql.log
+
+    # list of properties
+    property.hive.log.level = INFO
+    property.hive.root.logger = console
+    property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}
+    property.hive.log.file = hive.log
+
+    # list of all appenders
+    appenders = console
+
+    # console appender
+    appender.console.type = Console
+    appender.console.name = console
+    appender.console.target = SYSTEM_ERR
+    appender.console.layout.type = PatternLayout
+    appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
+
+    # list of all loggers
+    loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX
+
+    logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
+    logger.NIOServerCnxn.level = WARN
+
+    logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
+    logger.ClientCnxnSocketNIO.level = WARN
+
+    logger.DataNucleus.name = DataNucleus
+    logger.DataNucleus.level = ERROR
+
+    logger.Datastore.name = Datastore
+    logger.Datastore.level = ERROR
+
+    logger.JPOX.name = JPOX
+    logger.JPOX.level = ERROR
+
+    # root logger
+    rootLogger.level = ${sys:hive.log.level}
+    rootLogger.appenderRefs = root
+    rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}
+
+  hive-exec-log4j2.properties: |
+    status = INFO
+    name = HiveLog4j2
+    packages = org.apache.hadoop.hive.ql.log
+
+    # list of properties
+    property.hive.log.level = INFO
+    property.hive.root.logger = console
+    property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}
+    property.hive.log.file = hive.log
+
+    # list of all appenders
+    appenders = console
+
+    # console appender
+    appender.console.type = Console
+    appender.console.name = console
+    appender.console.target = SYSTEM_ERR
+    appender.console.layout.type = PatternLayout
+    appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
+
+    # list of all loggers
+    loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX
+
+    logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
+    logger.NIOServerCnxn.level = WARN
+
+    logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
+    logger.ClientCnxnSocketNIO.level = WARN
+
+    logger.DataNucleus.name = DataNucleus
+    logger.DataNucleus.level = ERROR
+
+    logger.Datastore.name = Datastore
+    logger.Datastore.level = ERROR
+
+    logger.JPOX.name = JPOX
+    logger.JPOX.level = ERROR
+
+    # root logger
+    rootLogger.level = ${sys:hive.log.level}
+    rootLogger.appenderRefs = root
+    rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}
+
diff --git a/charts/presto/templates/hive-metastore-config.yaml b/charts/presto/templates/hive-metastore-config.yaml
diff --git a/charts/presto/templates/hive-metastore-statefulset.yaml b/charts/presto/templates/hive-metastore-statefulset.yaml
@@ -28,9 +28,8 @@ spec:
 {{ toYaml .Values.spec.hive.labels | indent 8 }}
 {{- end }}
       annotations:
-        hive-common-configmap-hash: {{ include (print $.Template.BasePath "/hive-common-config.yaml") . | sha256sum }}
-        hive-common-secret-hash: {{ include (print $.Template.BasePath "/hive-common-secrets.yaml") . | sha256sum }}
-        hive-metastore-configmap-hash: {{ include (print $.Template.BasePath "/hive-metastore-config.yaml") . | sha256sum }}
+        hive-configmap-hash: {{ include (print $.Template.BasePath "/hive-configmap.yaml") . | sha256sum }}
+        hive-scripts-hash: {{ include (print $.Template.BasePath "/hive-scripts-configmap.yaml") . | sha256sum }}
 {{- if .Values.spec.hive.annotations }}
 {{ toYaml .Values.spec.hive.annotations | indent 8 }}
 {{- end }}
@@ -43,7 +42,8 @@ spec:
 {{- end }}
       containers:
       - name: metastore
-        args: ["--service", "metastore"]
+        command: ["/hive-scripts/entrypoint.sh"]
+        args: ["/opt/hive/bin/hive", "--service", "metastore"]
         image: "{{ .Values.spec.hive.image.repository }}:{{ .Values.spec.hive.image.tag }}"
         imagePullPolicy: {{ .Values.spec.hive.image.pullPolicy }}
         ports:
@@ -52,10 +52,7 @@ spec:
           protocol: TCP
         env:
         - name: HIVE_LOGLEVEL
-          valueFrom:
-            configMapKeyRef:
-              name: hive-server-config
-              key: log-level
+          value: {{ upper .Values.spec.hive.metastore.config.logLevel | quote}}
 {{ include "hive-env" . | indent 8 }}
         - name: MY_MEM_REQUEST
           valueFrom:
@@ -68,6 +65,14 @@ spec:
               containerName: metastore
               resource: limits.memory
         volumeMounts:
+        - name: hive-config
+          mountPath: /hive-config
+        - name: hive-scripts
+          mountPath: /hive-scripts
+{{- if .Values.spec.hive.config.useHdfsConfigMap }}
+        - name: hdfs-config
+          mountPath: /hadoop-config
+{{- end }}
         - name: hive-metastore-db-data
           mountPath: /var/lib/hive
         # openshift requires volumeMounts for VOLUMEs in a Dockerfile
@@ -90,6 +95,18 @@ spec:
       terminationGracePeriodSeconds: {{ .Values.spec.hive.terminationGracePeriodSeconds }}
       serviceAccount: hive
       volumes:
+      - name: hive-config
+        configMap:
+          name: hive-config
+      - name: hive-scripts
+        configMap:
+          name: hive-scripts
+          defaultMode: 0555
+{{- if .Values.spec.hive.config.useHdfsConfigMap }}
+      - name: hdfs-config
+        configMap:
+          name: {{ .Values.spec.hive.config.hdfsConfigMapName }}
+{{- end }}
       # these emptyDir volumes are necessary because Openshift requires VOLUMEs
       # in a Dockerfile have a corresponding volumeMount
       - name: hive-warehouse-empty