diff --git a/Makefile b/Makefile index a80a23e8a..f67acf5fc 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,6 @@ endif DOCKER_COMMON_NAMES := \ reporting-operator \ metering-operator \ - hive \ metering-e2e DOCKER_BUILD_NAMES = $(DOCKER_COMMON_NAMES) @@ -60,7 +59,6 @@ DOCKER_BASE_URL := quay.io/coreos METERING_OPERATOR_IMAGE := $(DOCKER_BASE_URL)/metering-helm-operator REPORTING_OPERATOR_IMAGE := $(DOCKER_BASE_URL)/metering-reporting-operator -HIVE_IMAGE := $(DOCKER_BASE_URL)/metering-hive METERING_E2E_IMAGE := $(DOCKER_BASE_URL)/metering-e2e GIT_SHA := $(shell git rev-parse HEAD) @@ -187,9 +185,6 @@ metering-e2e-docker-build: Dockerfile.e2e metering-operator-docker-build: Dockerfile.metering-operator $(MAKE) docker-build DOCKERFILE=$< IMAGE_NAME=$(METERING_OPERATOR_IMAGE) DOCKER_BUILD_CONTEXT=$(ROOT_DIR) -hive-docker-build: images/hive/Dockerfile - $(MAKE) docker-build DOCKERFILE=$< IMAGE_NAME=$(HIVE_IMAGE) - # Update dependencies vendor: Gopkg.toml dep ensure -v diff --git a/charts/presto/templates/_helpers.tpl b/charts/presto/templates/_helpers.tpl index cdb3692b6..30f6b01ef 100644 --- a/charts/presto/templates/_helpers.tpl +++ b/charts/presto/templates/_helpers.tpl @@ -51,66 +51,6 @@ connector.name=jmx {{- end }} {{- define "hive-env" }} -- name: CORE_CONF_fs_s3a_access_key - valueFrom: - secretKeyRef: - name: hive-common-secrets - key: aws-access-key-id - optional: true -- name: CORE_CONF_fs_s3a_secret_key - valueFrom: - secretKeyRef: - name: hive-common-secrets - key: aws-secret-access-key - optional: true -- name: CORE_CONF_fs_defaultFS - valueFrom: - configMapKeyRef: - name: hive-common-config - key: default-fs - optional: true -- name: HIVE_SITE_CONF_hive_metastore_uris - valueFrom: - configMapKeyRef: - name: hive-common-config - key: metastore-uris -- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionURL - valueFrom: - configMapKeyRef: - name: hive-common-config - key: db-connection-url -- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName - valueFrom: - configMapKeyRef: - name: hive-common-config - key: db-connection-driver -- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName - valueFrom: - secretKeyRef: - name: hive-common-secrets - key: db-connection-username - optional: true -- name: HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword - valueFrom: - secretKeyRef: - name: hive-common-secrets - key: db-connection-password - optional: true -- name: HIVE_SITE_CONF_hive_metastore_schema_verification - valueFrom: - configMapKeyRef: - name: hive-common-config - key: enable-metastore-schema-verification -- name: HIVE_SITE_CONF_datanucleus_schema_autoCreateAll - valueFrom: - configMapKeyRef: - name: hive-common-config - key: auto-create-metastore-schema -- name: HIVE_SITE_CONF_hive_default_fileformat - valueFrom: - configMapKeyRef: - name: hive-common-config - key: default-file-format - name: MY_NODE_NAME valueFrom: fieldRef: @@ -125,4 +65,16 @@ connector.name=jmx fieldPath: metadata.namespace - name: JAVA_MAX_MEM_RATIO value: "50" +- name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: hive-secrets + key: aws-access-key-id + optional: true +- name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: hive-secrets + key: aws-secret-access-key + optional: true {{- end }} diff --git a/charts/presto/templates/hive-common-config.yaml b/charts/presto/templates/hive-common-config.yaml deleted file mode 100644 index 9953400c2..000000000 --- a/charts/presto/templates/hive-common-config.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: hive-common-config -{{- block "extraMetadata" . }} -{{- end }} -data: -{{- if .Values.spec.hive.config.defaultfs }} - default-fs: {{ .Values.spec.hive.config.defaultfs | quote}} -{{- end }} - db-connection-url: {{ .Values.spec.hive.config.dbConnectionURL | quote}} - db-connection-driver: {{ .Values.spec.hive.config.dbConnectionDriver | quote}} - enable-metastore-schema-verification: {{ .Values.spec.hive.config.enableMetastoreSchemaVerification | quote}} - auto-create-metastore-schema: {{ .Values.spec.hive.config.autoCreateMetastoreSchema | quote}} - default-file-format: {{ .Values.spec.hive.config.defaultFileFormat | quote}} - metastore-uris: {{ .Values.spec.hive.config.metastoreURIs | quote}} - diff --git a/charts/presto/templates/hive-configmap.yaml b/charts/presto/templates/hive-configmap.yaml new file mode 100644 index 000000000..1b34d49f5 --- /dev/null +++ b/charts/presto/templates/hive-configmap.yaml @@ -0,0 +1,144 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hive-config +{{- block "extraMetadata" . }} +{{- end }} +data: + hive-site.xml: | + + + hive.server2.enable.doAs + false + + + hive.server2.use.SSL + false + + + hive.server2.authentication + NOSASL + + + hive.metastore.uris + {{ .Values.spec.hive.config.metastoreURIs }} + + + javax.jdo.option.ConnectionURL + {{ .Values.spec.hive.config.dbConnectionURL }} + + + javax.jdo.option.ConnectionDriverName + {{ .Values.spec.hive.config.dbConnectionDriver }} + + + javax.jdo.option.ConnectionUserName + {{ .Values.spec.hive.config.dbConnectionUsername }} + + + javax.jdo.option.ConnectionPassword + {{ .Values.spec.hive.config.dbConnectionPassword }} + + + datanucleus.schema.autoCreateAll + {{ .Values.spec.hive.config.autoCreateMetastoreSchema }} + + + hive.metastore.schema.verification + {{ .Values.spec.hive.config.enableMetastoreSchemaVerification }} + + + hive.default.fileformat + {{ .Values.spec.hive.config.defaultFileFormat }} + + + + + hive-log4j2.properties: | + status = INFO + name = HiveLog4j2 + packages = org.apache.hadoop.hive.ql.log + + # list of properties + property.hive.log.level = INFO + property.hive.root.logger = console + property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} + property.hive.log.file = hive.log + + # list of all appenders + appenders = console + + # console appender + appender.console.type = Console + appender.console.name = console + appender.console.target = SYSTEM_ERR + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + + # list of all loggers + loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX + + logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn + logger.NIOServerCnxn.level = WARN + + logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO + logger.ClientCnxnSocketNIO.level = WARN + + logger.DataNucleus.name = DataNucleus + logger.DataNucleus.level = ERROR + + logger.Datastore.name = Datastore + logger.Datastore.level = ERROR + + logger.JPOX.name = JPOX + logger.JPOX.level = ERROR + + # root logger + rootLogger.level = ${sys:hive.log.level} + rootLogger.appenderRefs = root + rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} + + hive-exec-log4j2.properties: | + status = INFO + name = HiveLog4j2 + packages = org.apache.hadoop.hive.ql.log + + # list of properties + property.hive.log.level = INFO + property.hive.root.logger = console + property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} + property.hive.log.file = hive.log + + # list of all appenders + appenders = console + + # console appender + appender.console.type = Console + appender.console.name = console + appender.console.target = SYSTEM_ERR + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n + + # list of all loggers + loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX + + logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn + logger.NIOServerCnxn.level = WARN + + logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO + logger.ClientCnxnSocketNIO.level = WARN + + logger.DataNucleus.name = DataNucleus + logger.DataNucleus.level = ERROR + + logger.Datastore.name = Datastore + logger.Datastore.level = ERROR + + logger.JPOX.name = JPOX + logger.JPOX.level = ERROR + + # root logger + rootLogger.level = ${sys:hive.log.level} + rootLogger.appenderRefs = root + rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} + diff --git a/charts/presto/templates/hive-metastore-config.yaml b/charts/presto/templates/hive-metastore-config.yaml deleted file mode 100644 index bcce31a77..000000000 --- a/charts/presto/templates/hive-metastore-config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: hive-metastore-config -{{- block "extraMetadata" . }} -{{- end }} -data: - log-level: {{ upper .Values.spec.hive.metastore.config.logLevel | quote}} diff --git a/charts/presto/templates/hive-metastore-statefulset.yaml b/charts/presto/templates/hive-metastore-statefulset.yaml index c5fa3ffe8..906883176 100644 --- a/charts/presto/templates/hive-metastore-statefulset.yaml +++ b/charts/presto/templates/hive-metastore-statefulset.yaml @@ -28,9 +28,8 @@ spec: {{ toYaml .Values.spec.hive.labels | indent 8 }} {{- end }} annotations: - hive-common-configmap-hash: {{ include (print $.Template.BasePath "/hive-common-config.yaml") . | sha256sum }} - hive-common-secret-hash: {{ include (print $.Template.BasePath "/hive-common-secrets.yaml") . | sha256sum }} - hive-metastore-configmap-hash: {{ include (print $.Template.BasePath "/hive-metastore-config.yaml") . | sha256sum }} + hive-configmap-hash: {{ include (print $.Template.BasePath "/hive-configmap.yaml") . | sha256sum }} + hive-scripts-hash: {{ include (print $.Template.BasePath "/hive-scripts-configmap.yaml") . | sha256sum }} {{- if .Values.spec.hive.annotations }} {{ toYaml .Values.spec.hive.annotations | indent 8 }} {{- end }} @@ -43,7 +42,8 @@ spec: {{- end }} containers: - name: metastore - args: ["--service", "metastore"] + command: ["/hive-scripts/entrypoint.sh"] + args: ["/opt/hive/bin/hive", "--service", "metastore"] image: "{{ .Values.spec.hive.image.repository }}:{{ .Values.spec.hive.image.tag }}" imagePullPolicy: {{ .Values.spec.hive.image.pullPolicy }} ports: @@ -52,10 +52,7 @@ spec: protocol: TCP env: - name: HIVE_LOGLEVEL - valueFrom: - configMapKeyRef: - name: hive-server-config - key: log-level + value: {{ upper .Values.spec.hive.metastore.config.logLevel | quote}} {{ include "hive-env" . | indent 8 }} - name: MY_MEM_REQUEST valueFrom: @@ -68,6 +65,14 @@ spec: containerName: metastore resource: limits.memory volumeMounts: + - name: hive-config + mountPath: /hive-config + - name: hive-scripts + mountPath: /hive-scripts +{{- if .Values.spec.hive.config.useHdfsConfigMap }} + - name: hdfs-config + mountPath: /hadoop-config +{{- end }} - name: hive-metastore-db-data mountPath: /var/lib/hive # openshift requires volumeMounts for VOLUMEs in a Dockerfile @@ -90,6 +95,18 @@ spec: terminationGracePeriodSeconds: {{ .Values.spec.hive.terminationGracePeriodSeconds }} serviceAccount: hive volumes: + - name: hive-config + configMap: + name: hive-config + - name: hive-scripts + configMap: + name: hive-scripts + defaultMode: 0555 +{{- if .Values.spec.hive.config.useHdfsConfigMap }} + - name: hdfs-config + configMap: + name: {{ .Values.spec.hive.config.hdfsConfigMapName }} +{{- end }} # these emptyDir volumes are necessary because Openshift requires VOLUMEs # in a Dockerfile have a corresponding volumeMount - name: hive-warehouse-empty diff --git a/charts/presto/templates/hive-scripts-configmap.yaml b/charts/presto/templates/hive-scripts-configmap.yaml new file mode 100644 index 000000000..1ffaec0b3 --- /dev/null +++ b/charts/presto/templates/hive-scripts-configmap.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hive-scripts +{{- block "extraMetadata" . }} +{{- end }} +data: + entrypoint.sh: | + #!/bin/bash -e + + max_memory() { + local memory_limit=$1 + local ratio=${JAVA_MAX_MEM_RATIO:-50} + echo "${memory_limit} ${ratio} 1048576" | awk '{printf "%d\n" , ($1*$2)/(100*$3) + 0.5}' + } + + # Check for container memory limits/request and use it to set JVM Heap size. + # Defaults to 50% of the limit/request value. + if [ -n "$MY_MEM_LIMIT" ]; then + export HADOOP_HEAPSIZE="$( max_memory $MY_MEM_LIMIT )" + elif [ -n "$MY_MEM_REQUEST" ]; then + export HADOOP_HEAPSIZE="$( max_memory $MY_MEM_REQUEST )" + fi + + if [ -z "$HADOOP_HEAPSIZE" ]; then + echo "Unable to automatically set HADOOP_HEAPSIZE" + else + echo "Setting HADOOP_HEAPSIZE to ${HADOOP_HEAPSIZE}M" + fi + + # add UID to /etc/passwd if missing + if ! whoami &> /dev/null; then + if [ -w /etc/passwd ]; then + echo "${USER_NAME:-hadoop}:x:$(id -u):0:${USER_NAME:-hadoop} user:${HOME}:/sbin/nologin" >> /etc/passwd + fi + fi + + # symlink our configuration files to the correct location + if [ -f /hadoop-config/core-site.xml ]; then + ln -s -f /hadoop-config/core-site.xml /etc/hadoop/core-site.xml + else + echo "/hadoop-config/core-site.xml doesnt exist, skipping symlink" + fi + if [ -f /hadoop-config/hdfs-site.xml ]; then + ln -s -f /hadoop-config/hdfs-site.xml /etc/hadoop/hdfs-site.xml + else + echo "/hadoop-config/hdfs-site.xml doesnt exist, skipping symlink" + fi + ln -s -f /hive-config/hive-site.xml $HIVE_HOME/conf/hive-site.xml + ln -s -f /hive-config/hive-log4j2.properties $HIVE_HOME/conf/hive-log4j2.properties + ln -s -f /hive-config/hive-exec-log4j2.properties $HIVE_HOME/conf/hive-exec-log4j2.properties + + export HIVE_LOGLEVEL="${HIVE_LOGLEVEL:-INFO}" + export HIVE_METASTORE_HADOOP_OPTS=" -Dhive.log.level=${HIVE_LOGLEVEL} " + export HIVE_OPTS="$HIVE_OPTS --hiveconf hive.root.logger=${HIVE_LOGLEVEL},console " + + exec $@ diff --git a/charts/presto/templates/hive-common-secrets.yaml b/charts/presto/templates/hive-secrets.yaml similarity index 50% rename from charts/presto/templates/hive-common-secrets.yaml rename to charts/presto/templates/hive-secrets.yaml index 65f7d6ad3..675cf678b 100644 --- a/charts/presto/templates/hive-common-secrets.yaml +++ b/charts/presto/templates/hive-secrets.yaml @@ -1,16 +1,10 @@ apiVersion: v1 kind: Secret metadata: - name: hive-common-secrets + name: hive-secrets {{- block "extraMetadata" . }} {{- end }} data: -{{- if .Values.spec.hive.config.dbConnectionUsername }} - db-connection-username: {{ .Values.spec.hive.config.dbConnectionUsername | b64enc | quote }} -{{- end }} -{{- if .Values.spec.hive.config.dbConnectionPassword }} - db-connection-password: {{ .Values.spec.hive.config.dbConnectionPassword | b64enc | quote }} -{{- end }} {{- if .Values.spec.config.awsAccessKeyID }} aws-access-key-id: {{ .Values.spec.config.awsAccessKeyID | b64enc | quote}} {{- end}} diff --git a/charts/presto/templates/hive-server-config.yaml b/charts/presto/templates/hive-server-config.yaml deleted file mode 100644 index c58adddff..000000000 --- a/charts/presto/templates/hive-server-config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: hive-server-config -{{- block "extraMetadata" . }} -{{- end }} -data: - log-level: {{ upper .Values.spec.hive.server.config.logLevel | quote}} diff --git a/charts/presto/templates/hive-server-statefulset.yaml b/charts/presto/templates/hive-server-statefulset.yaml index e30ad0d9d..72237283c 100644 --- a/charts/presto/templates/hive-server-statefulset.yaml +++ b/charts/presto/templates/hive-server-statefulset.yaml @@ -28,9 +28,9 @@ spec: {{ toYaml .Values.spec.hive.labels | indent 8 }} {{- end }} annotations: - hive-common-configmap-hash: {{ include (print $.Template.BasePath "/hive-common-config.yaml") . | sha256sum }} - hive-common-secret-hash: {{ include (print $.Template.BasePath "/hive-common-secrets.yaml") . | sha256sum }} - hive-server-configmap-hash: {{ include (print $.Template.BasePath "/hive-server-config.yaml") . | sha256sum }} + hive-configmap-hash: {{ include (print $.Template.BasePath "/hive-configmap.yaml") . | sha256sum }} + hive-scripts-hash: {{ include (print $.Template.BasePath "/hive-scripts-configmap.yaml") . | sha256sum }} + hive-secrets-hash: {{ include (print $.Template.BasePath "/hive-secrets.yaml") . | sha256sum }} {{- if .Values.spec.hive.annotations }} {{ toYaml .Values.spec.hive.annotations | indent 8 }} {{- end }} @@ -43,7 +43,8 @@ spec: {{- end }} containers: - name: hiveserver2 - args: ["--service", "hiveserver2"] + command: ["/hive-scripts/entrypoint.sh"] + args: ["/opt/hive/bin/hive", "--service", "hiveserver2"] image: "{{ .Values.spec.hive.image.repository }}:{{ .Values.spec.hive.image.tag }}" imagePullPolicy: {{ .Values.spec.hive.image.pullPolicy }} ports: @@ -56,10 +57,7 @@ spec: terminationMessagePath: /dev/termination-log env: - name: HIVE_LOGLEVEL - valueFrom: - configMapKeyRef: - name: hive-server-config - key: log-level + value: {{ upper .Values.spec.hive.server.config.logLevel | quote}} {{ include "hive-env" . | indent 8 }} - name: MY_MEM_REQUEST valueFrom: @@ -72,6 +70,14 @@ spec: containerName: hiveserver2 resource: limits.memory volumeMounts: + - name: hive-config + mountPath: /hive-config + - name: hive-scripts + mountPath: /hive-scripts +{{- if .Values.spec.hive.config.useHdfsConfigMap }} + - name: hdfs-config + mountPath: /hadoop-config +{{- end}} # openshift requires volumeMounts for VOLUMEs in a Dockerfile - name: hive-metastore-db-data mountPath: /var/lib/hive @@ -94,6 +100,18 @@ spec: terminationGracePeriodSeconds: {{ .Values.spec.hive.terminationGracePeriodSeconds }} serviceAccount: hive volumes: + - name: hive-config + configMap: + name: hive-config + - name: hive-scripts + configMap: + name: hive-scripts + defaultMode: 0555 +{{- if .Values.spec.hive.config.useHdfsConfigMap }} + - name: hdfs-config + configMap: + name: {{ .Values.spec.hive.config.hdfsConfigMapName }} +{{- end }} # these emptyDir volumes are necessary because Openshift requires VOLUMEs # in a Dockerfile have a corresponding volumeMount - name: hive-warehouse-empty diff --git a/charts/presto/values.yaml b/charts/presto/values.yaml index 5a733cace..8ad0ce881 100644 --- a/charts/presto/values.yaml +++ b/charts/presto/values.yaml @@ -76,8 +76,8 @@ spec: hive: image: - repository: quay.io/coreos/metering-hive - tag: latest + repository: quay.io/coreos/hive + tag: metering-2.3.3 pullPolicy: Always config: @@ -91,6 +91,8 @@ spec: defaultFileFormat: "orc" defaultCompression: "zlib" metastoreURIs: "thrift://hive-metastore:9083" + useHdfsConfigMap: true + hdfsConfigMapName: hdfs-config securityContext: runAsNonRoot: true diff --git a/hack/deploy-custom.sh b/hack/deploy-custom.sh index 9e4824425..c303d9381 100755 --- a/hack/deploy-custom.sh +++ b/hack/deploy-custom.sh @@ -70,8 +70,6 @@ spec: terminationGracePeriodSeconds: ${TERMINATION_GRACE_PERIOD_SECONDS} hive: terminationGracePeriodSeconds: ${TERMINATION_GRACE_PERIOD_SECONDS} - image: - tag: ${DEPLOY_TAG} hdfs: spec: diff --git a/hack/render-metering-chart-override-values.sh b/hack/render-metering-chart-override-values.sh index ee48792b9..7f45f3f90 100755 --- a/hack/render-metering-chart-override-values.sh +++ b/hack/render-metering-chart-override-values.sh @@ -9,9 +9,4 @@ reporting-operator: spec: image: tag: $1 -presto: - spec: - hive: - image: - tag: $1 EOF diff --git a/images/hive/Dockerfile b/images/hive/Dockerfile deleted file mode 100644 index 23a6bd4c9..000000000 --- a/images/hive/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -FROM quay.io/coreos/metering-hadoop:latest - -ENV HIVE_VERSION=2.3.3 -ENV HIVE_HOME=/opt/hive -ENV PATH=$HIVE_HOME/bin:$PATH - -RUN mkdir -p /opt -WORKDIR /opt - -USER root -# Install Hive -RUN set -x \ - && curl -fSLs https://apache.osuosl.org/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz | tar -xz && mv apache-hive-$HIVE_VERSION-bin hive - -ENV HADOOP_CLASSPATH /opt/hive/hcatalog/share/hcatalog/*:${HADOOP_CLASSPATH} -ENV HIVE_AUX_JARS_PATH /usr/hdp/current/hive-server2/auxlib - -ENV POSTGRESQL_JDBC_JAR postgresql-42.2.2.jar -# Using mysql-connector-java-8.0.11 resulted in hive schema creation failing due to incorrect syntax. -ENV MYSQL_JDBC_VERSION mysql-connector-java-5.1.46 -ENV MYSQL_JDBC_JAR $MYSQL_JDBC_VERSION.jar - -# Install PostgreSQL JDBC -RUN set -x \ - && curl -fSLs -o "$HIVE_HOME/lib/$POSTGRESQL_JDBC_JAR" "https://jdbc.postgresql.org/download/$POSTGRESQL_JDBC_JAR" - -# Install MySQL JDBC -RUN set -x \ - && curl -fSLs "https://dev.mysql.com/get/Downloads/Connector-J/$MYSQL_JDBC_VERSION.tar.gz" | tar -zx --strip-components=1 -C "$HIVE_HOME/lib" "$MYSQL_JDBC_VERSION/$MYSQL_JDBC_JAR" - -# Configure JSON SerDe and AWS Jars -RUN mkdir -p /usr/hdp/current/hive-server2/auxlib && ln -s ${HADOOP_HOME}/share/hadoop/tools/lib/*aws* /opt/hive/lib -COPY json-serde-1.3.8-jar-with-dependencies.jar /usr/hdp/current/hive-server2/auxlib - -COPY metastore.sh /opt/hive/bin/ext/metastore.sh -COPY hive-entrypoint.sh /usr/local/bin/hive-entrypoint.sh -COPY conf/* $HIVE_HOME/conf/ - -RUN \ - mkdir -p /var/lib/hive /user/hive/warehouse && \ - chown -R 1002:0 /opt /var/lib/hive /user/hive/warehouse && \ - chmod -R 770 /opt /var/lib/hive /user/hive/warehouse /etc/passwd - -VOLUME /user/hive/warehouse /var/lib/hive -ENTRYPOINT ["hive-entrypoint.sh"] - -USER 1002 diff --git a/images/hive/conf/beeline-log4j2.properties b/images/hive/conf/beeline-log4j2.properties deleted file mode 100644 index 103d72253..000000000 --- a/images/hive/conf/beeline-log4j2.properties +++ /dev/null @@ -1,45 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -status = INFO -name = BeelineLog4j2 -packages = org.apache.hadoop.hive.ql.log - -# list of properties -property.hive.log.level = WARN -property.hive.root.logger = console - -# list of all appenders -appenders = console - -# console appender -appender.console.type = Console -appender.console.name = console -appender.console.target = SYSTEM_ERR -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n - -# list of all loggers -loggers = HiveConnection - -# HiveConnection logs useful info for dynamic service discovery -logger.HiveConnection.name = org.apache.hive.jdbc.HiveConnection -logger.HiveConnection.level = INFO - -# root logger -rootLogger.level = ${sys:hive.log.level} -rootLogger.appenderRefs = root -rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/images/hive/conf/hive-env.sh b/images/hive/conf/hive-env.sh deleted file mode 100644 index 91a2905d0..000000000 --- a/images/hive/conf/hive-env.sh +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Set Hive and Hadoop environment variables here. These variables can be used -# to control the execution of Hive. It should be used by admins to configure -# the Hive installation (so that users do not have to set environment variables -# or set command line parameters to get correct behavior). -# -# The hive service being invoked (CLI/HWI etc.) is available via the environment -# variable SERVICE - - -# Hive Client memory usage can be an issue if a large number of clients -# are running at the same time. The flags below have been useful in -# reducing memory usage: -# -# if [ "$SERVICE" = "cli" ]; then -# if [ -z "$DEBUG" ]; then -# export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" -# else -# export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" -# fi -# fi - -# The heap size of the jvm stared by hive shell script can be controlled via: -# -# export HADOOP_HEAPSIZE=1024 -# -# Larger heap size may be required when running queries over large number of files or partitions. -# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be -# appropriate for hive server (hwi etc). - - -# Set HADOOP_HOME to point to a specific hadoop install directory -# HADOOP_HOME=${bin}/../../hadoop - -# Hive Configuration Directory can be controlled by: -# export HIVE_CONF_DIR= - -# Folder containing extra ibraries required for hive compilation/execution can be controlled by: -# export HIVE_AUX_JARS_PATH= diff --git a/images/hive/conf/hive-exec-log4j2.properties b/images/hive/conf/hive-exec-log4j2.properties deleted file mode 100644 index 3cf7cbc34..000000000 --- a/images/hive/conf/hive-exec-log4j2.properties +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -status = INFO -name = HiveExecLog4j2 -packages = org.apache.hadoop.hive.ql.log - -# list of properties -property.hive.log.level = INFO -property.hive.root.logger = console -property.hive.query.id = hadoop -property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} -property.hive.log.file = ${sys:hive.query.id}.log - -# list of all appenders -appenders = console - -# console appender -appender.console.type = Console -appender.console.name = console -appender.console.target = SYSTEM_ERR -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n - -# list of all loggers -loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX - -logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn -logger.NIOServerCnxn.level = WARN - -logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO -logger.ClientCnxnSocketNIO.level = WARN - -logger.DataNucleus.name = DataNucleus -logger.DataNucleus.level = ERROR - -logger.Datastore.name = Datastore -logger.Datastore.level = ERROR - -logger.JPOX.name = JPOX -logger.JPOX.level = ERROR - -# root logger -rootLogger.level = ${sys:hive.log.level} -rootLogger.appenderRefs = root -rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/images/hive/conf/hive-log4j2.properties b/images/hive/conf/hive-log4j2.properties deleted file mode 100644 index 1c0d1032d..000000000 --- a/images/hive/conf/hive-log4j2.properties +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -status = INFO -name = HiveLog4j2 -packages = org.apache.hadoop.hive.ql.log - -# list of properties -property.hive.log.level = INFO -property.hive.root.logger = console -property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} -property.hive.log.file = hive.log - -# list of all appenders -appenders = console - -# console appender -appender.console.type = Console -appender.console.name = console -appender.console.target = SYSTEM_ERR -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n - -# list of all loggers -loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX - -logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn -logger.NIOServerCnxn.level = WARN - -logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO -logger.ClientCnxnSocketNIO.level = WARN - -logger.DataNucleus.name = DataNucleus -logger.DataNucleus.level = ERROR - -logger.Datastore.name = Datastore -logger.Datastore.level = ERROR - -logger.JPOX.name = JPOX -logger.JPOX.level = ERROR - -# root logger -rootLogger.level = ${sys:hive.log.level} -rootLogger.appenderRefs = root -rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} diff --git a/images/hive/conf/hive-site.xml b/images/hive/conf/hive-site.xml deleted file mode 100644 index 2c05f38a4..000000000 --- a/images/hive/conf/hive-site.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - hive.server2.enable.doAs - false - - - hive.server2.use.SSL - false - - - hive.server2.authentication - NOSASL - - - datanucleus.schema.autoCreateAll - true - - diff --git a/images/hive/conf/ivysettings.xml b/images/hive/conf/ivysettings.xml deleted file mode 100644 index 47f29b26e..000000000 --- a/images/hive/conf/ivysettings.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/images/hive/conf/llap-daemon-log4j2.properties b/images/hive/conf/llap-daemon-log4j2.properties deleted file mode 100644 index 5051ca505..000000000 --- a/images/hive/conf/llap-daemon-log4j2.properties +++ /dev/null @@ -1,93 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -status = INFO -name = LlapDaemonLog4j2 -packages = org.apache.hadoop.hive.ql.log - -# list of properties -property.llap.daemon.log.level = INFO -property.llap.daemon.root.logger = console -property.llap.daemon.log.dir = . -property.llap.daemon.log.file = llapdaemon.log -property.llap.daemon.historylog.file = llapdaemon_history.log -property.llap.daemon.log.maxfilesize = 256MB -property.llap.daemon.log.maxbackupindex = 20 - -# list of all appenders -appenders = console, RFA, HISTORYAPPENDER - -# console appender -appender.console.type = Console -appender.console.name = console -appender.console.target = SYSTEM_ERR -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t%x] %p %c{2} : %m%n - -# rolling file appender -appender.RFA.type = RollingFile -appender.RFA.name = RFA -appender.RFA.fileName = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.log.file} -appender.RFA.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.log.file}_%i -appender.RFA.layout.type = PatternLayout -appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t%x]: %c{2} (%F:%M(%L)) - %m%n -appender.RFA.policies.type = Policies -appender.RFA.policies.size.type = SizeBasedTriggeringPolicy -appender.RFA.policies.size.size = ${sys:llap.daemon.log.maxfilesize} -appender.RFA.strategy.type = DefaultRolloverStrategy -appender.RFA.strategy.max = ${sys:llap.daemon.log.maxbackupindex} - -# history file appender -appender.HISTORYAPPENDER.type = RollingFile -appender.HISTORYAPPENDER.name = HISTORYAPPENDER -appender.HISTORYAPPENDER.fileName = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file} -appender.HISTORYAPPENDER.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file}_%i -appender.HISTORYAPPENDER.layout.type = PatternLayout -appender.HISTORYAPPENDER.layout.pattern = %m%n -appender.HISTORYAPPENDER.policies.type = Policies -appender.HISTORYAPPENDER.policies.size.type = SizeBasedTriggeringPolicy -appender.HISTORYAPPENDER.policies.size.size = ${sys:llap.daemon.log.maxfilesize} -appender.HISTORYAPPENDER.strategy.type = DefaultRolloverStrategy -appender.HISTORYAPPENDER.strategy.max = ${sys:llap.daemon.log.maxbackupindex} - -# list of all loggers -loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HistoryLogger - -logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn -logger.NIOServerCnxn.level = WARN - -logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO -logger.ClientCnxnSocketNIO.level = WARN - -logger.DataNucleus.name = DataNucleus -logger.DataNucleus.level = ERROR - -logger.Datastore.name = Datastore -logger.Datastore.level = ERROR - -logger.JPOX.name = JPOX -logger.JPOX.level = ERROR - -logger.HistoryLogger.name = org.apache.hadoop.hive.llap.daemon.HistoryLogger -logger.HistoryLogger.level = INFO -logger.HistoryLogger.additivity = false -logger.HistoryLogger.appenderRefs = HistoryAppender -logger.HistoryLogger.appenderRef.HistoryAppender.ref = HISTORYAPPENDER - -# root logger -rootLogger.level = ${sys:llap.daemon.log.level} -rootLogger.appenderRefs = root -rootLogger.appenderRef.root.ref = ${sys:llap.daemon.root.logger} diff --git a/images/hive/hive-entrypoint.sh b/images/hive/hive-entrypoint.sh deleted file mode 100755 index 49c000005..000000000 --- a/images/hive/hive-entrypoint.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -export HIVE_LOGLEVEL="${HIVE_LOGLEVEL:-INFO}" -export HIVE_METASTORE_HADOOP_OPTS=" -Dhive.log.level=${HIVE_LOGLEVEL} " -export HIVE_OPTS="$HIVE_OPTS --hiveconf hive.root.logger=${HIVE_LOGLEVEL},console " - -exec /usr/local/bin/entrypoint.sh /opt/hive/bin/hive "$@" diff --git a/images/hive/json-serde-1.3.8-jar-with-dependencies.jar b/images/hive/json-serde-1.3.8-jar-with-dependencies.jar deleted file mode 100644 index 58b2920f4..000000000 Binary files a/images/hive/json-serde-1.3.8-jar-with-dependencies.jar and /dev/null differ diff --git a/images/hive/metastore.sh b/images/hive/metastore.sh deleted file mode 100644 index cc08c9569..000000000 --- a/images/hive/metastore.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -THISSERVICE=metastore -export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} " - -metastore() { - echo "$(timestamp): Starting Hive Metastore Server" - CLASS=org.apache.hadoop.hive.metastore.HiveMetaStore - if $cygwin; then - HIVE_LIB=`cygpath -w "$HIVE_LIB"` - fi - JAR=${HIVE_LIB}/hive-metastore-*.jar - - # hadoop 20 or newer - skip the aux_jars option and hiveconf - - export HADOOP_CLIENT_OPTS=" -Dproc_metastore $HADOOP_CLIENT_OPTS " - export HADOOP_OPTS="$HIVE_METASTORE_HADOOP_OPTS $HADOOP_OPTS" - exec $HADOOP jar $JAR $CLASS "$@" -} - -metastore_help() { - metastore -h -} - -timestamp() -{ - date +"%Y-%m-%d %T" -} diff --git a/manifests/metering-config/latest-versions.yaml b/manifests/metering-config/latest-versions.yaml index b2433e693..1501b6d14 100644 --- a/manifests/metering-config/latest-versions.yaml +++ b/manifests/metering-config/latest-versions.yaml @@ -8,10 +8,3 @@ spec: image: repository: quay.io/coreos/metering-reporting-operator tag: latest - - presto: - spec: - hive: - image: - repository: quay.io/coreos/metering-hive - tag: latest