Skip to content

Commit 6960904

Browse files
kumaabkumaab
committed
RANGER-4918: Add support for Impala in docker
1 parent 2e34537 commit 6960904

14 files changed

+447
-15
lines changed

‎dev-support/ranger-docker/.env

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ KAFKA_VERSION=2.8.2
3939
KNOX_VERSION=2.0.0
4040
TRINO_VERSION=377
4141

42+
IMPALA_VERSION=4.0.0
43+
4244
# versions of ranger services
4345
RANGER_VERSION=3.0.0-SNAPSHOT
4446
KMS_VERSION=3.0.0-SNAPSHOT

‎dev-support/ranger-docker/Dockerfile.impalad

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
ARG IMPALA_VERSION
18+
FROM apache/impala:${IMPALA_VERSION}-impalad_coord_exec
19+
20+
USER root
21+
22+
RUN mkdir -p /var/lib/ranger/impala
23+
RUN mkdir -p -m 755 /var/log/impala
24+
RUN chown impala:impala /var/lib/ranger/impala
25+
RUN chown impala:impala -R /var/log/impala
26+
27+
# Install impala-shell and its dependencies
28+
RUN apt-get update
29+
RUN apt-get install -y libkrb5-dev
30+
RUN apt-get install -y python3-dev python3-pip python3
31+
RUN apt-get install -y gcc g++
32+
RUN apt-get install -y libsasl2-dev libssl-dev
33+
RUN apt-get clean && rm -rf /var/lib/apt/lists/*
34+
35+
RUN pip3 install impala-shell
36+
USER impala
37+
38+
ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\
39+
"-log_dir=/opt/impala/logs",\
40+
"-abort_on_config_error=false", "-state_store_host=impala-statestored",\
41+
"-catalog_service_host=impala-catalogd", "-mem_limit_includes_jvm=true",\
42+
"-use_local_catalog=true", "--rpc_use_loopback=true",\
43+
"-use_resolved_hostname=true",\
44+
"--ranger_service_type=hive",\
45+
"--ranger_app_id=impala",\
46+
"--authorization_provider=ranger",\
47+
""]

‎dev-support/ranger-docker/Dockerfile.ranger-base

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ RUN groupadd ranger && \
4949
useradd -g hadoop -ms /bin/bash hive && \
5050
useradd -g hadoop -ms /bin/bash hbase && \
5151
useradd -g hadoop -ms /bin/bash kafka && \
52+
useradd -g hadoop -ms /bin/bash impala && \
5253
groupadd knox && \
5354
useradd -g knox -ms /bin/bash knox && \
5455
mkdir -p /home/ranger/dist && \

‎dev-support/ranger-docker/Dockerfile.ranger-hive

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,5 +50,6 @@ ENV HIVE_HOME /opt/hive
5050
ENV HADOOP_HOME /opt/hadoop
5151
ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/hadoop/bin
5252

53+
COPY ./config/hive/hive-log4j2.properties ${HIVE_HOME}/hive-log4j2.properties
5354

5455
ENTRYPOINT [ "/home/ranger/scripts/ranger-hive.sh" ]

‎dev-support/ranger-docker/config/hive/hive-log4j2.properties

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
status = INFO
18+
name = HiveLog4j2
19+
packages = org.apache.hadoop.hive.ql.log
20+
21+
# list of properties
22+
property.hive.log.level = INFO
23+
property.hive.root.logger = DRFA
24+
property.hive.log.dir = /opt/hive
25+
property.hive.log.file = hive.log
26+
property.hive.perflogger.log.level = INFO
27+
28+
# list of all appenders
29+
appenders = console, DRFA
30+
31+
# console appender
32+
appender.console.type = Console
33+
appender.console.name = console
34+
appender.console.target = SYSTEM_ERR
35+
appender.console.layout.type = PatternLayout
36+
appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
37+
38+
# daily rolling file appender
39+
appender.DRFA.type = RollingRandomAccessFile
40+
appender.DRFA.name = DRFA
41+
appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file}
42+
# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session
43+
appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}
44+
appender.DRFA.layout.type = PatternLayout
45+
appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
46+
appender.DRFA.policies.type = Policies
47+
appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
48+
appender.DRFA.policies.time.interval = 1
49+
appender.DRFA.policies.time.modulate = true
50+
appender.DRFA.strategy.type = DefaultRolloverStrategy
51+
appender.DRFA.strategy.max = 30
52+
53+
# list of all loggers
54+
loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger, AmazonAws, ApacheHttp
55+
56+
logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
57+
logger.NIOServerCnxn.level = WARN
58+
59+
logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
60+
logger.ClientCnxnSocketNIO.level = WARN
61+
62+
logger.DataNucleus.name = DataNucleus
63+
logger.DataNucleus.level = ERROR
64+
65+
logger.Datastore.name = Datastore
66+
logger.Datastore.level = ERROR
67+
68+
logger.JPOX.name = JPOX
69+
logger.JPOX.level = ERROR
70+
71+
logger.AmazonAws.name=com.amazonaws
72+
logger.AmazonAws.level = INFO
73+
74+
logger.ApacheHttp.name=org.apache.http
75+
logger.ApacheHttp.level = INFO
76+
77+
logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger
78+
logger.PerfLogger.level = ${sys:hive.perflogger.log.level}
79+
80+
# root logger
81+
rootLogger.level = ${sys:hive.log.level}
82+
rootLogger.appenderRefs = root
83+
rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}

‎dev-support/ranger-docker/config/impala/hive-site.xml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><!--
3+
/*
4+
* Licensed to the Apache Software Foundation (ASF) under one
5+
* or more contributor license agreements. See the NOTICE file
6+
* distributed with this work for additional information
7+
* regarding copyright ownership. The ASF licenses this file
8+
* to you under the Apache License, Version 2.0 (the
9+
* "License"); you may not use this file except in compliance
10+
* with the License. You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
-->
21+
<configuration>
22+
<property>
23+
<name>hive.metastore.uris</name>
24+
<value>thrift://ranger-hive:9083</value>
25+
</property>
26+
<property>
27+
<name>hive.metastore.event.db.notification.api.auth</name>
28+
<value>false</value>
29+
</property>
30+
</configuration>

‎dev-support/ranger-docker/config/impala/ranger-hive-audit.xml

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
<configuration>
2+
<property>
3+
<name>xasecure.audit.is.enabled</name>
4+
<value>true</value>
5+
</property>
6+
<property>
7+
<name>xasecure.audit.provider.summary.enabled</name>
8+
<value>true</value>
9+
</property>
10+
<property>
11+
<name>xasecure.audit.destination.hdfs</name>
12+
<value>true</value>
13+
</property>
14+
<property>
15+
<name>xasecure.audit.destination.hdfs.dir</name>
16+
<value>hdfs://ranger-hadoop:9000/ranger/audit</value>
17+
</property>
18+
<property>
19+
<name>xasecure.audit.destination.hdfs.batch.filespool.dir</name>
20+
<value>/var/log/impala/audit/hdfs/spool</value>
21+
</property>
22+
<property>
23+
<name>xasecure.audit.destination.solr</name>
24+
<value>true</value>
25+
</property>
26+
<property>
27+
<name>xasecure.audit.destination.solr.urls</name>
28+
<value>http://ranger-solr:8983/solr/ranger_audits</value>
29+
</property>
30+
<property>
31+
<name>xasecure.audit.destination.solr.zookeepers</name>
32+
<value></value>
33+
</property>
34+
<property>
35+
<name>xasecure.audit.solr.solr_url</name>
36+
<value>http://ranger-solr:8983/solr/ranger_audits</value>
37+
</property>
38+
<property>
39+
<name>xasecure.audit.destination.solr.batch.filespool.dir</name>
40+
<value>/var/log/impala/audit/solr/spool</value>
41+
</property>
42+
<property>
43+
<name>xasecure.audit.jaas.Client.loginModuleName</name>
44+
<value></value>
45+
</property>
46+
<property>
47+
<name>xasecure.audit.jaas.Client.loginModuleControlFlag</name>
48+
<value>required</value>
49+
</property>
50+
<property>
51+
<name>xasecure.audit.jaas.Client.option.useKeyTab</name>
52+
<value>false</value>
53+
</property>
54+
<property>
55+
<name>xasecure.audit.jaas.Client.option.storeKey</name>
56+
<value>false</value>
57+
</property>
58+
<property>
59+
<name>xasecure.audit.jaas.Client.option.serviceName</name>
60+
<value>solr</value>
61+
</property>
62+
<property>
63+
<name>xasecure.audit.destination.solr.force.use.inmemory.jaas.config</name>
64+
<value>true</value>
65+
</property>
66+
<property>
67+
<name>xasecure.audit.jaas.Client.option.principal</name>
68+
<value></value>
69+
</property>
70+
<property>
71+
<name>xasecure.audit.jaas.Client.option.keyTab</name>
72+
<value></value>
73+
</property>
74+
<property>
75+
<name>xasecure.audit.destination.metrics</name>
76+
<value>false</value>
77+
</property>
78+
</configuration>

‎dev-support/ranger-docker/config/impala/ranger-hive-security.xml

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one or more
4+
contributor license agreements. See the NOTICE file distributed with
5+
this work for additional information regarding copyright ownership.
6+
The ASF licenses this file to You under the Apache License, Version 2.0
7+
(the "License"); you may not use this file except in compliance with
8+
the License. You may obtain a copy of the License at
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
-->
16+
<configuration>
17+
<property>
18+
<name>ranger.plugin.hive.service.name</name>
19+
<value>dev_hive</value>
20+
<description>Name of the Ranger service containing policies.</description>
21+
</property>
22+
<property>
23+
<name>ranger.plugin.hive.policy.rest.url</name>
24+
<value>http://ranger:6080</value>
25+
<description>URL to Ranger Admin.</description>
26+
</property>
27+
<property>
28+
<name>ranger.plugin.hive.policy.cache.dir</name>
29+
<value>/var/lib/ranger/impala/policy-cache</value>
30+
<description>
31+
Directory where Ranger policies are cached after successful retrieval from the source.
32+
</description>
33+
</property>
34+
<property>
35+
<name>ranger.plugin.hive.policy.source.impl</name>
36+
<value>org.apache.ranger.admin.client.RangerAdminRESTClient</value>
37+
</property>
38+
<property>
39+
<name>ranger.plugin.hive.policy.pollIntervalMs</name>
40+
<value>30000</value>
41+
<description>
42+
Polling interval in milliseconds to poll for changes in policies.
43+
</description>
44+
</property>
45+
<property>
46+
<name>ranger.plugin.hive.policy.rest.client.connection.timeoutMs</name>
47+
<value>120000</value>
48+
<description>
49+
RangerRestClient connection timeout in milliseconds.
50+
</description>
51+
</property>
52+
<property>
53+
<name>ranger.plugin.hive.policy.rest.client.read.timeoutMs</name>
54+
<value>30000</value>
55+
<description>
56+
RangerRestClient read timeout in milliseconds.
57+
</description>
58+
</property>
59+
<property>
60+
<name>ranger.plugin.hive.disable.cache.if.servicenotfound</name>
61+
<value>false</value>
62+
</property>
63+
<property>
64+
<name>ranger.plugin.hive.use.x-forwarded-for.ipaddress</name>
65+
<value>false</value>
66+
</property>
67+
<property>
68+
<name>ranger.plugin.hive.trusted.proxy.ipaddress</name>
69+
<value></value>
70+
</property>
71+
<property>
72+
<name>xasecure.hive.update.xapolicies.on.grant.revoke</name>
73+
<value>true</value>
74+
</property>
75+
<property>
76+
<name>ranger.plugin.hive.urlauth.filesystem.schemes</name>
77+
<value>hdfs:,file:,wasb:,adl:</value>
78+
</property>
79+
</configuration>

0 commit comments

Comments
 (0)