Gh 0022 Add Hive Action

.gitignore

@@ -16,12 +16,11 @@ target
 
 # Oozie build
 core/pig*
-build
-mem
+core/hive-*
 core/build
 core/mem
 examples/oozietests
 mkdistro-*.out
 distro/downloads
 SecurityAuth.audit
-
+build

client/pom.xml

@@ -57,6 +57,16 @@
             <groupId>javax.persistence</groupId>
             <artifactId>persistence-api</artifactId>
             <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-lang</groupId>
+                    <artifactId>commons-lang</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
         </dependency>
     </dependencies>
 

client/src/main/java/org/apache/oozie/cli/OozieCLI.java

@@ -1169,6 +1169,8 @@ private void validateCommand(CommandLine commandLine) throws OozieCLIException {
                 List<StreamSource> sources = new ArrayList<StreamSource>();
                 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
                         "oozie-workflow-0.1.xsd")));
+                sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
+                        "hive-action-0.2.xsd")));
                 SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
                 Schema schema = factory.newSchema(sources.toArray(new StreamSource[sources.size()]));
                 Validator validator = schema.newValidator();

client/src/main/resources/hive-action-0.2.xsd

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+           xmlns:hive="uri:oozie:hive-action:0.2" elementFormDefault="qualified"
+           targetNamespace="uri:oozie:hive-action:0.2">
+
+    <xs:element name="hive" type="hive:ACTION"/>
+
+    <xs:complexType name="ACTION">
+        <xs:sequence>
+            <xs:element name="job-tracker" type="xs:string" minOccurs="1" maxOccurs="1"/>
+            <xs:element name="name-node" type="xs:string" minOccurs="1" maxOccurs="1"/>
+            <xs:element name="prepare" type="hive:PREPARE" minOccurs="0" maxOccurs="1"/>
+            <xs:element name="job-xml" type="xs:string" minOccurs="0" maxOccurs="1"/>
+            <xs:element name="configuration" type="hive:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+            <xs:element name="script" type="xs:string" minOccurs="1" maxOccurs="1"/>
+            <xs:element name="param" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+            <xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+            <xs:element name="archive" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+        </xs:sequence>
+    </xs:complexType>
+
+    <xs:complexType name="CONFIGURATION">
+        <xs:sequence>
+            <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+                <xs:complexType>
+                    <xs:sequence>
+                        <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+                        <xs:element name="value" minOccurs="1" maxOccurs="1" type="xs:string"/>
+                        <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+                    </xs:sequence>
+                </xs:complexType>
+            </xs:element>
+        </xs:sequence>
+    </xs:complexType>
+
+    <xs:complexType name="PREPARE">
+        <xs:sequence>
+            <xs:element name="delete" type="hive:DELETE" minOccurs="0" maxOccurs="unbounded"/>
+            <xs:element name="mkdir" type="hive:MKDIR" minOccurs="0" maxOccurs="unbounded"/>
+        </xs:sequence>
+    </xs:complexType>
+
+    <xs:complexType name="DELETE">
+        <xs:attribute name="path" type="xs:string" use="required"/>
+    </xs:complexType>
+
+    <xs:complexType name="MKDIR">
+        <xs:attribute name="path" type="xs:string" use="required"/>
+    </xs:complexType>
+
+</xs:schema>

core/pom.xml

@@ -60,7 +60,18 @@
         <dependency>
             <groupId>javax.persistence</groupId>
             <artifactId>persistence-api</artifactId>
-            <scope>compile</scope>
+            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-lang</groupId>
+                    <artifactId>commons-lang</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
         </dependency>
 
         <dependency>
@@ -76,14 +87,12 @@
         </dependency>
 
         <dependency>
-            <!-- groupId to be correct by GH-0226 -->
-            <groupId>com.yahoo.hadoop</groupId>
+            <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-core</artifactId>
             <scope>provided</scope>
         </dependency>
         <dependency>
-            <!-- groupId to be correct by GH-0226 -->
-            <groupId>com.yahoo.hadoop</groupId>
+            <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-test</artifactId>
             <scope>test</scope>
         </dependency>
@@ -92,6 +101,12 @@
             <groupId>com.yahoo.oozie</groupId>
             <artifactId>oozie-sharelib</artifactId>
             <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.datanucleus</groupId>
+                    <artifactId>datanucleus-enhancer</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>

core/src/main/conf/oozie-site.xml

@@ -16,6 +16,22 @@
 
 <configuration>
 
+    <!-- Uncomment the following  properties to enable Hive actions
+
+    <property>
+        <name>oozie.service.ActionService.executor.ext.classes</name>
+        <value>
+            org.apache.oozie.action.hadoop.HiveActionExecutor
+        </value>
+    </property>
+
+    <property>
+        <name>oozie.service.SchemaService.wf.ext.schemas</name>
+        <value>hive-action-0.2.xsd</value>
+    </property>
+
+    -->
+
 <!--
     Refer to the oozie-default.xml file for the complete list of
     Oozie configuration properties and their default values.

core/src/main/java/org/apache/oozie/ErrorCode.java

@@ -146,6 +146,8 @@ public enum ErrorCode {
     E0900(XLog.OPS, "Jobtracker [{0}] not allowed, not in Oozie's whitelist"),
     E0901(XLog.OPS, "Namenode [{0}] not allowed, not in Oozie's whitelist"),
     E0902(XLog.OPS, "Exception occured: [{0}]"),
+    E0903(XLog.OPS, "Configuration does not have Jobtracker Kerberos name or rule"),
+    E0904(XLog.OPS, "Configuration does not have Namenode Kerberos name or rule"),
 
     E1001(XLog.STD, "Could not read the coordinator job definition, {0}"),
     E1002(XLog.STD, "Invalid coordinator application URI [{0}], {1}"),

core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java

@@ -121,7 +121,7 @@ private FileSystem getFileSystemFor(Path path, Context context) throws HadoopAcc
         String user = context.getWorkflow().getUser();
         String group = context.getWorkflow().getGroup();
         return Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, path.toUri(),
-                new Configuration());
+                context.getProtoActionConf());
     }
 
     /**

core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java

@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS;
+
+import java.io.StringReader;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.action.ActionExecutorException;
+import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.util.XConfiguration;
+import org.apache.oozie.util.XmlUtils;
+import org.jdom.Element;
+import org.jdom.JDOMException;
+import org.jdom.Namespace;
+
+public class HiveActionExecutor extends JavaActionExecutor {
+    private static final String OOZIE_HIVE_DEFAULTS = "oozie.hive.defaults";
+
+    public HiveActionExecutor() {
+        super("hive");
+    }
+
+    @Override
+    protected List<Class> getLauncherClasses() {
+        List<Class> classes = super.getLauncherClasses();
+        classes.add(LauncherMain.class);
+        classes.add(MapReduceMain.class);
+        classes.add(HiveMain.class);
+        return classes;
+    }
+
+    @Override
+    protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
+        return launcherConf.get(CONF_OOZIE_ACTION_MAIN_CLASS, HiveMain.class.getName());
+    }
+
+    public Configuration setupHiveDefault(Configuration conf, Path appPath, Element actionXml)
+            throws ActionExecutorException {
+        try {
+            //Setting up hive-default.xml file if specified by the Hive action
+            Element actionConf = actionXml.getChild("configuration", actionXml.getNamespace());
+            if (actionConf != null) {
+                String strConf = XmlUtils.prettyPrint(actionConf).toString();
+                XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
+                if (inlineConf.get(OOZIE_HIVE_DEFAULTS) != null) {
+                    Path hiveDefaults = new Path(inlineConf.get(OOZIE_HIVE_DEFAULTS));
+                    // hive-default.xml will be softlinked to the working dir which is in the launcher CP.
+                    // the softlink is done as 'oozie-user-hive-default.xml' and Oozie HiveMain class will
+                    // check if the Hive being used has a hive-default.xml or not, if not it will rename
+                    // it as hive-default.xml before invoking hive
+                    addToCache(conf, appPath, hiveDefaults + "#" + HiveMain.USER_HIVE_DEFAULT_FILE, false);
+                }
+            }
+            return conf;
+        }
+        catch (Exception ex) {
+            throw convertException(ex);
+        }
+    }
+
+    @Override
+    protected Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context)
+            throws ActionExecutorException {
+        try {
+            super.setupLauncherConf(conf, actionXml, appPath, context);
+            Namespace ns = actionXml.getNamespace();
+
+            setupHiveDefault(conf, appPath, actionXml);
+
+            String script = actionXml.getChild("script", ns).getTextTrim();
+            String scriptName = new Path(script).getName();
+            addToCache(conf, appPath, script + "#" + scriptName, false);
+            return conf;
+        }
+        catch (Exception ex) {
+            throw convertException(ex);
+        }
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml,
+                                  Path appPath) throws ActionExecutorException {
+        Configuration conf = super.setupActionConf(actionConf, context, actionXml, appPath);
+
+        Namespace ns = actionXml.getNamespace();
+        String script = actionXml.getChild("script", ns).getTextTrim();
+        String scriptName = new Path(script).getName();
+        addToCache(conf, appPath, script + "#" + scriptName, false);
+
+        List<Element> params = (List<Element>) actionXml.getChildren("param", ns);
+        String[] strParams = new String[params.size()];
+        for (int i = 0; i < params.size(); i++) {
+            strParams[i] = params.get(i).getTextTrim();
+        }
+
+        HiveMain.setHiveScript(conf, scriptName, strParams);
+        return conf;
+    }
+
+    @Override
+    protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
+        return true;
+    }
+
+}