Feature/seqware 1.2 (#12)

* java 8
* postgres 9.1
* CDH 5.4.4

roles/seqware-code/tasks/main.yml

@@ -16,8 +16,10 @@
   when: create_user
 
 - name:  Set up SeqWare working directories
-  file: path={{ item.path }} state={{ item.state }} owner={{ item.owner }} mode={{ item.mode}} group={{ item.group }}
+  file: path={{ item.path }} state={{ item.state }} owner={{ item.owner }} mode={{ item.mode}} group={{ item.group }} recurse=yes
   with_items:
+     - { state: 'directory', path: '/mnt/home/{{ user_name }}', owner: '{{ user_name }}', group: '{{ user_name }}', mode: '0700' }
+     - { state: 'directory', path: '~{{ user_name }}', owner: '{{ user_name }}', group: '{{ user_name }}', mode: '0700' }
      - { state: 'directory', path: '~{{ user_name }}/bin', owner: '{{ user_name }}', group: '{{ user_name }}', mode: '0700' }
      - { state: 'directory', path: '~{{ user_name }}/jars', owner: '{{ user_name }}', group: '{{ user_name }}', mode: '0700' }
      - { state: 'directory', path: '~{{ user_name }}/crons', owner: '{{ user_name }}', group: '{{ user_name }}', mode: '0700' }
@@ -69,7 +71,7 @@
   lineinfile: 
     dest: ~{{ user_name }}/.bash_profile
     create: yes
-    line: 'export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=512m"'
+    line: 'export MAVEN_OPTS="-Xmx1024m"'
     state: present
     insertafter: EOF
     owner: "{{ user_name }}"

roles/seqware-code/templates/settings.j2

@@ -8,6 +8,10 @@ SW_REST_URL=http://{{ inventory_hostname }}:8080/SeqWareWebService
 # the username and password to connect to the REST API used by SeqWare Pipeline to write back processing info to the DB 
 [email protected]
 SW_REST_PASS=admin
+
+# optional: In atypical environments, the default h_vmem constraint for SGE is too stringent. Override them using this (units in megabytes)
+SW_CONTROL_NODE_MEMORY=4000
+
 # the base URL for the admin web service
 SW_ADMIN_REST_URL=http://localhost:38080/seqware-admin-webservice
 # used for direct database connection by admin tools

roles/seqware-common-infrastructure/tasks/main.yml

@@ -2,7 +2,7 @@
 # file: roles/seqware-common-infrastructure/tasks/main.yml
 - name:  Add Hadoop repository
   lineinfile: 
-    line: 'deb [arch=amd64] http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh precise-cdh4.5.0 contrib'
+    line: 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh5.4.4 contrib'
     dest: /etc/apt/sources.list.d/cloudera.list
     state: present
     create: yes
@@ -11,15 +11,15 @@
 
 - name:  Add Hadoop repository
   lineinfile: 
-    line: 'deb-src http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh precise-cdh4.5.0 contrib'
+    line: 'deb-src http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh5.4.4 contrib'
     dest: /etc/apt/sources.list.d/cloudera.list
     state: present
     create: yes
     mode: 0755
   register: hadooprepo
 
 - name:  Add signing key
-  apt_key: url=http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh/archive.key state=present
+  apt_key: url=http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/archive.key state=present
   register: hadooprepo
 
 - apt: update_cache=yes

roles/seqware-helloworld/tasks/main.yml

@@ -1,8 +1,14 @@
 ---
 # file: roles/seqware-helloworld/tasks/main.yml
 
+# weird, not sure why this may be an issue in Ubuntu 14.04
+- name:  Ensure users own their home directories
+  file: path={{ item.path }} state={{ item.state }} owner={{ item.owner }} mode={{ item.mode}} group={{ item.group }} recurse=yes
+  with_items:
+     - { state: 'directory', path: '/mnt/home/{{ user_name }}', owner: '{{ user_name }}', group: '{{ user_name }}', mode: '0700' }
+
 - include: 'git.yml'
   when: seqware_provider == "git"
 
 - include: 'artifactory.yml'
-  when: seqware_provider == "artifactory"
+  when: seqware_provider == "artifactory"

roles/seqware-helloworld/vars/main.yml

@@ -5,7 +5,7 @@
 # not sure about the best way of making sure both have the same defaults
 # however, a command-line variable overrides both at the same time
 seqware_provider: "artifactory"
-seqware_version: "1.1.0"
+seqware_version: "1.2.0-alpha.0"
 
 # variable specific to this role
 helloworld_s3_location: "https://s3.amazonaws.com/oicr.workflow.bundles/released-bundles/Workflow_Bundle_HelloWorld_1.0-SNAPSHOT_SeqWare_{{ seqware_version }}.zip"

roles/seqware-master-infrastructure/files/oozie-env.sh

@@ -1,29 +1,84 @@
-#! /bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
+#!/bin/bash
 #
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#
+
+export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:/usr/lib/hadoop/lib/native"
+
+# Set Oozie specific environment variables here.
 
-export OOZIE_CONFIG=/etc/oozie/conf
 export OOZIE_DATA=/var/lib/oozie
-export OOZIE_LOG=/var/log/oozie
 export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat
 export CATALINA_TMPDIR=/var/lib/oozie
 export CATALINA_PID=/var/run/oozie/oozie.pid
-export CATALINA_BASE=/usr/lib/oozie/oozie-server-0.20
-export CATALINA_OPTS=-Xmx2048m
+export CATALINA_BASE=/var/lib/oozie/tomcat-deployment
+
+# Settings for the Embedded Tomcat that runs Oozie
+# Java System properties for Oozie should be specified in this variable
+#
 export OOZIE_HTTPS_PORT=11443
 export OOZIE_HTTPS_KEYSTORE_PASS=password
 export CATALINA_OPTS="$CATALINA_OPTS -Doozie.https.port=${OOZIE_HTTPS_PORT}"
 export CATALINA_OPTS="$CATALINA_OPTS -Doozie.https.keystore.pass=${OOZIE_HTTPS_KEYSTORE_PASS}"
+export CATALINA_OPTS="$CATALINA_OPTS -Xmx2048m"
+
+# Oozie configuration file to load from Oozie configuration directory
+#
+# export OOZIE_CONFIG_FILE=oozie-site.xml
+export OOZIE_CONFIG=/etc/oozie/conf
+
+# Oozie logs directory
+#
+# export OOZIE_LOG=${OOZIE_HOME}/logs
+export OOZIE_LOG=/var/log/oozie
+
+# Oozie Log4J configuration file to load from Oozie configuration directory
+#
+# export OOZIE_LOG4J_FILE=oozie-log4j.properties
+
+# Reload interval of the Log4J configuration file, in seconds
+#
+# export OOZIE_LOG4J_RELOAD=10
+
+# The port Oozie server runs
+#
+# export OOZIE_HTTP_PORT=11000
+
+# The port Oozie server runs if using SSL (HTTPS)
+#
+# export OOZIE_HTTPS_PORT=11443
+
+# The host name Oozie server runs on
+#
+# export OOZIE_HTTP_HOSTNAME=`hostname -f`
+
+# The base URL for callback URLs to Oozie
+#
+# export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie"
+
+# The location of the keystore for the Oozie server if using SSL (HTTPS)
+#
+# export OOZIE_HTTPS_KEYSTORE_FILE=${HOME}/.keystore
+
+# The password of the keystore for the Oozie server if using SSL (HTTPS)
+#
+# export OOZIE_HTTPS_KEYSTORE_PASS=password
+
+# The Oozie Instance ID
+#
+# export OOZIE_INSTANCE_ID="${OOZIE_HTTP_HOSTNAME}"
+

roles/seqware-master-infrastructure/files/oozie-site.xml

@@ -7,9 +7,9 @@
   to you under the Apache License, Version 2.0 (the
   "License"); you may not use this file except in compliance
   with the License.  You may obtain a copy of the License at
-  
+
        http://www.apache.org/licenses/LICENSE-2.0
-  
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -26,7 +26,7 @@
     <property><name>oozie.service.ActionCheckerService.action.check.delay</name><value>10</value></property>
     <property><name>oozie.service.LiteWorkflowStoreService.user.retry.max</name><value>30</value></property>
     <property><name>oozie.action.retries.max</name><value>30</value></property>
-    <property><name>oozie.service.WorkflowAppService.WorkflowDefinitionMaxLength</name><value>10000000</value></property>
+    <property><name>oozie.service.WorkflowAppService.WorkflowDefinitionMaxLength</name><value>10000000</value></property>  
     <property><name>oozie.service.LiteWorkflowStoreService.user.retry.error.code.ext</name><value>SGE82,SGE137</value></property>
     <property><name>oozie.validate.ForkJoin</name><value>false</value></property>
 
@@ -38,13 +38,19 @@
             org.apache.oozie.action.hadoop.HiveActionExecutor,
             org.apache.oozie.action.hadoop.ShellActionExecutor,
             org.apache.oozie.action.hadoop.SqoopActionExecutor,
-            org.apache.oozie.action.hadoop.DistcpActionExecutor
+            org.apache.oozie.action.hadoop.DistcpActionExecutor,
+            org.apache.oozie.action.hadoop.Hive2ActionExecutor
         </value>
     </property>
 
     <property>
         <name>oozie.service.SchemaService.wf.ext.schemas</name>
-        <value>sge-action-1.0.xsd,shell-action-0.1.xsd,shell-action-0.2.xsd,email-action-0.1.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,ssh-action-0.1.xsd,ssh-action-0.2.xsd,distcp-action-0.1.xsd</value>
+        <value>
+            shell-action-0.1.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,email-action-0.1.xsd,email-action-0.2.xsd,
+            hive-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,sqoop-action-0.2.xsd,
+            sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,ssh-action-0.1.xsd,ssh-action-0.2.xsd,distcp-action-0.1.xsd,
+            distcp-action-0.2.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd,hive2-action-0.1.xsd, sge-action-1.0.xsd
+        </value>
     </property>
 
     <property>
@@ -156,14 +162,16 @@
     </property>
 
     <property>
-        <name>oozie.service.JPAService.jdbc.username</name><value>oozie</value>
+        <name>oozie.service.JPAService.jdbc.username</name>
+        <value>oozie</value>
         <description>
             DB user name.
         </description>
     </property>
 
     <property>
-        <name>oozie.service.JPAService.jdbc.password</name><value>oozie</value>
+        <name>oozie.service.JPAService.jdbc.password</name>
+        <value>oozie</value>
         <description>
             DB user password.
 
@@ -280,17 +288,6 @@
         </description>
     </property>
 
-    <property>
-        <name>oozie.authentication.signature.secret</name>
-        <value>oozie</value>
-        <description>
-            The signature secret for signing the authentication tokens.
-            If not set a random secret is generated at startup time.
-            In order to authentiation to work correctly across multiple hosts
-            the secret must be the same across al the hosts.
-        </description>
-    </property>
-
     <property>
       <name>oozie.authentication.cookie.domain</name>
       <value></value>