Merge pull request #6 from SeqWare/feature/oozie-sge_inside_docker

Feature/oozie sge inside docker
SeqWare · Feb 6, 2015 · 9ba4d28 · 9ba4d28
2 parents 9505a61 + 807cbd9
commit 9ba4d28
Show file tree

Hide file tree

Showing 12 changed files with 100 additions and 11 deletions.
diff --git a/docker-start.yml b/docker-start.yml
@@ -0,0 +1,31 @@
+--- 
+# file: docker-start.yml
+# This file is used for installing a seqware container
+
+- hosts: all
+  sudo: True
+  tasks:
+  - name: Start all required services
+    service: name={{ item }} state=restarted 
+    with_items: 
+     - postgresql
+     - hadoop-hdfs-datanode
+     - hadoop-hdfs-namenode
+     - hadoop-0.20-mapreduce-jobtracker
+     - hadoop-0.20-mapreduce-tasktracker
+     - oozie
+     - gridengine-master
+     - gridengine-exec
+  - name: Bump up control node memory allocation for high-mem machines
+    lineinfile: dest=~seqware/.seqware/settings regexp='^SW_CONTROL_NODE_MEMORY.*' line='SW_CONTROL_NODE_MEMORY=4000' state=present
+    when: ansible_memtotal_mb > 100000
+  - name: Redirect oozie working directory so that workflow runs are persisted
+    lineinfile: dest=~seqware/.seqware/settings regexp='^OOZIE_WORK_DIR.*' line='OOZIE_WORK_DIR=/datastore' state=present
+  - name: Start tomcat7 
+    service: name=tomcat7 state=started
+    ignore_errors: yes
+
+- hosts: all 
+  sudo: True
+  roles:
+    - { role: grid-engine, single_node: True , grid_engine_master: True }
diff --git a/roles/docker/README.md b/roles/docker/README.md
@@ -0,0 +1,3 @@
+Docker role
+-------------
+This sets up random stuff that is needed to run inside docker
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
@@ -0,0 +1,7 @@
+- name: install extra stuff that is normally present
+  apt: pkg={{ item }} state=present update_cache=yes cache_valid_time=3600
+  with_items:
+    - sudo 
+    - curl 
+    - vim
+    - openssh-server 
diff --git a/roles/grid-engine/tasks/master.yml b/roles/grid-engine/tasks/master.yml
@@ -7,6 +7,12 @@
 - name: SGE | Install gridengine-master 
   apt: pkg=gridengine-master state=present update_cache=yes cache_valid_time=3600
 
+- name: SGE | Template act_qmaster
+  template: src=act_qmaster.j2 dest=/var/lib/gridengine/default/common/act_qmaster
+
+- name: SGE | Ensure gridengine-master started 
+  service: name=gridengine-master state=started
+
 - name: SGE | Configure master
   command: qconf -as {{ inventory_hostname }}
   register: configure_master_as
@@ -46,7 +52,7 @@
 
 # loop through and setup worker hosts, if this was better it would modify existing hosts as well instead of ignoring errors
 - name: SGE | Add worker hosts as execution hosts
-  shell: export EDITOR=/etc/gridengine/templates/sge_execution_host_{{ item }}.sh && qconf -ae
+  shell: export EDITOR=/etc/gridengine/templates/sge_execution_host_{{ item }}.sh && qconf -ae ; qconf -me {{ item }}
   with_items: groups.worker
   when: execution_host_template.changed 
   ignore_errors: yes
@@ -58,7 +64,7 @@
   when: single_node
 
 - name: SGE | Single node host as execution host
-  shell: export EDITOR=/etc/gridengine/templates/sge_execution_host_master.sh && qconf -ae
+  shell: export EDITOR=/etc/gridengine/templates/sge_execution_host_master.sh && qconf -ae ; qconf -me master
   when: execution_host_template.changed
   ignore_errors: yes
 
@@ -88,12 +94,19 @@
 # template handles hostlist, worker queue slots, target load average
 - name: SGE | Template main queue
   template: src=aq.j2 dest=/etc/gridengine/templates/sge_aq
-  when: not 'main.q' in sge_current_queues.stdout_lines
   register: main_queue_template
 
 - name: SGE | Add main queue
   command: qconf -Aq /etc/gridengine/templates/sge_aq
-  when: not 'main.q' in sge_current_queues.stdout_lines or main_queue_template.changed
+  when: not 'main.q' in sge_current_queues.stdout_lines
+
+- name: SGE | Template modify queue
+  template: src=mq.j2 dest=/etc/gridengine/templates/sge_mq.sh mode=0555
+  register: modify_main_queue_template
+
+- name: SGE | Modify main queue
+  shell: export EDITOR=/etc/gridengine/templates/sge_mq.sh && qconf -mq main.q 
+  when: modify_main_queue_template.changed
 
 - name: SGE | Check parallel environment status
   command: qconf -sp serial
@@ -114,4 +127,4 @@
       - qconf -aattr queue pe_list serial main.q
   when:  "'serial is not a parallel environment' in sge_parallel_env.stderr or serial_profile.changed"
 
-# TODO: bindle 1.2.1 re-runs the sge setup script on reboot. Determine whether this is really needed
+# TODO: bindle 1.2.1 re-runs the sge setup script on reboot. Determine whether this is really needed
diff --git a/roles/grid-engine/templates/act_qmaster.j2 b/roles/grid-engine/templates/act_qmaster.j2
@@ -0,0 +1 @@
+{{ inventory_hostname }}
diff --git a/roles/grid-engine/templates/mq.j2 b/roles/grid-engine/templates/mq.j2
@@ -0,0 +1,4 @@
+#!/bin/sh
+sleep 1
+perl -pi -e 's/^load_thresholds.*$/load_thresholds np_load_avg={{ ansible_processor_vcpus }}/' $1
+perl -pi -e 's/^slots.*$/slots {{ ansible_processor_vcpus }}{% for host in groups['all_groups'] %},[{{ host }}={{ ansible_processor_vcpus }}] {% endfor %}/' $1
diff --git a/roles/postgres/files/pg_hba.conf b/roles/postgres/files/pg_hba.conf
@@ -83,7 +83,7 @@
 # maintenance (custom daily cronjobs, replication, and similar tasks).
 #
 # Database administrative login by Unix domain socket
-local   all             postgres                                peer
+local   all             postgres                                trust 
 
 # TYPE  DATABASE        USER            ADDRESS                 METHOD
 

diff --git a/roles/postgres/files/postgresql.conf b/roles/postgres/files/postgresql.conf
@@ -77,7 +77,7 @@ unix_socket_directory = '/var/run/postgresql'		# (change requires restart)
 # - Security and Authentication -
 
 #authentication_timeout = 1min		# 1s-600s
-ssl = true				# (change requires restart)
+# ssl = true				# (change requires restart)
 #ssl_ciphers = 'ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH'	# allowed SSL ciphers
 					# (change requires restart)
 #ssl_renegotiation_limit = 512MB	# amount of data between renegotiations

diff --git a/roles/postgres/tasks/main.yml b/roles/postgres/tasks/main.yml
@@ -1,4 +1,17 @@
----
+# Ensure a locale exists.
+- name: Ensure UTF-8 exists
+  sudo: yes
+  shell: locale-gen en_US.UTF-8 && dpkg-reconfigure locales 
+
+# try setting locale? https://stackoverflow.com/questions/27292109/how-to-set-shell-variable-in-ansible-globally
+- name: dev locale
+  action: command sudo update-locale LC_ALL=en_US.utf8
+- name: set default locale
+  sudo: yes
+  lineinfile: dest=/etc/default/locale
+    regexp="LC_ALL"
+    line="LC_ALL=\"en_US.utf8\""
+
 - name: install postgres pkgs
   apt: pkg={{ item }} state=present update_cache=yes cache_valid_time=3600
   with_items:
@@ -22,6 +35,13 @@
   service: name=postgresql state=restarted enabled=yes
   when: postgresconf.changed
 
+- name: ensure postgres started 
+  service: name=postgresql state=started enabled=yes
+
+- name: make sure we can sudo as postgres
+  lineinfile: >
+     dest=/etc/sudoers state=present regexp='^seqware ALL=(ALL) NOPASSWD: ALL' line='seqware ALL=(ALL) NOPASSWD: ALL'
+
 - name: make a db user
   sudo_user: postgres
   postgresql_user: user=seqware password=seqware role_attr_flags=CREATEDB,NOSUPERUSER
@@ -58,4 +78,4 @@
 
 - name: setup seqware as a superuser
   sudo_user: postgres
-  postgresql_privs: database=postgres type=group objs=postgres role=seqware admin_option=yes
+  postgresql_privs: database=postgres type=group objs=postgres role=seqware admin_option=yes
diff --git a/roles/seqware-common-infrastructure/tasks/main.yml b/roles/seqware-common-infrastructure/tasks/main.yml
@@ -93,4 +93,4 @@
   file: state=link src=/mnt/seqware-oozie dest=/usr/tmp/seqware-oozie owner=seqware group=seqware
 
 - name:  Create datastore link
-  file: state=link src=/mnt/datastore dest=/datastore owner=seqware group=seqware
+  file: state=link src=/mnt/datastore dest=/datastore owner=seqware group=seqware
diff --git a/roles/seqware-webservice/handlers/main.yml b/roles/seqware-webservice/handlers/main.yml
@@ -1,2 +1,4 @@
+# Weird, why does this fail in a container?
 - name: restart tomcat7
   service: name=tomcat7 state=restarted
+  ignore_errors: yes
diff --git a/seqware-install.yml b/seqware-install.yml
@@ -11,14 +11,17 @@
   tasks:
   - name: Build hosts file for all hosts
     lineinfile: dest=/etc/hosts regexp='.*{{ item }}$' line="{{ hostvars[item].ansible_default_ipv4.address }} {{item}}" state=present
-    when: hostvars[item].ansible_default_ipv4.address is defined
+    when: hostvars[item].ansible_default_ipv4.address is defined and docker is not defined
     with_items: groups['all']
   - name: Setting hostname to {{ inventory_hostname }}
     hostname: name={{ inventory_hostname }}
+    when: docker is not defined
   - name: Disable automatic retrieval of hostname from dhcp on Amazon EC2
     template: src=roles/common/templates/dhclient.conf dest=/etc/dhcp/dhclient.conf
+    when: docker is not defined
   - name: Disable setting hostname from /etc/hostname on Amazon EC2
     template: src=roles/common/templates/hostname.conf dest=/etc/init/hostname.conf
+    when: docker is not defined
   - name: Disable ephemeral mount at /mnt for amazon AWS
     shell: umount /mnt
     register: umount_output
@@ -48,6 +51,11 @@
       glusterfs_version: 3.5 
   include: roles/storage/common/tasks/play_deploy_shared_storage.yml
 
+- hosts: all 
+  sudo: True
+  roles:
+    - { role: docker }
+
 # The grid master depends on the hosts file, as we need to know at least who we
 # are by this stage. 
 - hosts: master