update cloud templates

.gitignore

@@ -1,4 +1,5 @@
 .idea
+.ruby-version
 /build/
 /build_debug/
 /build-debug/

openstudiocore/ruby/cloud/aws.rb.in

@@ -197,33 +197,39 @@ def create_struct(instance, procs)
 end
 
 def find_processors(instance)
+  lookup = {
+      'm3.medium' => 1,
+      'm3.large' => 2,
+      'm3.xlarge' => 4,
+      'm3.2xlarge' => 8,
+      'cc2.8xlarge' => 32,
+      'c3.large' => 2,
+      'c3.xlarge' => 4,
+      'c3.2xlarge' => 8,
+      'c3.4xlarge' => 16,
+      'c3.8xlarge' => 32,
+      'r3.large' => 2,
+      'r3.xlarge' => 4,
+      'r3.2xlarge' => 8,
+      'r3.4xlarge' => 16,
+      'r3.8xlarge' => 32,
+      't1.micro' => 1,
+      'm1.small' => 1,
+      'm2.xlarge' => 2,
+      'm2.2xlarge' => 4,
+      'm2.4xlarge' => 8,
+    }
+
   processors = 1
-  case instance
-    when 'cc2.8xlarge'
-      processors = 16
-    when 'c1.xlarge'
-      processors = 8
-    when 'm2.4xlarge'
-      processors = 8
-    when 'm2.2xlarge'
-      processors = 4
-    when 'm2.xlarge'
-      processors = 2
-    when 'm1.xlarge'
-      processors = 4
-    when 'm1.large'
-      processors = 2
-    when 'm3.xlarge'
-      processors = 2
-    when 'm3.2xlarge'
-      processors = 4
+  if lookup.key?(instance)
+    processors = lookup[instance]
+  else
+    error(-1, "Could not find the number of processors for #{instance}")
   end
 
   return processors
 end
 
-
-
 def launch_server
   user_data = File.read(File.expand_path(File.dirname(__FILE__))+'/server_script.sh')
   @logger.info("server user_data #{user_data.inspect}")

openstudiocore/ruby/cloud/server_script.sh

@@ -1,43 +1,97 @@
 #!/bin/sh
 
+# AWS Server Bootstrap File
+# This script is used to configure the AWS boxes
+
 # Change Host File Entries
 ENTRY="localhost localhost master"
 FILE=/etc/hosts
 if grep -q "$ENTRY" $FILE; then
   echo "entry already exists"
 else
-  sudo sh -c "echo $ENTRY >> /etc/hosts"
+  sh -c "echo $ENTRY >> /etc/hosts"
 fi
 
+rm -rf /mnt
+mkdir /mnt
+
 # copy all the setup scripts to the appropriate home directory
+# the scripts are called by the AWS connector for passwordless ssh config
 cp /data/launch-instance/setup* /home/ubuntu/
 chmod 775 /home/ubuntu/setup*
 chown ubuntu:ubuntu /home/ubuntu/setup*
 
-# Set permissions on rails apps folders
-sudo chmod 777 /var/www/rails/openstudio/public
+# stop the various services that use mongo
+service delayed_job stop
+service apache2 stop
+service mongodb stop
+
+# remove mongo db & add it back
+mkdir -p /mnt/mongodb/data
+chown mongodb:nogroup /mnt/mongodb/data
+rm -rf /var/lib/mongodb
+
+# restart mongo - old images has mongodb as the service. New ones use mongod
+service mongodb start
+service mongod start
+
+# delay the continuation because mongo is a forked process and when it initializes
+# it has to create the preallocated journal files (takes ~ 90 seconds on a slower system)
+sleep 2m
+
+# restart the rails application
+service apache2 stop
+service apache2 start
+
+# Add in the database indexes after making the db directory
+chmod 777 /var/www/rails/openstudio/public
+su - ubuntu -c 'cd /var/www/rails/openstudio && bundle exec rake db:purge'
+su - ubuntu -c 'cd /var/www/rails/openstudio && bundle exec rake db:mongoid:create_indexes'
+
+## Worker Data Configuration -- On Vagrant this is a separate file
 
-# Force the generation of OpenStudio on the EBS mount and copy worker files
-sudo rm -rf /mnt/openstudio
-sudo mkdir -p /mnt/openstudio
-sudo chmod -R 777 /mnt/openstudio
+# Force the generation of various directories that are in the EBS mnt
+rm -rf /mnt/openstudio
+mkdir -p /mnt/openstudio
+chown -R ubuntu:www-data /mnt/openstudio
+chmod -R 775 /mnt/openstudio
+
+# save application files into the right directory
 cp -rf /data/worker-nodes/* /mnt/openstudio/
 
-# Unzip the worker files (including Mongoid models) and set permissions one last time
-# Note that the 777 is redundant but needed until we actually deploy ACL
+# install workflow dependencies - not yet needed for OpenStudio but still including
+su - ubuntu -c 'cd /mnt/openstudio && rm -f'
+rm -f /mnt/openstudio/Gemfile.lock
+cd /mnt/openstudio && bundle update
+su - ubuntu -c 'cd /mnt/openstudio && bundle update'
+
+# copy over the models needed for mongo
+cd /mnt/openstudio/rails-models && unzip -o rails-models.zip -d models
+# Support the old  extraction as well - keep until we start using the new images
 cd /mnt/openstudio/rails-models && unzip -o rails-models.zip
-sudo chmod -R 777 /mnt/openstudio
-
-# Force the generation of MongoDB dbpath on the EBS mount
-sudo rm -rf /mnt/mongodb/data
-sudo mkdir -p /mnt/mongodb/data
-sudo chown mongodb:nogroup /mnt/mongodb/data
-sudo service mongodb restart
-sudo service delayed_job restart
-
-# Add database indexes after it gets mounted for the first time
-cd /var/www/rails/openstudio
-sudo rake db:mongoid:create_indexes
+
+# rerun the permissions after unzipping the files
+chown -R ubuntu:www-data /mnt/openstudio
+find /mnt/openstudio -type d -print0 | xargs -0 chmod 775
+find /mnt/openstudio -type f -print0 | xargs -0 chmod 664
+
+## End Worker Data Configuration
+
+# restart rserve
+service Rserve restart
+
+# restart delayed jobs
+service delayed_job start
+
+# Delay 1 minutes to make sure everything had tme to start.
+# This is a hack, sorry.
+sleep 1m
+
+# -- Old settings that may still be needed on the old images --
+# Set permissions on rails apps folders
+#sudo chmod 777 /var/www/rails/openstudio/public
+#sudo chmod -R 777 /mnt/openstudio
+# -- End old settings 
 
 #file flag the user_data has completed
 cat /dev/null > /home/ubuntu/user_data_done

openstudiocore/ruby/cloud/worker_script.sh.template

@@ -1,38 +1,68 @@
 #!/bin/sh
 
 # AWS Worker Bootstrap File
+# File used to configure worker nodes on AWS
 
 # Change Host File Entries
 ENTRY="SERVER_IP SERVER_HOSTNAME SERVER_ALIAS"
 FILE=/etc/hosts
 if grep -q "$ENTRY" $FILE; then
   echo "entry already exists"
 else
-  sudo sh -c "echo $ENTRY >> /etc/hosts"
+  sh -c "echo $ENTRY >> /etc/hosts"
 fi
 
 # copy all the setup scripts to the appropriate home directory
 cp /data/launch-instance/setup* /home/ubuntu/
 chmod 775 /home/ubuntu/setup*
 chown ubuntu:ubuntu /home/ubuntu/setup*
 
-# Force the generation of OpenStudio on the EBS mount and copy worker files
-sudo rm -rf /mnt/openstudio
-sudo mkdir -p /mnt/openstudio
-sudo chmod -R 777 /mnt/openstudio
+## Worker Data Configuration -- On Vagrant this is a separate file
+
+# Force the generation of various directories that are in the EBS mnt
+rm -rf /mnt/openstudio
+mkdir -p /mnt/openstudio
+chown -R ubuntu:www-data /mnt/openstudio
+chmod -R 775 /mnt/openstudio
+
+# save application files into the right directory
 cp -rf /data/worker-nodes/* /mnt/openstudio/
 
-# Unzip the worker files (including Mongoid models) and set permissions one last time
-# Note that the 777 is redundant but needed until we actually deploy ACL
+# install workflow dependencies
+su - ubuntu -c 'cd /mnt/openstudio && rm -f'
+rm -f /mnt/openstudio/Gemfile.lock
+cd /mnt/openstudio && bundle update
+su - ubuntu -c 'cd /mnt/openstudio && bundle update'
+
+# copy over the models needed for mongo
+cd /mnt/openstudio/rails-models && unzip -o rails-models.zip -d models
+# Support the old  extraction as well - keep until we start using the new images
 cd /mnt/openstudio/rails-models && unzip -o rails-models.zip
-sudo chmod -R 777 /mnt/openstudio
+
+# rerun the permissions after unzipping the files
+chown -R ubuntu:www-data /mnt/openstudio
+find /mnt/openstudio -type d -print0 | xargs -0 chmod 775
+find /mnt/openstudio -type f -print0 | xargs -0 chmod 664
+
+## End Worker Data Configuration
 
 #turn off hyperthreading
-for cpunum in $(
-  cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | 
-  cut -s -d, -f2- | tr ',' '\n' | sort -un); do
-    echo 0 > /sys/devices/system/cpu/cpu$cpunum/online
-done
+# NL: do not turn off hyperthreading, rather, just limit the number of cores and see if it
+# makes a difference.
+#for cpunum in $(
+#  cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list |
+#  cut -s -d, -f2- | tr ',' '\n' | sort -un); do
+#    echo 0 > /sys/devices/system/cpu/cpu$cpunum/online
+#done
+
+# -- Old settings that may still be needed on the old images --
+# Set permissions on rails apps folders
+#sudo chmod -R 777 /mnt/openstudio
+#sudo chmod -R 777 /mnt/openstudio
+# -- End old settings 
+
 
 #file flag the user_data has completed
 cat /dev/null > /home/ubuntu/user_data_done
+
+

openstudiocore/src/utilities/cloud/AWSProvider.cpp

@@ -585,7 +585,7 @@ namespace openstudio{
     }
 
     std::string AWSProvider_Impl::defaultWorkerInstanceType() {
-      return "cc2.8xlarge";
+      return "c3.4xlarge";
     }
 
     std::vector<unsigned> AWSProvider_Impl::serverProcessorCounts() {
@@ -742,8 +742,19 @@ namespace openstudio{
         //info.push_back(awsComputerInformation);
 
         awsComputerInformation.instanceType = "cc2.8xlarge";
-        awsComputerInformation.prettyName = "Cluster Compute Eight Extra Large";
-        awsComputerInformation.processorCount = 16; // Hyperthreading disabled
+        awsComputerInformation.prettyName = "Previous Generation: Compute Optimized 8x Large";
+        awsComputerInformation.processorCount = 16; // Hyperthreading enabled
+        info.push_back(awsComputerInformation);
+
+
+        awsComputerInformation.instanceType = "c3.4xlarge";
+        awsComputerInformation.prettyName = "Compute Optimized 4x Large";
+        awsComputerInformation.processorCount = 16; // Hyperthreading enabled
+        info.push_back(awsComputerInformation);
+
+        awsComputerInformation.instanceType = "c3.8xlarge";
+        awsComputerInformation.prettyName = "Cluster Compute 8x Large";
+        awsComputerInformation.processorCount = 32; // Hyperthreading enabled
         info.push_back(awsComputerInformation);
       }
       return info;