Merge pull request #8 from PDCbc/dev

Dev

.gitignore

@@ -13,3 +13,4 @@ spec/generated
 monitoring/nagios-config/
 monitoring/nagios-nrpe-server-config/
 generate_nagios3_config.sh~
+Gemfile.lock

Dockerfile

@@ -42,6 +42,7 @@ RUN ( \
       echo "fi"; \
       echo "chown -R autossh:autossh /home/autossh/.ssh/"; \
       echo ""; \
+      echo "/app/util/install-nrpe.sh"; \
       echo ""; \
       echo "# Start service"; \
       echo "#"; \

util/demographicsImporter/demographicsImporter.js

@@ -0,0 +1,175 @@
+// Retrieve
+var MongoClient = require('mongodb').MongoClient;
+var util = require('util');
+
+var preconditions = true;
+
+if(process.env.QUERY_TITLE === null || process.env.QUERY_TITLE === undefined)
+{
+         console.log('ERROR: QUERY_TITLE environment variable not set.');
+}
+else
+{
+	console.log('QUERY_TITLE: ' + process.env.QUERY_TITLE);
+}
+
+
+if(process.env.RETRO_QUERY_TITLE === null || process.env.RETRO_QUERY_TITLE === undefined)
+{
+         console.log('ERROR: RETRO_QUERY_TITLE environment variable not set.');
+}
+else
+{
+	console.log('RETRO_QUERY_TITLE: ' + process.env.RETRO_QUERY_TITLE);
+}
+// Connect to the db
+MongoClient.connect('mongodb://localhost:27017/query_composer_development', function(err, db) {
+  if(err) { return console.dir(err); }
+
+  db.collection('queries', null,
+    function(err, queriesCollection)
+    {
+      if(err) { throw err; }
+
+      //fetch the retro query excution
+      //****************************************************************************
+      queriesCollection.find({title:process.env.RETRO_QUERY_TITLE}).toArray(
+        function(err, retroQueries)
+        {
+          if(err) { throw err; }
+
+          if(retroQueries.length != 1)
+          {
+            throw new Error('Not one and only one retro query: ' + process.env.RETRO_QUERY_TITLE);
+          }
+
+          var retroQuery = retroQueries[0];
+
+          if(retroQuery.executions.length != 1)
+          {
+            throw new Error('Not one and only one execution for retro query: ' + process.env.RETRO_QUERY_TITLE);
+          }
+
+          retroQuery.executions = retroQuery.executions.sort(
+            function(a,b)
+            {
+              return a.time > b.time ? 1 : b.time > a.time ? -1 : 0;
+            }
+          );
+
+          var retroQueryExecution = retroQuery.executions[0];
+
+          //****************************************************************************
+
+          //fetch query executions
+          //****************************************************************************
+          var queries = queriesCollection.find({title:process.env.QUERY_TITLE}).toArray(
+            function(err, queries)
+            {
+              if(err) { throw err; }
+
+              if(queries.length != 1)
+              {
+                throw new Error('Not one and only one query: ' + process.env.QUERY_TITLE);
+              }
+
+              var query = queries[0];
+
+              var queryExecutions = query.executions;
+              //****************************************************************************
+
+              //fetch retro retroResults
+              //****************************************************************************
+              var retroResults = retroQueryExecution.aggregate_result;
+              //****************************************************************************
+
+              //build simulated executions
+              //****************************************************************************
+              var simulatedExecutions = {};
+
+              for( var key in retroResults)
+              {
+                if( !retroResults.hasOwnProperty(key))
+                {
+                  continue;
+                }
+
+                var execution = JSON.parse(key);
+
+                var date = execution.date/1000;//leave in milliseconds
+                var simulatedExecution;
+
+                if(!execution.gender || !execution.ageRange || !execution.pid || !execution.date)
+		{
+			throw new Error('Error: Missing field');
+		}
+
+		var ar_key = execution.gender + '_' +
+                             execution.ageRange + '_' +
+                             execution.pid;
+
+		if(!simulatedExecutions[date])
+		{
+			var aggregate_result = {};
+			aggregate_result[ar_key] = retroResults[key];
+			simulatedExecution = {'_id':retroQueryExecution._id,
+                                      'aggregate_result':aggregate_result,
+                                      'notification':null,
+                                      'time':date,
+                                      'simulated':true
+                                     };
+			simulatedExecutions[date] = simulatedExecution;
+		}
+		else
+		{
+			simulatedExecution = simulatedExecutions[date];
+			simulatedExecution.aggregate_result[ar_key] = retroResults[key];
+		}
+              }
+              //****************************************************************************
+
+              //add the simulated executions to the execution List
+              //****************************************************************************
+              for( var se in simulatedExecutions)
+              {
+                if( !simulatedExecutions.hasOwnProperty(se))
+                {
+                  continue;
+                }
+
+                queryExecutions.push( simulatedExecutions[se] );
+              }
+              //****************************************************************************
+
+              //update the query with the retro results
+              //****************************************************************************
+
+              queriesCollection.updateOne({title:process.env.QUERY_TITLE}, {$set:{executions:queryExecutions}}, {upsert:true},
+                  function(err, result)
+                  {
+                    if(err)
+                    {
+                      throw new Error(err);
+                    }
+
+		    db.close(
+			function(err, result)
+			{
+				if(err)
+				{
+					console.log('ERROR: closing db - ' + error);
+				}
+				else
+				{
+					console.log('SUCCESS');
+				}
+			});
+                  });
+
+              //****************************************************************************
+            });
+
+        });
+    }
+  );
+});

util/nrpe/add_nrpe_command.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# This script adds a command to the NRPE service, command names must be compatible with 
+# 	the command names used by the Nagios server to access the NRPE commands. 
+# 
+# This script takes 3 arguments: $ ./add_nrpe_command.sh <PATH_TO_SCRIPT> <ENDPOINT_NUMBER> <PATH_TO_NRPE_CONFIG>, for example: 
+# 
+# 	$ ./add_nrpe_command.sh /usr/local/lib/nagios/check_processes.sh 5 /etc/nagios/nrpe_local.cfg
+# 
+# If the PATH_TO_NRPE_CONFIG argument is not given a default /etc/nagios/nrpe_local.cfg is used. 
+#
+# Exit Codes for the script are: 
+#
+# 	0 - Completed noramally, command succcessfully added.
+# 	1 - Error due to invalid parameters
+# 	2 - Did not complete due to command already being installed. 
+
+if [ $# -ne 2 ] && [ $# -ne 3 ]; then
+	echo ""
+	echo "ERROR: Invalid number of parameters to script, received "$#" parameters, expected 2 or 3". 
+	echo "Usage: $ ./add_nrpe_command.sh <PATH_TO_SCRIPT> <ENDPOINT_NUMBER> <PATH_TO_NRPE_CONFIG>"
+	echo ""
+	exit 1
+fi
+
+CMD_SCRIPT=$1
+EP=$2
+
+if [ $# == 3 ]; then 
+	NRPE_CONFIG=$3
+else
+	NRPE_CONFIG=/etc/nagios/nrpe_local.cfg
+fi 
+
+# Check that all files exist as expected. 
+
+# Check that config file is in place.
+if ! [ -w $NRPE_CONFIG ]; then
+	echo "ERROR: no such file: $NRPE_CONFIG"	
+	exit 1
+fi
+
+# Check that nrpe script file is in place.
+if ! [ -x $CMD_SCRIPT ]; then
+	echo "ERROR: no such file: $CMD_SCRIPT"
+	exit 1
+fi
+
+# Do some manipulation of the names.
+# TODO: Update this to use PDC-XXXX naming scheme; changes need to occur on Nagios server side aswell. 
+
+SCRIPT_NAME=$(basename $CMD_SCRIPT)
+EP_NUM=$(printf "%03d" $EP)
+EP_NAME="pdc"$EP_NUM
+CMD_NAME=${SCRIPT_NAME%.*}"_"$EP_NAME
+CMD="command["$CMD_NAME"]="$CMD_SCRIPT" "$EP
+
+# Check if the script is already installed. 
+
+if grep -q $CMD_NAME $NRPE_CONFIG; then
+	# The command is already installed, we will not overwrite this.
+	#
+	echo "WARNING: $CMD_NAME is already in the $NRPE_CONFIG file, will not overwrite."	
+	exit 2
+else
+	# The command is not already installed, will append this to the config file.
+	#
+	echo $CMD >> $NRPE_CONFIG
+	echo "INFO: Adding $CMD to $NRPE_CONFIG file. "
+	exit 0 
+fi

util/nrpe/deploy_monitoring_scripts.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# This script deploys the scripts required for NRPE to monitor endpoints.
+#
+# !!!!!!!!!!!!!!!!! THIS SCRIPT WILL REMOVE ANY EXISTING NRPE CONFIGURATIONS !!!!!!!!!!!!!!!!!!!!!!!
+# 
+# Assumptions: 
+# 	
+# 	* This script assumes that the PDC composer code is deployed within the /app directory of this host.
+#
+# In summary this script: 
+# 
+# 	1) Creates (or clears) the directory /usr/local/lib/nagios/
+# 	2) Copies the monitoring scripts from ./monitoring_scripts/ into /usr/local/lib/nagios/	
+# 	3) Sets up commands in /etc/nagios/nrpe_local.cfg to use the scripts.
+# 	4) Restarts the NRPE server.
+# 
+# This scripts takes a list of endpoint id's to deploy monitoring for example: 
+#
+#	$ deploy_monitoring_scripts.sh 1 2 3 4 5
+#
+# Will set up the scripts for endpoints 1 through 5
+
+echo ""
+echo ""
+echo "---------------------------------------------------"
+echo "THIS SCRIPT WILL REMOVE ANY EXISTING NRPE CONFIGURATIONS"
+read -p "Press ENTER to continue, CTRL-C to halt."
+echo "---------------------------------------------------"
+echo ""
+echo ""
+
+if [ $# == 0 ]; then
+	echo "ERROR: This script takes endpoint number arguments, $# were provided, exiting..."
+	echo ""
+	exit 1
+
+fi
+
+# Set up some variables
+
+BASE_DIR=/app/util/nrpe/
+SCRIPT_DEPLOY_DIR=/usr/local/lib/nagios
+NRPE_CFG_FILE=/etc/nagios/nrpe_local.cfg
+
+# 1) Remove any existing scripts within the /usr/local/lib/nagios/ directory.
+
+rm -rf $SCRIPT_DEPLOY_DIR
+
+mkdir -p $SCRIPT_DEPLOY_DIR
+
+# 2) Copy in the default scripts from within the ./monitoring_scripts/ directory. 
+
+cp -r $BASE_DIR""monitoring_scripts/* $SCRIPT_DEPLOY_DIR
+
+# 3) Set up the commands in /etc/nagios/nrpe_local.cfg, we use another script to support this.
+
+# 3.1) Remove existing config file
+
+rm -rf $NRPE_CFG_FILE
+touch $NRPE_CFG_FILE
+
+for ep in $@ #gets all arguments to the script.
+do
+	echo ""
+	echo "Configuring scripts for endpoint: "$ep
+	echo "---------------------------------"
+
+	for f in $SCRIPT_DEPLOY_DIR/* #get all executable filese
+	do
+		if [[ -x $f ]]; then
+			./add_nrpe_command.sh $f $ep $NRPE_CFG_FILE
+		fi
+	done
+	echo "================================"
+done
+
+# 4) Restart nagios-nrpe server
+
+echo ""
+echo "Restarting NRPE Server: "
+echo "------------------------"
+
+sudo service nagios-nrpe-server restart
+
+echo ""
+echo "-----------------------"
+echo "All Done, Exiting..."
+
+exit 0

util/nrpe/endpoints.txt

@@ -0,0 +1 @@
+1 2 3 4

util/nrpe/install-nrpe.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+PWD=$(pwd)
+
+NRPE_PORT=3010 
+NRPE_CONFIG=/etc/nagios/nrpe.cfg
+
+# install the NRPE service
+
+sudo apt-get update
+sudo apt-get install nagios-nrpe-plugin -y 
+
+# Set the port for NRPE to listen on: 
+
+sed -i 's/server_port=[0-9]\+/server_port='$NRPE_PORT'/g' $NRPE_CONFIG
+
+# Make the /usr/local/lib/nagios directory for storing plugin scripts
+
+mkdir -p /usr/local/lib/nagios
+
+# Make a nrpe_local.cfg file if it does not already exist
+
+touch /etc/nagios/nrpe_local.cfg 
+
+# Move into the working directory
+
+cd /app/util/nrpe/
+
+# Use the file endpoints.txt as a list of endpoint numbers to use.
+
+./deploy_monitoring_scripts.sh $(cat endpoints.txt)
+
+# Move back to original directory
+
+cd $PWD