Fixing supervisord

Fixing killing of processes

Author: jeniawhite

src/deploy/NVA_build/deploy_base.sh

@@ -351,8 +351,8 @@ function fix_security_issues {
 
 function setup_supervisors {
 	deploy_log "setup_supervisors start"
-
     mkdir -p /tmp/supervisor
+    mv /usr/bin/supervisord /usr/bin/supervisord_orig
     # Generate default supervisord config
     echo_supervisord_conf > /etc/supervisord.conf
     sed -i 's:logfile=.*:logfile=/tmp/supervisor/supervisord.log:' /etc/supervisord.conf
@@ -362,15 +362,17 @@ function setup_supervisors {
     # Autostart supervisor
     deploy_log "setup_supervisors autostart"
     cp -f ${CORE_DIR}/src/deploy/NVA_build/supervisord.orig /etc/rc.d/init.d/supervisord
+    cp -f ${CORE_DIR}/src/deploy/NVA_build/supervisord.orig /usr/bin/supervisord
     chmod 777 /etc/rc.d/init.d/supervisord
+    chmod 777 /usr/bin/supervisord
     chkconfig supervisord on
 
     # Add NooBaa services configuration to supervisor
     deploy_log "setup_supervisors adding noobaa config to supervisord"
     echo "[include]" >> /etc/supervisord.conf
     echo "files = /etc/noobaa_supervisor.conf" >> /etc/supervisord.conf
     cp -f ${CORE_DIR}/src/deploy/NVA_build/noobaa_supervisor.conf /etc
-    ${SUPERD}
+    ${SUPERD} start
     ${SUPERCTL} reread
     ${SUPERCTL} update
 

src/deploy/NVA_build/supervisord.orig

@@ -4,30 +4,57 @@
 
 . /etc/rc.d/init.d/functions
 
-SUPERVISORD="/usr/bin/supervisord"
+SUPERVISORD="/usr/bin/supervisord_orig"
 PIDFILE="/tmp/supervisord.pid"
 
 start() {
     if [ ! -x "$SUPERVISORD" ]; then
         echo "$SUPERVISORD is not executable."
+        logger -p local0.info "$SUPERVISORD is not executable."
         exit 1
     fi
-
     echo "Starting ..."
+    logger -p local0.info "Starting ..."
     ulimit -n 102400; $SUPERVISORD --pidfile $PIDFILE
 
     return $?
 }
 
 stop() {
     echo "Stopping ..."
+    logger -p local0.info "Stopping ..."
+    local supervisord_orig_pid=$(pgrep -lf $SUPERVISORD | awk '{print $1}')
+    echo "supervisord_orig_pid=$supervisord_orig_pid"
+    logger -p local0.info "supervisord_orig_pid=$supervisord_orig_pid"
+    kill -9 ${supervisord_orig_pid}
+
+    local web_server_pid=$(pgrep -lf web_server.js | awk '{print $1}')
+    echo "web_server_pid=$web_server_pid"
+    logger -p local0.info "web_server_pid=$web_server_pid"
+    kill -9 ${web_server_pid}
+
+    local bg_workers_pid=$(pgrep -lf bg_workers.js | awk '{print $1}')
+    echo "bg_workers_pid=$bg_workers_pid"
+    logger -p local0.info "bg_workers_pid=$bg_workers_pid"
+    kill -9 ${bg_workers_pid}
+
+    local hosted_agents_starter_pid=$(pgrep -lf hosted_agents_starter.js | awk '{print $1}')
+    echo "hosted_agents_starter_pid=$hosted_agents_starter_pid"
+    logger -p local0.info "hosted_agents_starter_pid=$hosted_agents_starter_pid"
+    kill -9 ${hosted_agents_starter_pid}
+
+    local s3rver_starter_pids=$(pgrep -lf s3rver_starter.js | awk '{print $1}')
+    echo "s3rver_starter_pids=$s3rver_starter_pids"
+    logger -p local0.info "s3rver_starter_pids=$s3rver_starter_pids"
+    kill -9 ${s3rver_starter_pids}
+
+    local mongo_pids=$(pgrep -lf mongo | awk '{print $1}')
+    echo "mongo_pids=$mongo_pids"
+    logger -p local0.info "mongo_pids=$mongo_pids"
+    kill -9 ${mongo_pids}
+
     local p=$(cat $PIDFILE)
     kill -9 ${p}
-    #unlink /tmp/supervisor.sock
-    #local pid=$(lsof -i :8080|awk  '{ print $2 }' | sed '1 d')
-    #for p in ${pid} ; do
-      #kill -9 ${p}
-    #done
     [ $? -eq 0 ] && rm -f $PIDFILE
     return $retval
 }

src/deploy/init.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-/usr/bin/supervisord
+/usr/bin/supervisord start

src/test/scripts/reset_cluster.sh

@@ -8,9 +8,9 @@ echo "removing $(hostname) from the cluster..."
 cp -f /root/node_modules/noobaa-core/src/deploy/NVA_build/noobaa_supervisor.conf /etc/noobaa_supervisor.conf
 sed -i "s:MONGO_RS_URL.*::" /root/node_modules/noobaa-core/.env
 supervisorctl shutdown
-supervisord
+supervisord start
 sleep 5
 /usr/bin/mongo nbcore --eval "db.dropDatabase()"
 /usr/bin/mongo local --eval "db.dropDatabase()"
 supervisorctl shutdown
-supervisord
+supervisord start

src/test/system_tests/test_md_aggregator.js

@@ -132,7 +132,7 @@ function init_system_to_ntp() {
         .finally(() => P.resolve()
             .then(() => {
                 console.log('start supervisord');
-                return promise_utils.exec('supervisord', {
+                return promise_utils.exec('supervisord start', {
                     ignore_rc: false,
                     return_stdout: false
                 });

src/upgrade/upgrade.js

@@ -141,7 +141,7 @@ function mongo_upgrade() {
     dbg.log0('mongo_upgrade: Called');
     let secret;
     return disable_autostart()
-        .then(() => promise_utils.exec(`${SUPERD}`, {
+        .then(() => promise_utils.exec(`${SUPERD} start`, {
             ignore_rc: false,
             return_stdout: true,
             trim_stdout: true

src/upgrade/upgrade_wrapper.js

@@ -157,9 +157,16 @@ function post_upgrade() {
             })
         )
         .then(() => exec(`echo "${agent_version_var}" >> ${CORE_DIR}/.env`))
+        .then(() => fs_utils.file_must_not_exist('/usr/bin/supervisord_orig'))
+        .then(
+            () => exec(`mv /usr/bin/supervisord /usr/bin/supervisord_orig`),
+            err => dbg.log0('supervisord_orig already exists, no changes required', err)
+        )
         .then(() => exec(`cp -f ${CORE_DIR}/src/deploy/NVA_build/supervisord.orig /etc/rc.d/init.d/supervisord`))
+        .then(() => exec(`cp -f ${CORE_DIR}/src/deploy/NVA_build/supervisord.orig /usr/bin/supervisord`))
         .then(() => dbg.log0('post_upgrade: Override supervisord'))
         .then(() => exec(`chmod 777 /etc/rc.d/init.d/supervisord`))
+        .then(() => exec(`chmod 777 /usr/bin/supervisord`))
         .then(() => dbg.log0('post_upgrade: Configure permissions to supervisord'))
         .then(() => exec(`cp -f ${CORE_DIR}/src/deploy/NVA_build/first_install_diaglog.sh /etc/profile.d/`))
         .then(() => dbg.log0('post_upgrade: Copying first install dialog into profile.d'))