Fix supervisord service not enabled on Ubuntu

Change supervisord service script from SysVinit to Systemd for all OS.
Add kitchen test to verify service is enabled. Also add check for other required services

This solves https://github.com/aws/aws-parallelcluster/wiki/ParallelCluster-3.0.0-on-Ubuntu-18-and-20:-scaling-daemon-is-down-after-a-head-node-reboot

Signed-off-by: Luca Carrogu <[email protected]>

Author: lukeseawalker

CHANGELOG.md

@@ -6,6 +6,16 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 3.x.x
 ------
 
+**CHANGES**
+- Change supervisord service script from SysVinit to Systemd.
+- Do not configure GPUs in Slurm when Nvidia driver is not installed.
+- 
+**BUG FIXES**
+- Fix supervisord service not enabled on Ubuntu.
+
+3.x.x
+------
+
 **CHANGES**
 - Do not configure GPUs in Slurm when Nvidia driver is not installed.
 

recipes/base_install.rb

@@ -174,13 +174,14 @@
   mode "0644"
 end
 
-# Put init script in place
-template "supervisord-init" do
-  source 'supervisord-init.erb'
-  path "/etc/init.d/supervisord"
+# Put supervisord service in place
+template "supervisord-service" do
+  source 'supervisord-service.erb'
+  path "/etc/systemd/system/supervisord.service"
   owner "root"
   group "root"
-  mode "0755"
+  mode "0644"
+  only_if { node['init_package'] == 'systemd' }
 end
 
 # AMI cleanup script

recipes/tests.rb

@@ -58,6 +58,9 @@
 # Slurm
 ###################
 if node['cluster']['scheduler'] == 'slurm'
+  execute 'check munge service is enabled' do
+    command "systemctl is-enabled munge"
+  end
   case node['cluster']['node_type']
   when 'HeadNode'
     execute 'execute sinfo' do
@@ -86,6 +89,9 @@
     execute 'ensure-pmix-shared-library-can-be-found' do
       command '/opt/pmix/bin/pmix_info'
     end
+    execute 'check slurmctld service is enabled' do
+      command "systemctl is-enabled slurmctld"
+    end
   when 'ComputeFleet'
     execute 'ls slurm root' do
       command "ls /opt/slurm"
@@ -128,6 +134,10 @@
   command chrony_check_command
 end
 
+execute 'check chrony service is enabled' do
+  command "systemctl is-enabled #{node['cluster']['chrony']['service']}"
+end
+
 execute 'check chrony conf' do
   command "chronyc waitsync 30; chronyc tracking | grep -i reference | grep 169.254.169.123"
   user node['cluster']['cluster_user']
@@ -155,6 +165,9 @@
 end
 
 if node['conditions']['dcv_supported'] && node['cluster']['dcv_enabled'] == "head_node" && node['cluster']['node_type'] == "HeadNode"
+  execute 'check dcvserver service is enabled' do
+    command "systemctl is-enabled dcvserver"
+  end
   execute 'check systemd default runlevel' do
     command "systemctl get-default | grep -i graphical.target"
   end
@@ -369,6 +382,9 @@ module load intelmpi && mpirun --help | grep '#{node['cluster']['intelmpi']['kit
   EPHEMERAL
   user node['cluster']['cluster_user']
 end
+execute 'check setup-ephemeral service is enabled' do
+  command "systemctl is-enabled setup-ephemeral"
+end
 
 ###################
 # Pcluster AWSBatch CLI
@@ -459,3 +475,18 @@ module load intelmpi && mpirun --help | grep '#{node['cluster']['intelmpi']['kit
     NOFFTW
   end
 end
+
+###################
+# Verify required service are enabled
+###################
+if node['cluster']['node_type'] == 'HeadNode'
+  execute 'check parallelcluster-iptables service is enabled' do
+    command "systemctl is-enabled parallelcluster-iptables"
+  end
+end
+execute 'check supervisord service is enabled' do
+  command "systemctl is-enabled supervisord"
+end
+execute 'check ec2blkdev service is enabled' do
+  command "systemctl is-enabled ec2blkdev"
+end

templates/default/supervisord-init.erb

@@ -0,0 +1,17 @@
+# Downloaded from:
+# https://git.launchpad.net/ubuntu/+source/supervisor/tree/debian/supervisor.service
+[Unit]
+Description=Supervisor process control system for UNIX
+Documentation=http://supervisord.org
+After=network.target
+
+[Service]
+ExecStart=<%= node['cluster']['cookbook_virtualenv_path'] %>/bin/supervisord -n -c /etc/supervisord.conf
+ExecStop=<%= node['cluster']['cookbook_virtualenv_path'] %>/bin/supervisorctl $OPTIONS shutdown
+ExecReload=<%= node['cluster']['cookbook_virtualenv_path'] %>/bin/supervisorctl -c /etc/supervisord.conf $OPTIONS reload
+KillMode=process
+Restart=on-failure
+RestartSec=50s
+
+[Install]
+WantedBy=multi-user.target