Merge pull request #124 from neighborhoods/KOJO-242-process-group-awa…

…reness

KOJO-242 | Process group awareness

src/Process/Pool.php

@@ -51,6 +51,10 @@ protected function _childExitSignal(InformationInterface $information): PoolInte
             $this->_getLogger()->debug("Child process[$childProcessId] is not in the pool for process[$processId].");
         }
 
+        if ($information->getExitValue() === SIGKILL) {
+            $this->_getProcessPoolStrategy()->handlePotentiallyStrayProcesses();
+        }
+
         return $this;
     }
 

src/Process/Pool/Strategy.php

@@ -213,4 +213,84 @@ protected function _unPauseListenerProcesses(): Strategy
 
         return $this;
     }
+
+    public function handlePotentiallyStrayProcesses() : StrategyInterface
+    {
+        $this->_getLogger()->notice('SIGKILL of Worker process detected');
+
+        // get the status file for every process
+        foreach (glob('/proc/[0-9]*/status') as $procStatusFile) {
+            $processId = null;
+            $parentProcessId = null;
+            $processGroupId = null;
+
+            try {
+                // files in /proc/ can be deleted at any time
+                // suppress PHP's warning (which will become an error)
+                // we'll check for failure after attempting to open
+                $procStatusFd = @fopen($procStatusFile, 'r');
+
+                // this file has been deleted (or we don't have permission)
+                if ($procStatusFd === false) {
+                    continue;
+                }
+
+                // parse the file for the IDs above
+                while (($line = fgets($procStatusFd))) {
+                    [$item] = sscanf($line, "Pid:\t%s");
+                    if ($item !== null) {
+                        $processId = (int)$item;
+                    }
+
+                    [$item] = sscanf($line, "PPid:\t%s");
+                    if ($item !== null) {
+                        $parentProcessId = (int)$item;
+                    }
+
+                    [$item] = sscanf($line, "NSpgid:\t%s");
+                    if ($item !== null) {
+                        $processGroupId = (int)$item;
+                    }
+
+                    // if we've extracted all the information we need,
+                    // check if this is a process we need to clean up
+                    if ($processId && $parentProcessId && $processGroupId) {
+                        if (
+                            // was it orphaned
+                            $parentProcessId === 1 &&
+                            // was it spawned from the same ancestor (i.e. the Server)
+                            $processGroupId === $this->_getProcessPool()->getProcess()->getProcessGroupId() &&
+                            // guard against false positives
+                            // is not the Root
+                            $processId !== $this->_getProcessPool()->getProcess()->getProcessId() &&
+                            // is not init
+                            $processId !== 1
+                        ) {
+                            $this->_getLogger()->warning(
+                                'Terminating orphaned process',
+                                [
+                                    'process_id' => $processId,
+                                    'parent_process_id' => $parentProcessId,
+                                    'process_group_id' => $processGroupId,
+                                    // Root information is included in the kojo_metadata
+                                ]
+                            );
+                            // SIGKILL must be used here because watchdogs won't handle any signals
+                            // any (unexpected) orphan processes that result from this will be
+                            // cleaned up in the next pass
+                            posix_kill($processId, SIGKILL);
+                        }
+                        // move on to the next /proc/ file regardless
+                        break;
+                    }
+                }
+            } finally {
+                if ($procStatusFd !== false) {
+                    fclose($procStatusFd);
+                }
+            }
+        }
+
+        return $this;
+    }
 }

src/Process/Pool/Strategy/Server.php

@@ -67,4 +67,13 @@ public function initializePool(): StrategyInterface
 
         return $this;
     }
+
+    public function handlePotentiallyStrayProcesses() : StrategyInterface
+    {
+        // since the Root is the only child of the Server, this will only be invoked on SIGKILL of the Root
+        // that shouldn't be a problem, since all the children of the Root are responsible for terminating
+        // themselves (as opposed to children of Workers, which need their parents to terminate them)
+
+        return $this;
+    }
 }

src/Process/Pool/Strategy/Worker.php

@@ -70,4 +70,12 @@ public function initializePool(): StrategyInterface
 
         return $this;
     }
+
+    public function handlePotentiallyStrayProcesses() : StrategyInterface
+    {
+        // Watchdogs are always SIGKILLed because they need to be unable to handle signals like SIGTERM
+        // so this is normal operation, no action needs to be taken
+
+        return $this;
+    }
 }

src/Process/Pool/StrategyInterface.php

@@ -47,4 +47,6 @@ public function setFillProcessTypeCode(string $fillProcessTypeCode): StrategyInt
     public function setMaximumLoadAverage(float $maximumLoadAverage): StrategyInterface;
 
     public function getMaximumLoadAverage(): float;
-}
+
+    public function handlePotentiallyStrayProcesses() : StrategyInterface;
+}

src/ProcessAbstract.php

@@ -29,6 +29,7 @@ protected function _initialize(): ProcessAbstract
         $this->_getApmNewRelic()->endTransaction();
         $this->_setParentProcessId(posix_getppid());
         $this->_setProcessId(posix_getpid());
+        $this->_setProcessGroupId(posix_getpgrp());
 
         $this->getProcessPoolLoggerMessageMetadataBuilder()->setProcess($this);
         if ($this->_hasProcessPool()) {
@@ -203,6 +204,18 @@ public function getParentProcessId(): int
         return $this->_read(self::PROP_PARENT_PROCESS_ID);
     }
 
+    protected function _setProcessGroupId(int $processGroupId): ProcessAbstract
+    {
+        $this->_create(self::PROP_PROCESS_GROUP_ID, $processGroupId);
+
+        return $this;
+    }
+
+    public function getProcessGroupId(): int
+    {
+        return $this->_read(self::PROP_PROCESS_GROUP_ID);
+    }
+
     public function setExitCode(int $exitCode): ProcessInterface
     {
         $this->_create(self::PROP_EXIT_CODE, $exitCode);

src/ProcessInterface.php

@@ -14,6 +14,7 @@ interface ProcessInterface extends HandlerInterface
     public const PROP_PATH                                     = 'path';
     public const PROP_TERMINATION_SIGNAL_NUMBER                = 'termination_signal_number';
     public const PROP_PROCESS_ID                               = 'process_id';
+    public const PROP_PROCESS_GROUP_ID                         = 'process_group_id';
     public const PROP_TYPE_CODE                                = 'type_code';
     public const PROP_UUID                                     = 'uuid';
     public const PROP_UUID_MAXIMUM_INTEGER                     = 'uuid_maximum_integer';
@@ -29,6 +30,8 @@ public function start(): ProcessInterface;
 
     public function getProcessId(): int;
 
+    public function getProcessGroupId(): int;
+
     public function setLogger(LoggerInterface $logger);
 
     public function setThrottle(int $seconds = 0): ProcessInterface;