diff --git a/src/Process/Pool.php b/src/Process/Pool.php index 4cc6273f..2d13c4bb 100644 --- a/src/Process/Pool.php +++ b/src/Process/Pool.php @@ -51,6 +51,10 @@ protected function _childExitSignal(InformationInterface $information): PoolInte $this->_getLogger()->debug("Child process[$childProcessId] is not in the pool for process[$processId]."); } + if ($information->getExitValue() === SIGKILL) { + $this->_getProcessPoolStrategy()->handlePotentiallyStrayProcesses(); + } + return $this; } diff --git a/src/Process/Pool/Strategy.php b/src/Process/Pool/Strategy.php index ac30df6a..8c67edb1 100644 --- a/src/Process/Pool/Strategy.php +++ b/src/Process/Pool/Strategy.php @@ -213,4 +213,84 @@ protected function _unPauseListenerProcesses(): Strategy return $this; } + + public function handlePotentiallyStrayProcesses() : StrategyInterface + { + $this->_getLogger()->notice('SIGKILL of Worker process detected'); + + // get the status file for every process + foreach (glob('/proc/[0-9]*/status') as $procStatusFile) { + $processId = null; + $parentProcessId = null; + $processGroupId = null; + + try { + // files in /proc/ can be deleted at any time + // suppress PHP's warning (which will become an error) + // we'll check for failure after attempting to open + $procStatusFd = @fopen($procStatusFile, 'r'); + + // this file has been deleted (or we don't have permission) + if ($procStatusFd === false) { + continue; + } + + // parse the file for the IDs above + while (($line = fgets($procStatusFd))) { + [$item] = sscanf($line, "Pid:\t%s"); + if ($item !== null) { + $processId = (int)$item; + } + + [$item] = sscanf($line, "PPid:\t%s"); + if ($item !== null) { + $parentProcessId = (int)$item; + } + + [$item] = sscanf($line, "NSpgid:\t%s"); + if ($item !== null) { + $processGroupId = (int)$item; + } + + // if we've extracted all the information we need, + // check if this is a process we need to clean up + if ($processId && $parentProcessId && $processGroupId) { + if ( + // was it orphaned + $parentProcessId === 1 && + // was it spawned from the same ancestor (i.e. the Server) + $processGroupId === $this->_getProcessPool()->getProcess()->getProcessGroupId() && + // guard against false positives + // is not the Root + $processId !== $this->_getProcessPool()->getProcess()->getProcessId() && + // is not init + $processId !== 1 + ) { + $this->_getLogger()->warning( + 'Terminating orphaned process', + [ + 'process_id' => $processId, + 'parent_process_id' => $parentProcessId, + 'process_group_id' => $processGroupId, + // Root information is included in the kojo_metadata + ] + ); + // SIGKILL must be used here because watchdogs won't handle any signals + // any (unexpected) orphan processes that result from this will be + // cleaned up in the next pass + posix_kill($processId, SIGKILL); + } + // move on to the next /proc/ file regardless + break; + } + } + } finally { + if ($procStatusFd !== false) { + fclose($procStatusFd); + } + } + } + + return $this; + } } diff --git a/src/Process/Pool/Strategy/Server.php b/src/Process/Pool/Strategy/Server.php index 40d4b7a9..4d9eea08 100644 --- a/src/Process/Pool/Strategy/Server.php +++ b/src/Process/Pool/Strategy/Server.php @@ -67,4 +67,13 @@ public function initializePool(): StrategyInterface return $this; } + + public function handlePotentiallyStrayProcesses() : StrategyInterface + { + // since the Root is the only child of the Server, this will only be invoked on SIGKILL of the Root + // that shouldn't be a problem, since all the children of the Root are responsible for terminating + // themselves (as opposed to children of Workers, which need their parents to terminate them) + + return $this; + } } diff --git a/src/Process/Pool/Strategy/Worker.php b/src/Process/Pool/Strategy/Worker.php index 33792968..882b64c2 100644 --- a/src/Process/Pool/Strategy/Worker.php +++ b/src/Process/Pool/Strategy/Worker.php @@ -70,4 +70,12 @@ public function initializePool(): StrategyInterface return $this; } + + public function handlePotentiallyStrayProcesses() : StrategyInterface + { + // Watchdogs are always SIGKILLed because they need to be unable to handle signals like SIGTERM + // so this is normal operation, no action needs to be taken + + return $this; + } } diff --git a/src/Process/Pool/StrategyInterface.php b/src/Process/Pool/StrategyInterface.php index 3bca034e..4bebd110 100644 --- a/src/Process/Pool/StrategyInterface.php +++ b/src/Process/Pool/StrategyInterface.php @@ -47,4 +47,6 @@ public function setFillProcessTypeCode(string $fillProcessTypeCode): StrategyInt public function setMaximumLoadAverage(float $maximumLoadAverage): StrategyInterface; public function getMaximumLoadAverage(): float; -} \ No newline at end of file + + public function handlePotentiallyStrayProcesses() : StrategyInterface; +} diff --git a/src/ProcessAbstract.php b/src/ProcessAbstract.php index 6d083f4e..57684e59 100644 --- a/src/ProcessAbstract.php +++ b/src/ProcessAbstract.php @@ -29,6 +29,7 @@ protected function _initialize(): ProcessAbstract $this->_getApmNewRelic()->endTransaction(); $this->_setParentProcessId(posix_getppid()); $this->_setProcessId(posix_getpid()); + $this->_setProcessGroupId(posix_getpgrp()); $this->getProcessPoolLoggerMessageMetadataBuilder()->setProcess($this); if ($this->_hasProcessPool()) { @@ -203,6 +204,18 @@ public function getParentProcessId(): int return $this->_read(self::PROP_PARENT_PROCESS_ID); } + protected function _setProcessGroupId(int $processGroupId): ProcessAbstract + { + $this->_create(self::PROP_PROCESS_GROUP_ID, $processGroupId); + + return $this; + } + + public function getProcessGroupId(): int + { + return $this->_read(self::PROP_PROCESS_GROUP_ID); + } + public function setExitCode(int $exitCode): ProcessInterface { $this->_create(self::PROP_EXIT_CODE, $exitCode); diff --git a/src/ProcessInterface.php b/src/ProcessInterface.php index 599eb8a6..2a76c68c 100644 --- a/src/ProcessInterface.php +++ b/src/ProcessInterface.php @@ -14,6 +14,7 @@ interface ProcessInterface extends HandlerInterface public const PROP_PATH = 'path'; public const PROP_TERMINATION_SIGNAL_NUMBER = 'termination_signal_number'; public const PROP_PROCESS_ID = 'process_id'; + public const PROP_PROCESS_GROUP_ID = 'process_group_id'; public const PROP_TYPE_CODE = 'type_code'; public const PROP_UUID = 'uuid'; public const PROP_UUID_MAXIMUM_INTEGER = 'uuid_maximum_integer'; @@ -29,6 +30,8 @@ public function start(): ProcessInterface; public function getProcessId(): int; + public function getProcessGroupId(): int; + public function setLogger(LoggerInterface $logger); public function setThrottle(int $seconds = 0): ProcessInterface;