praefectus liveness probe fixes

opsway · Aug 25, 2022 · 2a85906 · 2a85906
1 parent e3d3add
commit 2a85906
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 11 deletions.
diff --git a/internal/metrics/stateStorage.go b/internal/metrics/stateStorage.go
@@ -77,12 +77,12 @@ func (s *StateStorage) WorkerStatePercentage(state WorkerState, start int64, end
 func (s *StateStorage) CheckIdleFreeze(timeSpentLimit time.Duration) bool {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	start := time.Now().UnixMilli() - timeSpentLimit.Milliseconds()
+	now := time.Now().UnixMilli()
 	for _, timeWorkerState := range s.storage {
 		if timeWorkerState.state != WorkerStateIdle {
 			continue
 		}
-		if timeWorkerState.timestamp < start && timeWorkerState.secondsInState == nil {
+		if now-timeWorkerState.timestamp > int64(timeSpentLimit*time.Millisecond) && timeWorkerState.secondsInState == nil {
 			return true
 		}
 	}

diff --git a/internal/workers/liveness.go b/internal/workers/liveness.go
@@ -40,8 +40,8 @@ func (s *Liveness) Check(request string, reply *bool) error {
 
 	now := time.Now().Unix()
 
-	upScaleFreeze := lastUpScale == nil || (now-lastUpScale.Timestamp) > int64(s.pool.config.ScaleTick.Seconds())
-	downScaleFreeze := lastDownScale == nil || (now-lastDownScale.Timestamp) > int64(s.pool.config.DownscaleTick.Seconds())
+	upScaleFreeze := lastUpScale == nil || (now-lastUpScale.Timestamp) > int64(s.pool.config.ScaleTick)
+	downScaleFreeze := lastDownScale == nil || (now-lastDownScale.Timestamp) > int64(s.pool.config.DownscaleTick)
 
 	workersHasIdleFreeze := s.pool.checkWorkersIdleFreeze(s.pool.config.ProcessIdleSpentLimit)
 
@@ -52,18 +52,18 @@ func (s *Liveness) Check(request string, reply *bool) error {
 		"runningProcesses":          runningProcesses,
 		"poolState":                 s.pool.state != PoolStopped,
 		"wsStorage":                 s.pool.wsStorage.Length() > 0,
-		"upScaleFreeze":             upScaleFreeze,
-		"DownScaleFreeze":           downScaleFreeze,
-		"workerHasFreeze":           workersHasIdleFreeze,
+		"upScaleFreeze":             upScaleFreeze == false,
+		"downScaleFreeze":           downScaleFreeze == false,
+		"workerHasFreeze":           workersHasIdleFreeze == false,
 		"workerIpcCheck":            workersIpcCheck,
 	}).Debug("Liveness result")
 
 	*reply = poolActiveWorkers > 0 &&
 		poolActiveWorkers <= runningWorkersNumber &&
 		s.pool.state != PoolStopped &&
 		s.pool.wsStorage.Length() > -1 &&
-		!upScaleFreeze &&
-		!downScaleFreeze &&
+		upScaleFreeze == false &&
+		downScaleFreeze == false &&
 		workersHasIdleFreeze == false &&
 		workersIpcCheck
 

diff --git a/internal/workers/scaler.go b/internal/workers/scaler.go
@@ -68,8 +68,8 @@ func (p *ScalePool) tryDownscaleWorkers() {
 	}
 	p.lastDownScale = &config.LastRunSeconds{Timestamp: time.Now().Unix()}
 	workerPercentage, activeWorkers, idleWorkers := p.processIdleWorkers()
-	if workerPercentage == 0 || idleWorkers == nil {
-		log.Panic("processIdleWorkers inappropriate result on active pool")
+	if workerPercentage == 0 && idleWorkers == nil {
+		log.Warning("processIdleWorkers inappropriate result on active pool")
 		return
 	}