Merge pull request #68 from it-novum/linux-cpu

openITCOCKPIT Monitoring Agent 3.0.9

Author: nook24

VERSION

@@ -1 +1 @@
-3.0.8
+3.0.9

basiclog/basiclog.go

@@ -0,0 +1,45 @@
+package basiclog
+
+import (
+	"fmt"
+	"runtime"
+	"strings"
+	"time"
+)
+
+// This very basic log gets used to log any config parser errors
+// to the syslog or Windows Event Log on systems that do not have a syslog.
+// This is because if the agent can not parse it's config.ini it has no logfile path.
+
+func (l *BasicLogger) Errorln(args ...interface{}) error {
+	msg := fmt.Sprint(args...)
+
+	t := time.Now()
+	source := l.source()
+	template := "time=\"%v\" level=error msg=\"%v\" source=\"%v\"\n"
+
+	msg = fmt.Sprintf(template, t.Format(time.RFC3339), msg, source)
+
+	// Print msg
+	fmt.Print(msg)
+
+	if l.handler == nil {
+		return fmt.Errorf("No log handler initialized! Message will only be printed to stdout.")
+	}
+
+	// Save msg to logfile via given handler
+	return l.LogError(msg)
+}
+
+func (l *BasicLogger) source() string {
+	_, file, line, ok := runtime.Caller(2)
+	if !ok {
+		file = "<???>"
+		line = 1
+	} else {
+		slash := strings.LastIndex(file, "/")
+		file = file[slash+1:]
+	}
+
+	return fmt.Sprintf("%s:%d", file, line)
+}

basiclog/basiclog_posix.go

@@ -0,0 +1,32 @@
+//go:build linux || darwin
+// +build linux darwin
+
+package basiclog
+
+import (
+	"log/syslog"
+)
+
+// This very basic log gets used to log any config parser errors
+// to the syslog or Windows Event Log on systems that do not have a syslog.
+// This is because if the agent can not parse it's config.ini it has no logfile path.
+
+type BasicLogger struct {
+	handler *syslog.Writer
+}
+
+func New() (*BasicLogger, error) {
+
+	logHandle, err := syslog.New(syslog.LOG_ERR, "openITCOCKPITAgent")
+	if err != nil {
+		return &BasicLogger{}, err
+	}
+
+	return &BasicLogger{
+		handler: logHandle,
+	}, nil
+}
+
+func (l *BasicLogger) LogError(msg string) error {
+	return l.handler.Err(msg)
+}

basiclog/basiclog_windows.go

@@ -0,0 +1,29 @@
+package basiclog
+
+import (
+	"golang.org/x/sys/windows/svc/eventlog"
+)
+
+// This very basic log gets used to log any config parser errors
+// to the syslog or Windows Event Log on systems that do not have a syslog.
+// This is because if the agent can not parse it's config.ini it has no logfile path.
+
+type BasicLogger struct {
+	handler *eventlog.Log
+}
+
+func New() (*BasicLogger, error) {
+
+	logHandle, err := eventlog.Open("openITCOCKPITAgent")
+	if err != nil {
+		return &BasicLogger{}, err
+	}
+
+	return &BasicLogger{
+		handler: logHandle,
+	}, nil
+}
+
+func (l *BasicLogger) LogError(msg string) error {
+	return l.handler.Error(1, msg)
+}

checks/agent_posix.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package checks

checks/agent_windows.go

@@ -5,8 +5,8 @@ import (
 	"runtime"
 	"time"
 
-	"github.com/StackExchange/wmi"
 	"github.com/it-novum/openitcockpit-agent-go/config"
+	"github.com/yusufpapurcu/wmi"
 	"golang.org/x/sys/windows/registry"
 )
 

checks/checks_libvirt_linux.go

@@ -1,3 +1,4 @@
+//go:build libvirt
 // +build libvirt
 
 package checks

checks/checks_linux.go

@@ -1,3 +1,4 @@
+//go:build !libvirt
 // +build !libvirt
 
 package checks

checks/cpu_posix.go

@@ -1,96 +1,147 @@
+//go:build linux || darwin
 // +build linux darwin
 
 package checks
 
 import (
 	"context"
-	"runtime"
+	"fmt"
+	"math"
 	"time"
 
 	"github.com/shirou/gopsutil/v3/cpu"
 )
 
+func saturatingSub(a, b float64) float64 {
+	if a > b {
+		return a - b
+	}
+	return 0
+}
+
+func calcuateBusy(t cpu.TimesStat) (float64, float64, float64) {
+	busy := t.User + t.Nice + t.System + t.Irq + t.Softirq + t.Steal
+	idle := t.Idle + t.Iowait
+	total := busy + idle
+	return busy, idle, total
+}
+
+func calculateUsagePercentage(prev, current cpu.TimesStat) float64 {
+	// This is highly inspired by
+	// https://github.com/shirou/gopsutil/blob/master/cpu/cpu.go#L107
+	// and htop
+	// https://github.com/htop-dev/htop/blob/main/linux/LinuxProcessList.c#L1948-L2006
+	// and yes - htop is reading /proc/stat as we do
+	prevBusy, _, prevTotal := calcuateBusy(prev)
+	currentBusy, _, currentTotal := calcuateBusy(current)
+
+	if currentBusy <= prevBusy {
+		return 0
+	}
+
+	if currentTotal <= prevTotal {
+		return 100
+	}
+	return math.Min(100, math.Max(0, (currentBusy-prevBusy)/(currentTotal-prevTotal)*100))
+}
+
 // Run the actual check
 // if error != nil the check result will be nil
 // ctx can be canceled and runs the timeout
 // CheckResult will be serialized after the return and should not change until the next call to Run
 func (c *CheckCpu) Run(ctx context.Context) (interface{}, error) {
 	result := &resultCpu{}
 
-	cpuPercentages, err := cpu.PercentWithContext(ctx, 1*time.Second, true)
-
-	if err == nil {
-		result.PercentagePerCore = cpuPercentages
+	prevTimeStats, err := cpu.TimesWithContext(ctx, true)
+	if err != nil {
+		return nil, err
 	}
 
-	//get total CPU percentage
-	cpuPercentageTotal, err := cpu.PercentWithContext(ctx, 1*time.Second, false)
+	if err := c.SleepWithContext(ctx, 1*time.Second); err != nil {
+		return nil, err
+	}
 
-	if err == nil {
-		result.PercentageTotal = cpuPercentageTotal[0]
+	timeStats, err := cpu.TimesWithContext(ctx, true)
+	if err != nil {
+		return nil, err
 	}
 
-	if runtime.GOOS == "linux" {
-		timeStats, err := cpu.TimesWithContext(ctx, false)
-
-		if err == nil {
-			total := timeStats[0].User + timeStats[0].Nice + timeStats[0].System + timeStats[0].Idle + timeStats[0].Iowait + timeStats[0].Irq + timeStats[0].Softirq
-			result.DetailsTotal = &cpuDetails{
-				User:   timeStats[0].User / total * 100,
-				Nice:   timeStats[0].Nice / total * 100,
-				System: timeStats[0].System / total * 100,
-				Iowait: timeStats[0].Iowait / total * 100,
-			}
-		}
+	if len(prevTimeStats) != len(timeStats) {
+		return nil, fmt.Errorf("number of CPU cores has changed %v != %v", len(prevTimeStats), len(timeStats))
+	}
 
-		//get timings per CPU
-		var timings []cpuDetails
-		timeStats, err = cpu.TimesWithContext(ctx, true)
-		if err == nil {
-			for _, timeStat := range timeStats {
-				total := timeStat.User + timeStat.Nice + timeStat.System + timeStat.Idle + timeStat.Iowait + timeStat.Irq + timeStat.Softirq
-				timings = append(timings, cpuDetails{
-					User:   timeStat.User / total * 100,
-					Nice:   timeStat.Nice / total * 100,
-					System: timeStat.System / total * 100,
-					Idle:   timeStat.Idle / total * 100,
-					Iowait: timeStat.Iowait / total * 100,
-				})
-			}
-			result.DetailsPerCore = timings
-		}
+	// Get CPU usage per core
+	var cpuPercentages []float64
+	for i := range prevTimeStats {
+		cpuPercentage := calculateUsagePercentage(prevTimeStats[i], timeStats[i])
+		cpuPercentages = append(cpuPercentages, cpuPercentage)
 	}
+	result.PercentagePerCore = cpuPercentages
 
-	if runtime.GOOS == "darwin" {
-		timeStats, err := cpu.TimesWithContext(ctx, false)
-
-		if err == nil {
-			total := timeStats[0].User + timeStats[0].Nice + timeStats[0].System + timeStats[0].Idle
-			result.DetailsTotal = &cpuDetails{
-				User:   timeStats[0].User / total * 100,
-				Nice:   timeStats[0].Nice / total * 100,
-				System: timeStats[0].System / total * 100,
-				Idle:   timeStats[0].Idle / total * 100,
-			}
+	// Get total CPU usage as percentage
+	var totalCpuPercentage float64
+	for i := range cpuPercentages {
+		totalCpuPercentage = totalCpuPercentage + cpuPercentages[i]
+	}
+	result.PercentageTotal = totalCpuPercentage / float64(len(cpuPercentages))
+
+	// Get CPU timing details per core
+	var timings []cpuDetails
+	var totalUser, totalNice, totalSystem, totalIoWait, totalIdle, totalTotal float64
+	for i := range prevTimeStats {
+		// Ignore t.Irq + t.Softirq + t.Steal to get 100% over all values we list in the json
+		prevTotal := prevTimeStats[i].User + prevTimeStats[i].Nice + prevTimeStats[i].System + prevTimeStats[i].Idle + prevTimeStats[i].Iowait
+		currentTotal := timeStats[i].User + timeStats[i].Nice + timeStats[i].System + timeStats[i].Idle + timeStats[i].Iowait
+
+		userDelta := saturatingSub(timeStats[i].User, prevTimeStats[i].User)
+		niceDelta := saturatingSub(timeStats[i].Nice, prevTimeStats[i].Nice)
+		systemDelta := saturatingSub(timeStats[i].System, prevTimeStats[i].System)
+		ioWaitDelta := saturatingSub(timeStats[i].Iowait, prevTimeStats[i].Iowait)
+		idleDelta := saturatingSub(timeStats[i].Idle, prevTimeStats[i].Idle)
+		totalDelta := saturatingSub(currentTotal, prevTotal)
+
+		if totalDelta == 0 {
+			timings = append(timings, cpuDetails{})
+		} else {
+			// Do not Division by zero (not sure if this could ever happen to be honest but better safe than sorry)
+			timings = append(timings, cpuDetails{
+				User:   userDelta / totalDelta * 100,
+				Nice:   niceDelta / totalDelta * 100,
+				System: systemDelta / totalDelta * 100,
+				Idle:   idleDelta / totalDelta * 100,
+				Iowait: ioWaitDelta / totalDelta * 100,
+			})
 		}
 
-		//get timings per CPU
-		var timings []cpuDetails
-		timeStats, err = cpu.TimesWithContext(ctx, true)
-		if err == nil {
-			for _, timeStat := range timeStats {
-				total := timeStat.User + timeStat.Nice + timeStat.System + timeStat.Idle
-				timings = append(timings, cpuDetails{
-					User:   timeStat.User / total * 100,
-					Nice:   timeStat.Nice / total * 100,
-					System: timeStat.System / total * 100,
-					Idle:   timeStat.Idle / total * 100,
-				})
-			}
-			result.DetailsPerCore = timings
-		}
+		totalUser = totalUser + userDelta
+		totalNice = totalNice + niceDelta
+		totalSystem = totalSystem + systemDelta
+		totalIoWait = totalIoWait + ioWaitDelta
+		totalIdle = totalIdle + idleDelta
+		totalTotal = totalTotal + totalDelta
+	}
+	result.DetailsPerCore = timings
+
+	// Get total CPU timing details
+	result.DetailsTotal = &cpuDetails{
+		User:   totalUser / totalTotal * 100,
+		Nice:   totalNice / totalTotal * 100,
+		System: totalSystem / totalTotal * 100,
+		Idle:   totalIdle / totalTotal * 100,
+		Iowait: totalIoWait / totalTotal * 100,
 	}
 
 	return result, nil
 
 }
+
+// Like sleep but can be canceled by context
+func (c *CheckCpu) SleepWithContext(ctx context.Context, interval time.Duration) error {
+	var timer = time.NewTimer(interval)
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}

checks/cpu_windows.go

@@ -3,7 +3,7 @@ package checks
 import (
 	"context"
 
-	"github.com/StackExchange/wmi"
+	"github.com/yusufpapurcu/wmi"
 )
 
 // https://wutils.com/wmi/root/cimv2/win32_perfformatteddata_perfos_processor/