mon: additional vm metrics

Author: pnsafonov

docs/prometheus.md

@@ -81,3 +81,13 @@ pind_vm_cpu{instance="micronode15:9091", job="pind", vm_name="ps-va-16-1-db"}
 `pind_vm_numa` - индекс NUMA ноды к которой привязана виртуалка. Если нагрузки нет, 
 то виртуалка остаётся в коммуналке и имеет значение -1. Если нагрузка есть, то
 принимает значение индекса NUMA ноды.
+
+`pind_vm_required_cores_phys` - требуемое число физических ядер процессора из пула,
+выделенного под нагрузку (pool_load).
+
+`pind_vm_required_cores` - требуемое число ядер (с учётом логических) процессора.
+
+`pind_vm_assigned_cores` - число фактически назначенных логических ядер процессора.
+
+`pind_vm_assigned_cores0` - процент назначенных (привязанных) ядер процессора.   
+Максимум 100% - всё назначилось, ядер хватило.

pkg/monitoring.go

@@ -45,11 +45,14 @@ func fillMonitoringState(ctx *Context) *mon_state.State {
 
 		numa0 := pinProc.GetNuma0()
 		proc := &mon_state.Proc{
-			VmName: pinProc.ProcInfo.VmName,
-			Time:   state.Time,
-			CPU:    pinProc.ProcInfo.cpu0,
-			Load:   pinProc.ProcInfo.load,
-			Numa0:  numa0,
+			VmName:            pinProc.ProcInfo.VmName,
+			Time:              state.Time,
+			CPU:               pinProc.ProcInfo.cpu0,
+			Load:              pinProc.ProcInfo.load,
+			Numa0:             numa0,
+			RequiredCoresPhys: pinProc.RequiredCoresPhys,
+			RequiredCores:     pinProc.RequiredCores,
+			AssignedCores:     pinProc.AssignedCores,
 		}
 
 		state.Procs[pinProc.ProcInfo.VmName] = proc

pkg/monitoring/collector/proc.go

@@ -10,9 +10,13 @@ type Proc struct {
 	VmName string
 	Proc   *mon_state.Proc
 
-	CPU   *prometheus.Desc
-	Load  *prometheus.Desc
-	Numa0 *prometheus.Desc
+	CPU               *prometheus.Desc
+	Load              *prometheus.Desc
+	Numa0             *prometheus.Desc
+	RequiredCoresPhys *prometheus.Desc
+	RequiredCores     *prometheus.Desc
+	AssignedCores     *prometheus.Desc
+	AssignedCores0    *prometheus.Desc
 }
 
 func NewProc(proc *mon_state.Proc) (*Proc, error) {
@@ -43,6 +47,23 @@ func NewProc(proc *mon_state.Proc) (*Proc, error) {
 		nil, labels0,
 	)
 
+	ent.RequiredCoresPhys = prometheus.NewDesc("pind_vm_required_cores_phys",
+		"vm required physical cpu cores",
+		nil, labels0,
+	)
+	ent.RequiredCores = prometheus.NewDesc("pind_vm_required_cores",
+		"vm required cpu cores",
+		nil, labels0,
+	)
+	ent.AssignedCores = prometheus.NewDesc("pind_vm_assigned_cores",
+		"vm assigned cpu cores",
+		nil, labels0,
+	)
+	ent.AssignedCores0 = prometheus.NewDesc("pind_vm_assigned_cores0",
+		"vm assigned cpu cores in percents, max 100%",
+		nil, labels0,
+	)
+
 	return ent, nil
 }
 
@@ -57,16 +78,33 @@ func (x *Proc) Describe(ch chan<- *prometheus.Desc) {
 }
 
 func (x *Proc) Collect(ch chan<- prometheus.Metric) {
+	assignedCores0 := float64(0)
+	if x.Proc.RequiredCores != 0 {
+		assignedCores0 = (float64(x.Proc.AssignedCores) / float64(x.Proc.RequiredCores)) * 100
+	}
+
 	m0 := prometheus.MustNewConstMetric(x.CPU, prometheus.GaugeValue, x.Proc.CPU)
 	m1 := prometheus.MustNewConstMetric(x.Load, prometheus.GaugeValue, x.Proc.GetLoad0())
 	m2 := prometheus.MustNewConstMetric(x.Numa0, prometheus.GaugeValue, float64(x.Proc.Numa0))
+	m3 := prometheus.MustNewConstMetric(x.RequiredCoresPhys, prometheus.GaugeValue, float64(x.Proc.RequiredCoresPhys))
+	m4 := prometheus.MustNewConstMetric(x.RequiredCores, prometheus.GaugeValue, float64(x.Proc.RequiredCores))
+	m5 := prometheus.MustNewConstMetric(x.AssignedCores, prometheus.GaugeValue, float64(x.Proc.AssignedCores))
+	m6 := prometheus.MustNewConstMetric(x.AssignedCores0, prometheus.GaugeValue, assignedCores0)
 
 	t0 := x.Proc.Time
 	mt0 := prometheus.NewMetricWithTimestamp(t0, m0)
 	mt1 := prometheus.NewMetricWithTimestamp(t0, m1)
 	mt2 := prometheus.NewMetricWithTimestamp(t0, m2)
+	mt3 := prometheus.NewMetricWithTimestamp(t0, m3)
+	mt4 := prometheus.NewMetricWithTimestamp(t0, m4)
+	mt5 := prometheus.NewMetricWithTimestamp(t0, m5)
+	mt6 := prometheus.NewMetricWithTimestamp(t0, m6)
 
 	ch <- mt0
 	ch <- mt1
 	ch <- mt2
+	ch <- mt3
+	ch <- mt4
+	ch <- mt5
+	ch <- mt6
 }

pkg/monitoring/mon_state/state.go

@@ -31,11 +31,14 @@ type PoolNode struct {
 }
 
 type Proc struct {
-	VmName string
-	Time   time.Time
-	CPU    float64
-	Load   bool
-	Numa0  int
+	VmName            string
+	Time              time.Time
+	CPU               float64
+	Load              bool
+	Numa0             int
+	RequiredCoresPhys int
+	RequiredCores     int
+	AssignedCores     int
 }
 
 func NewState() *State {

pkg/pin.go

@@ -41,6 +41,10 @@ type PinProc struct {
 	Threads     map[int]*PinThread // pid -> PinThread, add/remove/update
 	Node        *PoolNodeInfo      // process pinned to numa node
 	NotSelected PinCpus            // cores for not selected threads
+
+	RequiredCoresPhys int
+	RequiredCores     int
+	AssignedCores     int // actually pined cores count
 }
 
 type PinThread struct {
@@ -198,6 +202,12 @@ func getThreadByPID(threads []*ThreadInfo, pid int) (*ThreadInfo, bool) {
 	return nil, false
 }
 
+func (x *PinProc) resetAssignedCoresMetrics() {
+	//x.RequiredCoresPhys = 0
+	//x.RequiredCoresPhys = 0
+	x.AssignedCores = 0
+}
+
 func (x *PinProc) UpdateProc(proc *ProcInfo, state *PinState) {
 	// remove thread
 	for pid, thread := range x.Threads {
@@ -376,6 +386,7 @@ func (x *PinState) PinIdle() error {
 
 		if procInfo.Node != nil {
 			procInfo.Node = nil
+			procInfo.resetAssignedCoresMetrics()
 		}
 	}
 
@@ -417,6 +428,8 @@ func (x *PinState) PinLoad(ctx *Context) error {
 		if cpuCount <= 0 {
 			continue
 		}
+		procInfo.RequiredCoresPhys = cpuCountPhys
+		procInfo.RequiredCores = cpuCount
 
 		node := procInfo.Node
 		if node == nil {
@@ -447,6 +460,7 @@ func (x *PinState) PinLoad(ctx *Context) error {
 		}
 
 		assignedCount := node.assignCores(ctx, procInfo)
+		procInfo.AssignedCores = assignedCount
 		if assignedCount != cpuCount {
 			if ctx.Config.Service.Pool.PinMode == config.PinModeDelayed {
 				// evict delayed vms to idle cores