Add retry logic when susbcription fails

Author: mithun-pensando

redfish-exporter/.env

@@ -1,23 +1,20 @@
-UPDATED="2025-01-22"
+UPDATED="2024-09-24"
 DESCRIPTION="Redfish Event Listener/Exporter"
-LISTENER_IP="<Listener_IP>"
-LISTENER_PORT="<PORT>"
-METRICS_PORT="<MERTRICS_PORT>"
+LISTENER_IP="10.11.18.55"
+LISTENER_PORT="9003"
+METRICS_PORT="2112"
 USE_SSL="false"
 CERTFILE="path/to/certfile"
 KEYFILE="path/to/keyfile"
-SLURM_CONTROL_NODE="<SLURM_CONTROL_NODE_IP>"
-#List of '|' seperated reasons for avoiding drain action if there is a match
-SLURM_DRAIN_EXCLUDE_REASON_LIST="reason 1|reason 2"
+SLURM_CONTROL_NODE="10.235.34.47"
+SLURM_DRAIN_EXCLUDE_REASON_LIST="AMD|Pensando|RebootNeeded"
 SLURM_SCONTROL_PATH="/usr/bin/scontrol"
+TLS_TIMEOUT="15"
 
-# Match RAS events received based on severity and '|' seperated list of message fields and perform drain action with the DrainReasonPrefix set as the prefix in the reason 
-# Message can be left empty if it doesn't need to be matched against, in that case only severity is matched
-# only DrainNode action is supported for now
 TRIGGER_EVENTS="[\
-{\"Severity\":\"Critical\",\"Message\":\"message 1|This is a critical test event\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNeeded\"},\
-{\"Severity\":\"Info\",\"Message\":\"message 3\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNotNeeded\"},\
-{\"Severity\":\"Warning\",\"Message\":\"message 4|This is a test event message\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNotNeeded\"}
+{\"Severity\":\"Critical\",\"Message\":\"Image 'UBB_FPGA' is being verified at 'ERoT'|This is an e2e critical test event\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNeeded\"},\
+{\"Severity\":\"Info\",\"Message\":\"Image 'UBB_FPGA' is being verified at 'ERoT'\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNotNeeded\"},\
+{\"Severity\":\"Warning\",\"Message\":\"Image 'UBB_FPGA' is being verified at 'ERoT'|This is an e2e test event message\",\"Action\":\"DrainNode\", \"DrainReasonPrefix\":\"RebootNotNeeded\"}
 ]"
 
 # Subscription (v1.5+)
@@ -33,11 +30,11 @@ TRIGGER_EVENTS="[\
 
 # Deprecated <v1.5
 SUBSCRIPTION_PAYLOAD="{\
-\"Destination\":\"http://<Listener_IP:Port>\",\
+\"Destination\":\"http://10.11.18.55:9003\",\
 \"EventTypes\":[\"Alert\"],\
 \"Protocol\":\"Redfish\",\
 \"Context\":\"YourContextData\",\
-\"Oem\":{\"Supermicro\":{\"EnableSubscription\":true}}\
+\"Oem\":{\"Supermicro\": {\"EnableSubscription\": true}}\
 }"
 
 # Config for setting default labels in Prometheus counter metrics.
@@ -46,5 +43,10 @@ PROMETHEUS_CONFIG="{\
 }"
 
 REDFISH_SERVERS="[\
-{\"ip\":\"https://<BMC_IP>\",\"username\":\"<username>\",\"password\":\"<password>\",\"loginType\":\"Session\",\"slurmNode\":\"<nodename\"}
-]"
+{\"ip\":\"https://10.235.37.54\",\"username\":\"ADMIN\",\"password\":\"PHHCJZUHDV\",\"loginType\":\"Session\",\"slurmNode\":\"smc300x-ccs-aus-GPUFCE9\"},
+{\"ip\":\"https://10.235.37.48\",\"username\":\"ADMIN\",\"password\":\"PHHCJZUHDV\",\"loginType\":\"Session\",\"slurmNode\":\"smc300x-ccs-aus-GPUFCE9\"}
+
+]"
+
+REDFISH_SERVERS_COMMON_CONFIG="{\
+\"hostSuffix\":\"ipmi.cluster\",\"username\":\"<username>\",\"password\":\"<password>\"}"

redfish-exporter/config.go

@@ -19,12 +19,15 @@ package main
 import (
 	"crypto/tls"
 	"encoding/json"
+	"fmt"
 	"log"
+	"net"
 	"os"
 	"strconv"
 	"strings"
 
 	"github.com/joho/godotenv"
+	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -61,6 +64,7 @@ type Config struct {
 	context              *tls.Config
 	eventCount           int
 	dataBuffer           []byte
+	TlsTimeOut           string
 }
 
 type TriggerEvent struct {
@@ -74,7 +78,12 @@ type PrometheusConfig struct {
 	Severity []string `json:"Severity"`
 }
 
-func setupConfig() Config {
+type target struct {
+	Targets []string          `yaml:"targets"`
+	Labels  map[string]string `yaml:"labels"`
+}
+
+func setupConfig(targetFile string) Config {
 	// Load .env file
 	err := godotenv.Load()
 	if err != nil {
@@ -125,6 +134,7 @@ func setupConfig() Config {
 	AppConfig.SlurmUser = os.Getenv("SLURM_USER")
 	AppConfig.SlurmDrainExcludeStr = os.Getenv("SLURM_DRAIN_EXCLUDE_REASON_LIST")
 	AppConfig.SlurmScontrolPath = os.Getenv("SLURM_SCONTROL_PATH")
+	AppConfig.TlsTimeOut = os.Getenv("TLS_TIMEOUT")
 
 	subscriptionPayloadJSON := os.Getenv("SUBSCRIPTION_PAYLOAD")
 	if err := json.Unmarshal([]byte(subscriptionPayloadJSON), &AppConfig.SubscriptionPayload); err != nil {
@@ -154,10 +164,63 @@ func setupConfig() Config {
 	redfishServersJSON := os.Getenv("REDFISH_SERVERS")
 	if redfishServersJSON == "" {
 		log.Println("REDFISH_SERVERS environment variable is not set or is empty")
+	} else {
+		if err := json.Unmarshal([]byte(redfishServersJSON), &AppConfig.RedfishServers); err != nil {
+			log.Fatalf("Failed to parse REDFISH_SERVERS: %v", err)
+		}
+	}
+
+	// Read and parse the REDFISH_SERVERS_COMMON_CONFIG environment variable
+	redfishServersCommonConfigJSON := os.Getenv("REDFISH_SERVERS_COMMON_CONFIG")
+	if redfishServersCommonConfigJSON == "" {
+		log.Println("redfishServersCommonConfigJSON environment variable is not set or is empty")
 		return AppConfig
 	}
-	if err := json.Unmarshal([]byte(redfishServersJSON), &AppConfig.RedfishServers); err != nil {
-		log.Fatalf("Failed to parse REDFISH_SERVERS: %v", err)
+	redfishServersCommonConfig := RedfishServersCommongConfig{}
+	if err := json.Unmarshal([]byte(redfishServersCommonConfigJSON), &redfishServersCommonConfig); err != nil {
+		log.Fatalf("Failed to parse REDFISH_SERVERS_COMMON_CONFIG: %v", err)
+	}
+
+	if targetFile == "" {
+		log.Println("No target file provided")
+		return AppConfig
+	}
+
+	targetYamlFile, err := os.ReadFile(targetFile)
+
+	if err != nil {
+		log.Fatalf("Failed to read file: %v", targetFile)
+	}
+
+	targets := []target{}
+
+	err = yaml.Unmarshal(targetYamlFile, &targets)
+
+	if err != nil {
+		log.Fatalf("Error parsing target file: %v | err: %v", targetFile, err)
+	}
+
+	for _, t := range targets {
+		log.Println("target: ", t.Targets)
+
+		for _, hostName := range t.Targets {
+			// add this target to Redfish servers
+			server := RedfishServer{}
+			bmcHost := fmt.Sprintf(hostName+".%v", redfishServersCommonConfig.HostSuffix)
+			ips, err := net.LookupIP(bmcHost)
+			if err != nil || len(ips) == 0 {
+				log.Printf("[error] Couldn't get the IP for host: %v | ips: %v | err: %v", bmcHost, ips, err)
+				continue
+			}
+			log.Println("IPs: ", ips)
+
+			server.IP = fmt.Sprintf("https://%v", ips[0])
+			server.LoginType = "Session"
+			server.Username = redfishServersCommonConfig.UserName
+			server.Password = redfishServersCommonConfig.Password
+			server.SlurmNode = hostName
+			AppConfig.RedfishServers = append(AppConfig.RedfishServers, server)
+		}
 	}
 
 	return AppConfig

redfish-exporter/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/nod-ai/ADA/redfish-exporter v0.0.0-20241002210630-2ef2d1070d90
 	github.com/prometheus/client_golang v1.20.4
 	github.com/stmcginnis/gofish v0.19.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (

redfish-exporter/go.sum

@@ -26,3 +26,6 @@ golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
 golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

redfish-exporter/main.go

@@ -26,6 +26,7 @@ import (
 	"os/signal"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -36,15 +37,24 @@ import (
 
 func main() {
 	var (
-		enableSlurm = flag.Bool("enable-slurm", false, "Enable slurm")
+		targetFile          string
+		enableSlurm         = flag.Bool("enable-slurm", false, "Enable slurm")
+		subscriptionMapLock sync.Mutex // to guard access to the map
 	)
+
+	flag.StringVar(&targetFile, "target", "", "Path to the target file for host/slurm node names")
 	flag.Parse()
 
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 	log.Println("Starting Redfish Event Listener/Exporter")
 
+	/*
+		if targetFile == "" {
+			log.Fatalf("Target file for host/slurm node names not set: Usage: ./amd-redfish-exporter --enable-slurm -target <filename>")
+		}
+	*/
 	// Setup configuration
-	AppConfig := setupConfig()
+	AppConfig := setupConfig(targetFile)
 
 	// Log the initialized config
 	log.Printf("Initialized Config: %+v", AppConfig)
@@ -57,8 +67,10 @@ func main() {
 		go slurmQueue.ProcessEventActionQueue()
 	}
 
+	subscriptionMap := make(map[string]string)
+
 	// Subscribe the listener to the event stream for all servers
-	subscriptionMap, err := CreateSubscriptionsForAllServers(AppConfig.RedfishServers, AppConfig.SubscriptionPayload)
+	err := CreateSubscriptionsForAllServers(AppConfig.RedfishServers, AppConfig.SubscriptionPayload, subscriptionMap, &subscriptionMapLock, AppConfig.TlsTimeOut)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -99,7 +111,9 @@ func main() {
 	time.Sleep(time.Second)
 
 	// Unsubscribe the listener from all servers
+	subscriptionMapLock.Lock()
 	DeleteSubscriptionsFromAllServers(AppConfig.RedfishServers, subscriptionMap)
+	subscriptionMapLock.Unlock()
 
 	cancel()