Completed cmd service implementation - not tested

Author: codemental

README.md

@@ -1,3 +1,11 @@
 ### Clarum executor agent
 
-WIP
+### Lifecycle
+At this point we always start the agent, initiate mocks, run test actions and after all that we shut it down.
+Maybe in the future we will keep it alive with the mocks running to improve performance.
+
+Keeping it alive would require:
+- to manage the agent lifecycle
+- add config flag to keep the agent alive
+- introduce concept of TestRun
+- split config in agent startup configuration & TestRun configuration

api/cmd/clarum-cmd.proto

@@ -3,12 +3,22 @@ syntax = "proto3";
 option go_package = "github.com/go-clarum/agent/api/cmd";
 
 service CmdService {
-  rpc CreateCommandEndpoint(CommandEndpoint) returns (CommandCreateResponse) {}
+  rpc InitEndpoint(InitCmdEndpoint) returns (InitCmdEndpointResponse) {}
+  rpc ShutdownEndpoint(ShutdownCmdEndpoint) returns (ShutdownCmdEndpointResponse) {}
 }
 
-message CommandEndpoint {
+message InitCmdEndpoint {
   string name = 1;
+  int32 warmup_seconds = 2;
+  repeated string cmd_components = 3;
 }
-message CommandCreateResponse {
-  bool success = 1;
+message InitCmdEndpointResponse {
+  string error = 1;
+}
+
+message ShutdownCmdEndpoint {
+  string name = 1;
+}
+message ShutdownCmdEndpointResponse {
+  string error = 1;
 }

config/config.go

@@ -2,9 +2,8 @@ package config
 
 import (
 	"github.com/go-clarum/agent/files"
-	"github.com/go-clarum/agent/logging"
 	"gopkg.in/yaml.v3"
-	"log/slog"
+	"log"
 	"os"
 	"path"
 	"time"
@@ -25,12 +24,14 @@ type config struct {
 	}
 }
 
-func LoadConfig() {
+func init() {
+	initLogger := log.New(os.Stdout, "", log.LstdFlags|log.Lmicroseconds)
+
 	configFilePath := path.Join(*baseDir, *configFile)
 	conf, err := files.ReadYamlFileToStruct[config](configFilePath)
 	if err != nil {
 		dir, _ := os.Getwd()
-		logging.Infof("No config file found in [%s] - default values will be used instead", dir)
+		initLogger.Printf("No config file found in [%s] - default values will be used instead", dir)
 		conf = &config{}
 	}
 
@@ -39,7 +40,7 @@ func LoadConfig() {
 	c = conf
 
 	configYaml, _ := yaml.Marshal(conf)
-	logging.Infof("Using the following config:\n[\n%s]", configYaml)
+	initLogger.Printf("Using the following config:\n[\n%s]", configYaml)
 }
 
 func Version() string {
@@ -50,8 +51,8 @@ func BaseDir() string {
 	return *baseDir
 }
 
-func LoggingLevel() slog.Level {
-	return logging.ParseLevel(c.Logging.Level)
+func LoggingLevel() string {
+	return c.Logging.Level
 }
 
 func ActionTimeout() time.Duration {

files/files.go

@@ -2,29 +2,26 @@ package files
 
 import (
 	"errors"
-	"github.com/go-clarum/agent/logging"
+	"fmt"
 	"github.com/go-clarum/agent/validators/strings"
 	"gopkg.in/yaml.v3"
 	"os"
 )
 
 func ReadYamlFileToStruct[S any](filePath string) (*S, error) {
 	if strings.IsBlank(filePath) {
-		logging.Error("Unable to read file. File path is empty")
-		return nil, errors.New("file path is empty")
+		return nil, errors.New("unable to read file - file path is empty")
 	}
 
 	buf, err := os.ReadFile(filePath)
 	if err != nil {
-		logging.Errorf("Failed to %s", err.Error())
-		return nil, err
+		return nil, errors.New(fmt.Sprintf("unable to read file - %s", err))
 	}
 
 	out := new(S)
 
 	if err := yaml.Unmarshal(buf, out); err != nil {
-		logging.Errorf("Failed to unmarshal yaml file %s: %s", filePath, err.Error())
-		return nil, err
+		return nil, errors.New(fmt.Sprintf("failed to unmarshal yaml file %s: %s", filePath, err))
 	}
 
 	return out, err

logging/logging.go

@@ -2,74 +2,81 @@ package logging
 
 import (
 	"fmt"
+	"github.com/go-clarum/agent/config"
 	"log"
 	"log/slog"
 	"os"
 	"strings"
 )
 
-var internalLogger = log.New(os.Stdout, "", log.LstdFlags|log.Lmicroseconds)
-var defaultLogger = NewLogger(slog.LevelInfo, "")
+var activeLogLevel slog.Level
+var internalLogger *log.Logger
+var defaultLogger *Logger
+
+func init() {
+	activeLogLevel = parseLevel(config.LoggingLevel())
+	internalLogger = log.New(os.Stdout, "", log.LstdFlags|log.Lmicroseconds)
+	defaultLogger = NewLogger("")
+}
 
 type Logger struct {
-	level  slog.Level
-	prefix string
+	name string
 }
 
-func NewLogger(level slog.Level, prefix string) *Logger {
-	return &Logger{level, prefix}
+func NewLogger(name string) *Logger {
+	return &Logger{name}
 }
 
 func (logger *Logger) Info(message string) {
-	if logger.level <= slog.LevelInfo {
-		internalLogger.Println("INFO " + logger.prefix + message)
+	if activeLogLevel <= slog.LevelInfo {
+		internalLogger.Println("INFO " + logger.name + " " + message)
 	}
 }
 
 func (logger *Logger) Infof(format string, a ...any) {
-	if logger.level <= slog.LevelInfo {
-		internalLogger.Println("INFO " + logger.prefix + fmt.Sprintf(format, a...))
+	if activeLogLevel <= slog.LevelInfo {
+		internalLogger.Println("INFO " + logger.name + " " + fmt.Sprintf(format, a...))
 	}
 }
 
 func (logger *Logger) Debug(message string) {
-	if logger.level <= slog.LevelDebug {
-		internalLogger.Println("DEBUG " + logger.prefix + message)
+	if activeLogLevel <= slog.LevelDebug {
+		internalLogger.Println("DEBUG " + logger.name + " " + message)
 	}
 }
 
 func (logger *Logger) Debugf(format string, a ...any) {
-	if logger.level <= slog.LevelDebug {
-		internalLogger.Println("DEBUG " + logger.prefix + fmt.Sprintf(format, a...))
+	if activeLogLevel <= slog.LevelDebug {
+		internalLogger.Println("DEBUG " + logger.name + " " + fmt.Sprintf(format, a...))
 	}
 }
 
 func (logger *Logger) Warn(message string) {
-	if logger.level <= slog.LevelWarn {
-		internalLogger.Println("WARN " + logger.prefix + message)
+	if activeLogLevel <= slog.LevelWarn {
+		internalLogger.Println("WARN " + logger.name + " " + message)
 	}
 }
 
 func (logger *Logger) Warnf(format string, a ...any) {
-	if logger.level <= slog.LevelWarn {
-		internalLogger.Println("WARN " + logger.prefix + fmt.Sprintf(format, a...))
+	if activeLogLevel <= slog.LevelWarn {
+		internalLogger.Println("WARN " + logger.name + " " + fmt.Sprintf(format, a...))
 	}
 }
 
 func (logger *Logger) Error(message string) {
-	if logger.level <= slog.LevelError {
-		internalLogger.Println("ERROR " + logger.prefix + message)
+	if activeLogLevel <= slog.LevelError {
+		internalLogger.Println("ERROR " + logger.name + " " + message)
 	}
 }
 
 func (logger *Logger) Errorf(format string, a ...any) {
-	if logger.level <= slog.LevelError {
-		internalLogger.Println("ERROR " + logger.prefix + fmt.Sprintf(format, a...))
+	if activeLogLevel <= slog.LevelError {
+		internalLogger.Println("ERROR " + logger.name + " " + fmt.Sprintf(format, a...))
 	}
 }
 
-func (logger *Logger) Prefix() string {
-	return logger.prefix
+func (logger *Logger) Name() string {
+	return logger.name
 }
 
 // calls on the default logger
@@ -106,7 +113,7 @@ func Errorf(format string, a ...any) {
 	defaultLogger.Errorf(format, a...)
 }
 
-func ParseLevel(level string) slog.Level {
+func parseLevel(level string) slog.Level {
 	lcLevel := strings.ToLower(level)
 	var result slog.Level
 

main.go

@@ -16,7 +16,6 @@ func main() {
 	logging.Infof("Starting clarum agent v%s", config.Version())
 	control.ShutdownHook.Add(1)
 
-	config.LoadConfig()
 	initAndRunGrpcServer()
 
 	control.ShutdownHook.Wait()

services/cmd/cmd.go

@@ -2,20 +2,34 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	. "github.com/go-clarum/agent/api/cmd"
 	"github.com/go-clarum/agent/logging"
+	"github.com/go-clarum/agent/services/cmd/internal/service"
 	"google.golang.org/grpc"
 )
 
-type service struct {
+type cmdService struct {
 	UnimplementedCmdServiceServer
 }
 
 func RegisterCmdService(server *grpc.Server) {
 	logging.Infof("Registering CommandService")
-	RegisterCmdServiceServer(server, &service{})
+	RegisterCmdServiceServer(server, &cmdService{})
 }
 
-func (s *service) CreateCommandEndpoint(ctx context.Context, ce *CommandEndpoint) (*CommandCreateResponse, error) {
-	return &CommandCreateResponse{}, nil
+func (s *cmdService) InitEndpoint(ctx context.Context, req *InitCmdEndpoint) (*InitCmdEndpointResponse, error) {
+	err := service.InitializeEndpoint(req.Name, req.CmdComponents, req.WarmupSeconds)
+
+	return &InitCmdEndpointResponse{
+		Error: fmt.Sprintf("%s", err),
+	}, nil
+}
+
+func (s *cmdService) ShutdownEndpoint(ctx context.Context, req *ShutdownCmdEndpoint) (*ShutdownCmdEndpointResponse, error) {
+	err := service.ShutdownEndpoint(req.Name)
+
+	return &ShutdownCmdEndpointResponse{
+		Error: fmt.Sprintf("%s", err),
+	}, nil
 }

services/cmd/internal/service/endpoint.go

@@ -0,0 +1,111 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/go-clarum/agent/control"
+	"github.com/go-clarum/agent/durations"
+	"github.com/go-clarum/agent/logging"
+	clarumstrings "github.com/go-clarum/agent/validators/strings"
+	"os/exec"
+	"time"
+)
+
+type endpoint struct {
+	name          string
+	cmdComponents []string
+	warmup        time.Duration
+	cmd           *exec.Cmd
+	cmdCancel     context.CancelFunc
+	logger        *logging.Logger
+}
+
+func newCommandEndpoint(name string, cmdComponents []string, warmupSeconds int32) (*endpoint, error) {
+	if clarumstrings.IsBlank(name) {
+		return nil, errors.New("cannot create command endpoint - name is empty")
+	}
+
+	if len(cmdComponents) == 0 || clarumstrings.IsBlank(cmdComponents[0]) {
+		return nil, errors.New(fmt.Sprintf("cannot create command endpoint [%s] - cmd is empty", name))
+	}
+
+	warmupDuration := time.Duration(warmupSeconds)
+
+	return &endpoint{
+		name:          name,
+		cmdComponents: cmdComponents,
+		warmup:        durations.GetDurationWithDefault(warmupDuration, 1*time.Second),
+		logger:        logging.NewLogger(loggerName(name)),
+	}, nil
+}
+
+// Start the process from the given command & arguments.
+// The process will be started into a cancelable context so that we can
+// cancel it later in the post-integration test phase.
+func (endpoint *endpoint) start() error {
+	endpoint.logger.Infof("running cmd [%s]", endpoint.cmdComponents)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	endpoint.cmd = exec.CommandContext(ctx, endpoint.cmdComponents[0], endpoint.cmdComponents[1:]...)
+	endpoint.cmdCancel = cancel
+
+	endpoint.logger.Debug("starting command")
+	if err := endpoint.cmd.Start(); err != nil {
+		return err
+	} else {
+		endpoint.logger.Debug("cmd start successful")
+	}
+
+	time.Sleep(endpoint.warmup)
+	endpoint.logger.Debug("warmup ended")
+
+	return nil
+}
+
+// shutdown the running process. Since the process was created with a context, we will attempt to
+// call ctx.Cancel(). If it returns an error, the process will be killed just in case.
+// We also wait for the action here, so that the post-integration test phase ends successfully.
+// TODO: check this code again, some parts are redundant
+func (endpoint *endpoint) shutdown() error {
+	control.RunningActions.Add(1)
+	defer control.RunningActions.Done()
+
+	endpoint.logger.Infof("stopping cmd [%s]", endpoint.cmdComponents)
+
+	if endpoint.cmdCancel != nil {
+		endpoint.logger.Debug("cancelling cmd")
+		endpoint.cmdCancel()
+
+		if _, err := endpoint.cmd.Process.Wait(); err != nil {
+			endpoint.logger.Errorf("cmd.Wait() returned error - [%s]", err)
+			endpoint.killProcess()
+			return err
+		} else {
+			endpoint.logger.Debug("context cancel finished successfully")
+		}
+	} else {
+		if err := endpoint.cmd.Process.Release(); err != nil {
+			endpoint.logger.Errorf("cmd.Release() returned error - [%s]", err)
+			endpoint.killProcess()
+			return err
+		} else {
+			endpoint.logger.Debug("cmd kill successful")
+		}
+	}
+
+	return nil
+}
+
+func (endpoint *endpoint) killProcess() {
+	endpoint.logger.Info("killing process")
+
+	if err := endpoint.cmd.Process.Kill(); err != nil {
+		endpoint.logger.Errorf("cmd.Kill() returned error - [%s]", err)
+		return
+	}
+}
+
+func loggerName(cmdName string) string {
+	return fmt.Sprintf("Command %s", cmdName)
+}