Vault integration for secret storage

.github/workflows/release.yaml

@@ -168,8 +168,8 @@ jobs:
       - name: Build and push image
         uses: docker/build-push-action@v5
         with:
-          file: ./docker/mock-proxy/Dockerfile
-          context: .
+          file: Dockerfile
+          context: ./docker/mock-proxy
           push: true
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}

Makefile

@@ -56,6 +56,9 @@ dev-docker-compose-up: ## Start Docker compose
 dev-docker-compose-down: ## Stop Docker compose
 	docker compose -f docker/docker-compose.yaml down
 
+.PHONY: dev-docker-compose-restart
+dev-docker-compose-restart: dev-docker-compose-down dev-docker-compose-up
+
 .PHONY: lt
 lt: lint test ## Run linters and tests (always do this!)
 

adapters/secrets/hashicorp_vault.go

@@ -0,0 +1,162 @@
+package secrets
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	vault "github.com/hashicorp/vault/api"
+	authkubernetes "github.com/hashicorp/vault/api/auth/kubernetes"
+)
+
+type hashicorpVaultService struct {
+	client     *vault.Client
+	secretPath string
+	mountPath  string
+	log        *slog.Logger
+}
+
+type VaultConfig struct {
+	Address      string // Vault server address (e.g., http://localhost:8200)
+	Token        string // Vault token for authentication (used when AuthMethod=="token")
+	SecretPrefix string // Path prefix for secrets (e.g., "secrets/builder-hub")
+	MountPath    string // Vault KV v2 mount path (e.g., "secret", defaults to "secret")
+	AuthMethod   string // "token" (default) or "kubernetes"
+	Role         string // Role name for Kubernetes auth (required if AuthMethod=="kubernetes")
+	Jwt          string // ServiceAccount JWT for Kubernetes auth (required if AuthMethod=="kubernetes")
+}
+
+func NewHashicorpVaultService(ctx context.Context, log *slog.Logger, cfg VaultConfig) (*hashicorpVaultService, error) {
+	if cfg.MountPath == "" {
+		cfg.MountPath = "secret"
+	}
+
+	if cfg.AuthMethod != "token" && cfg.AuthMethod != "kubernetes" && cfg.AuthMethod != "" {
+		return nil, fmt.Errorf("unsupported AuthMethod %s", cfg.AuthMethod)
+	}
+
+	vcfg := vault.DefaultConfig()
+	vcfg.Address = cfg.Address
+	client, err := vault.NewClient(vcfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Vault client: %w", err)
+	}
+
+	svc := &hashicorpVaultService{
+		client:     client,
+		secretPath: cfg.SecretPrefix,
+		mountPath:  cfg.MountPath,
+		log:        log,
+	}
+
+	if cfg.AuthMethod == "kubernetes" {
+		if cfg.Jwt == "" {
+			return nil, fmt.Errorf("JWT is required for Kubernetes auth")
+		}
+		k8sAuth, err := authkubernetes.NewKubernetesAuth(cfg.Role, authkubernetes.WithServiceAccountToken(cfg.Jwt))
+		if err != nil {
+			return nil, fmt.Errorf("failed to initialize Kubernetes auth: %w", err)
+		}
+		authInfo, err := client.Auth().Login(ctx, k8sAuth)
+		if err != nil {
+			return nil, fmt.Errorf("kubernetes auth failed: %w", err)
+		}
+		watcher, err := client.NewLifetimeWatcher(&vault.LifetimeWatcherInput{Secret: authInfo})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create token lifetime watcher: %w", err)
+		}
+		go svc.watchTokenRenewal(ctx, watcher)
+	} else {
+		if cfg.Token == "" {
+			return nil, errors.New("token is required for vault auth")
+		}
+		client.SetToken(cfg.Token)
+	}
+
+	// Verify connection by attempting a read (404 is fine — path may not exist yet)
+	verifyCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	_, verifyErr := client.KVv2(cfg.MountPath).Get(verifyCtx, cfg.SecretPrefix)
+	if verifyErr != nil && !isVault404(verifyErr) {
+		return nil, fmt.Errorf("failed to verify Vault connection: %w", verifyErr)
+	}
+
+	return svc, nil
+}
+
+func (s *hashicorpVaultService) watchTokenRenewal(ctx context.Context, watcher *vault.LifetimeWatcher) {
+	go watcher.Start()
+	defer watcher.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case err := <-watcher.DoneCh():
+			if err != nil {
+				s.log.Error("vault token renewal stopped", "err", err)
+			}
+			return
+		case <-watcher.RenewCh():
+			s.log.Debug("vault token renewed")
+		}
+	}
+}
+
+func isVault404(err error) bool {
+	var responseErr *vault.ResponseError
+	return errors.As(err, &responseErr) && responseErr.StatusCode == 404
+}
+
+func (s *hashicorpVaultService) secretKVPath(builderName string) string {
+	if s.secretPath == "" {
+		return builderName
+	}
+	return fmt.Sprintf("%s/%s", s.secretPath, builderName)
+}
+
+// GetSecretValues retrieves secrets for a specific builder from Vault KV v2.
+// Implements application.SecretAccessor interface.
+func (s *hashicorpVaultService) GetSecretValues(ctx context.Context, builderName string) (json.RawMessage, error) {
+	path := s.secretKVPath(builderName)
+
+	secret, err := s.client.KVv2(s.mountPath).Get(ctx, path)
+	if err != nil {
+		if isVault404(err) {
+			return json.RawMessage("{}"), nil
+		}
+		return nil, fmt.Errorf("failed to read secret from Vault: %w", err)
+	}
+
+	if secret == nil || secret.Data == nil {
+		return json.RawMessage("{}"), nil
+	}
+
+	secretJSON, err := json.Marshal(secret.Data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal Vault secret: %w", err)
+	}
+
+	return json.RawMessage(secretJSON), nil
+}
+
+// SetSecretValues stores secrets for a specific builder in Vault KV v2.
+// Implements ports.AdminSecretService interface.
+func (s *hashicorpVaultService) SetSecretValues(ctx context.Context, builderName string, values json.RawMessage) error {
+	path := s.secretKVPath(builderName)
+
+	var dataMap map[string]any
+	if err := json.Unmarshal(values, &dataMap); err != nil {
+		return fmt.Errorf("failed to unmarshal secret values: %w", err)
+	}
+
+	_, err := s.client.KVv2(s.mountPath).Put(ctx, path, dataMap)
+	if err != nil {
+		return fmt.Errorf("failed to write secret to Vault: %w", err)
+	}
+
+	return nil
+}

adapters/secrets/service.go

@@ -1,22 +1,24 @@
-// Package secrets contains logic for adapter to aws secrets manager
+// Package secrets implements secrets storage backends
 package secrets
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/secretsmanager"
+	"github.com/flashbots/builder-hub/application"
 )
 
-type Service struct {
+type awsSecretsService struct {
 	sm           *secretsmanager.SecretsManager
 	secretPrefix string
 }
 
-func NewService(secretPrefix string) (*Service, error) {
+func NewAWSSecretsManagerService(secretPrefix string) (*awsSecretsService, error) {
 	sess, err := session.NewSession(&aws.Config{
 		Region: aws.String("us-east-2"),
 	})
@@ -27,21 +29,19 @@ func NewService(secretPrefix string) (*Service, error) {
 	// Create a Secrets Manager client
 	svc := secretsmanager.New(sess)
 
-	return &Service{sm: svc, secretPrefix: secretPrefix}, nil
+	return &awsSecretsService{sm: svc, secretPrefix: secretPrefix}, nil
 }
 
-var ErrMissingSecret = errors.New("missing secret for builder")
-
-func (s *Service) secretName(builderName string) string {
+func (s *awsSecretsService) secretName(builderName string) string {
 	return s.secretPrefix + "/" + builderName
 }
 
-func (s *Service) GetSecretValues(builderName string) (json.RawMessage, error) {
+func (s *awsSecretsService) GetSecretValues(ctx context.Context, builderName string) (json.RawMessage, error) {
 	input := &secretsmanager.GetSecretValueInput{
 		SecretId: aws.String(s.secretName(builderName)),
 	}
 
-	result, err := s.sm.GetSecretValue(input)
+	result, err := s.sm.GetSecretValueWithContext(ctx, input)
 	if err != nil {
 		// If the secret doesn't exist, return empty JSON for new builders
 		var awsErr awserr.Error
@@ -58,19 +58,19 @@ func (s *Service) GetSecretValues(builderName string) (json.RawMessage, error) {
 
 	builderSecret, ok := secretData[builderName]
 	if !ok {
-		return nil, ErrMissingSecret
+		return nil, application.ErrMissingSecret
 	}
 
 	return builderSecret, nil
 }
 
-func (s *Service) SetSecretValues(builderName string, values json.RawMessage) error {
+func (s *awsSecretsService) SetSecretValues(ctx context.Context, builderName string, values json.RawMessage) error {
 	secretName := s.secretName(builderName)
 	input := &secretsmanager.GetSecretValueInput{
 		SecretId: aws.String(secretName),
 	}
 
-	result, err := s.sm.GetSecretValue(input)
+	result, err := s.sm.GetSecretValueWithContext(ctx, input)
 	var secretData map[string]json.RawMessage
 
 	if err != nil {
@@ -88,7 +88,7 @@ func (s *Service) SetSecretValues(builderName string, values json.RawMessage) er
 				Name:         aws.String(secretName),
 				SecretString: aws.String(string(newSecretString)),
 			}
-			_, createErr := s.sm.CreateSecret(createInput)
+			_, createErr := s.sm.CreateSecretWithContext(ctx, createInput)
 			return createErr
 		}
 		return err
@@ -111,7 +111,7 @@ func (s *Service) SetSecretValues(builderName string, values json.RawMessage) er
 		SecretId:     aws.String(secretName),
 		SecretString: aws.String(string(newSecretString)),
 	}
-	_, err = s.sm.PutSecretValue(sv)
+	_, err = s.sm.PutSecretValueWithContext(ctx, sv)
 	if err != nil {
 		return err
 	}

application/service.go

@@ -3,6 +3,7 @@ package application
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 
@@ -19,8 +20,10 @@ type BuilderDataAccessor interface {
 	LogEvent(ctx context.Context, eventName, builderName, name string) error
 }
 
+var ErrMissingSecret = errors.New("missing secret for builder")
+
 type SecretAccessor interface {
-	GetSecretValues(builderName string) (json.RawMessage, error)
+	GetSecretValues(ctx context.Context, builderName string) (json.RawMessage, error)
 }
 
 type BuilderHub struct {
@@ -53,7 +56,7 @@ func (b *BuilderHub) GetConfigWithSecrets(ctx context.Context, builderName strin
 	if err != nil {
 		return nil, fmt.Errorf("failing to fetch config for builder %s %w", builderName, err)
 	}
-	secr, err := b.secretAccessor.GetSecretValues(builderName)
+	secr, err := b.secretAccessor.GetSecretValues(ctx, builderName)
 	if err != nil {
 		return nil, fmt.Errorf("failing to fetch secrets for builder %s %w", builderName, err)
 	}