Webhook management

docs/configuration_syntax.md

@@ -50,6 +50,23 @@ server:
     # environment variable)
     secret_token: 063f51ec-09a4-11eb-adc1-0242ac120002
 
+    add_webhooks: 
+      # Whether to add webhooks to projects from wildcards when
+      # exporter starts (optional, default: false)
+      on_init: false
+      # Whether to add webhooks to projects from wildcards
+      # on a regular basis (optional, default: false)
+      scheduled: false
+      # Interval in seconds to add webhooks to projects 
+      # from wildcards (optional, default: 43200)
+      interval_seconds: 43200
+
+    # GCPE Webhook endpoint URL 
+    webhook_url: https://gcpe.example.net/webhook
+
+    # Whether to remove webhooks when GCPE shutsdown
+    remove_webhooks: false
+
 # Redis configuration, optional and solely useful for an HA setup.
 # By default the data is held in memory of the exporter
 redis:

internal/cmd/run.go

@@ -98,6 +98,15 @@ func Run(cliCtx *cli.Context) (int, error) {
 
 	<-onShutdown
 	log.Info("received signal, attempting to gracefully exit..")
+
+	if c.Config.Server.Webhook.RemoveHooks == true {
+		if err := c.RemoveWebhooks(ctx); err != nil {
+			log.WithContext(ctx).WithError(err)
+		} else {
+			log.Info("Cleaning up webhooks")
+		}
+	}
+
 	ctxCancel()
 
 	httpServerContext, forceHTTPServerShutdown := context.WithTimeout(context.Background(), 5*time.Second)

internal/cmd/utils.go

@@ -70,6 +70,8 @@ func configure(ctx *cli.Context) (cfg config.Config, err error) {
 	log.WithFields(config.SchedulerConfig(cfg.Pull.RefsFromProjects).Log()).Info("pull refs from projects")
 	log.WithFields(config.SchedulerConfig(cfg.Pull.Metrics).Log()).Info("pull metrics")
 
+	log.WithFields(config.SchedulerConfig(cfg.Server.Webhook.AddWebhooks).Log()).Info("add webhooks")
+
 	log.WithFields(config.SchedulerConfig(cfg.GarbageCollect.Projects).Log()).Info("garbage collect projects")
 	log.WithFields(config.SchedulerConfig(cfg.GarbageCollect.Environments).Log()).Info("garbage collect environments")
 	log.WithFields(config.SchedulerConfig(cfg.GarbageCollect.Refs).Log()).Info("garbage collect refs")

pkg/config/config.go

@@ -90,6 +90,19 @@ type ServerWebhook struct {
 
 	// Secret token to authenticate legitimate webhook requests coming from the GitLab server
 	SecretToken string `validate:"required_if=Enabled true" yaml:"secret_token"`
+
+	// Schedule the addition of webhooks to all seleceted projects
+	AddWebhooks struct {
+		OnInit          bool `default:"false" yaml:"on_init"`
+		Scheduled       bool `default:"false" yaml:"scheduled"`
+		IntervalSeconds int  `default:"43200" validate:"gte=1" yaml:"interval_seconds"`
+	} `yaml:"add_webhooks"`
+
+	// Webhook URL
+	URL string `validate:"required_if=AddWebhooks.Scheduled true" yaml:"webhook_url"`
+
+	// Remove webhooks on shutdown
+	RemoveHooks bool `default:"false" yaml:"remove_webhooks"`
 }
 
 // Gitlab ..

pkg/config/config_test.go

@@ -72,6 +72,9 @@ func TestNew(t *testing.T) {
 	c.ProjectDefaults.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runners-manager-(\d*)\.gitlab\.com`
 	c.ProjectDefaults.Pull.Pipeline.Variables.Regexp = `.*`
 
+	c.Server.Webhook.AddWebhooks.IntervalSeconds = 43200
+	c.Server.Webhook.AddWebhooks.Scheduled = false
+
 	assert.Equal(t, c, New())
 }
 

pkg/controller/controller.go

@@ -62,7 +62,7 @@ func New(ctx context.Context, cfg config.Config, version string) (c Controller,
 	}
 
 	// Start the scheduler
-	c.Schedule(ctx, cfg.Pull, cfg.GarbageCollect)
+	c.Schedule(ctx, cfg.Pull, cfg.GarbageCollect, cfg.Server.Webhook)
 
 	return
 }
@@ -82,6 +82,7 @@ func (c *Controller) registerTasks() {
 		schemas.TaskTypePullRefMetrics:               c.TaskHandlerPullRefMetrics,
 		schemas.TaskTypePullRefsFromProject:          c.TaskHandlerPullRefsFromProject,
 		schemas.TaskTypePullRefsFromProjects:         c.TaskHandlerPullRefsFromProjects,
+		schemas.TaskTypeAddWebhooks:                  c.TaskHandlerAddWebhooks,
 	} {
 		_, _ = c.TaskController.TaskMap.Register(string(n), &taskq.TaskConfig{
 			Handler:    h,

pkg/controller/scheduler.go

@@ -302,8 +302,16 @@ func (c *Controller) TaskHandlerGarbageCollectMetrics(ctx context.Context) error
 	return c.GarbageCollectMetrics(ctx)
 }
 
+// TaskHandlerAddWebhooks ..
+func (c *Controller) TaskHandlerAddWebhooks(ctx context.Context) error {
+	defer c.unqueueTask(ctx, schemas.TaskTypeAddWebhooks, "_")
+	defer c.TaskController.monitorLastTaskScheduling(schemas.TaskTypeAddWebhooks)
+
+	return c.addWebhooks(ctx)
+}
+
 // Schedule ..
-func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.GarbageCollect) {
+func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.GarbageCollect, wh config.ServerWebhook) {
 	ctx, span := otel.Tracer(tracerName).Start(ctx, "controller:Schedule")
 	defer span.End()
 
@@ -316,6 +324,7 @@ func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.G
 		schemas.TaskTypeGarbageCollectEnvironments:   config.SchedulerConfig(gc.Environments),
 		schemas.TaskTypeGarbageCollectRefs:           config.SchedulerConfig(gc.Refs),
 		schemas.TaskTypeGarbageCollectMetrics:        config.SchedulerConfig(gc.Metrics),
+		schemas.TaskTypeAddWebhooks:                  config.SchedulerConfig(wh.AddWebhooks),
 	} {
 		if cfg.OnInit {
 			c.ScheduleTask(ctx, tt, "_")

pkg/controller/webhooks.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/openlyinc/pointy"
 	log "github.com/sirupsen/logrus"
 	goGitlab "github.com/xanzy/go-gitlab"
 
@@ -327,3 +328,95 @@ func isEnvMatchingWilcard(w config.Wildcard, env schemas.Environment) (matches b
 	// Then we check if the ref matches the project pull parameters
 	return isEnvMatchingProjectPullEnvironments(w.Pull.Environments, env)
 }
+
+// Add a webhook to every project matching the wildcards.
+func (c *Controller) addWebhooks(ctx context.Context) error {
+	for _, w := range c.Config.Wildcards {
+		projects, err := c.Gitlab.ListProjects(ctx, w)
+		if err != nil {
+			return err
+		}
+
+		if len(projects) == 0 { // if no wildcards read config.projects
+			for _, p := range c.Config.Projects {
+				sp := schemas.Project{Project: p}
+				projects = append(projects, sp)
+			}
+		}
+
+		for _, p := range projects {
+			hooks, err := c.Gitlab.GetProjectHooks(ctx, p.Name)
+			if err != nil {
+				return err
+			}
+
+			WURL := c.Config.Server.Webhook.URL
+			opts := goGitlab.AddProjectHookOptions{ // options for hook
+				PushEvents:            pointy.Bool(false),
+				PipelineEvents:        pointy.Bool(true),
+				DeploymentEvents:      pointy.Bool(true),
+				EnableSSLVerification: pointy.Bool(false),
+				URL:                   &WURL,
+				Token:                 &c.Config.Server.Webhook.SecretToken,
+			}
+
+			if len(hooks) == 0 { // if no hooks
+				_, err := c.Gitlab.AddProjectHook(ctx, p.Name, &opts)
+				if err != nil {
+					return err
+				}
+			} else {
+				exists := false
+				for _, h := range hooks {
+					if h.URL == WURL {
+						exists = true
+					}
+				}
+				if exists == false {
+					_, err := c.Gitlab.AddProjectHook(ctx, p.Name, &opts)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *Controller) RemoveWebhooks(ctx context.Context) error {
+	for _, w := range c.Config.Wildcards {
+		projects, err := c.Gitlab.ListProjects(ctx, w)
+		if err != nil {
+			return err
+		}
+
+		if len(projects) == 0 { // if no wildcards read config.projects
+			for _, p := range c.Config.Projects {
+				sp := schemas.Project{Project: p}
+				projects = append(projects, sp)
+			}
+		}
+
+		for _, p := range projects {
+			hooks, err := c.Gitlab.GetProjectHooks(ctx, p.Name)
+			if err != nil {
+				return err
+			}
+
+			WURL := c.Config.Server.Webhook.URL
+
+			for _, h := range hooks {
+				if h.URL == WURL {
+					err := c.Gitlab.RemoveProjectHook(ctx, p.Name, h.ID)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}

pkg/gitlab/webhooks.go

@@ -0,0 +1,86 @@
+package gitlab
+
+import (
+	"context"
+
+	log "github.com/sirupsen/logrus"
+	goGitlab "github.com/xanzy/go-gitlab"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+)
+
+// GetProjectHooks ..
+func (c *Client) GetProjectHooks(ctx context.Context, projectName string) (hooks []*goGitlab.ProjectHook, err error) {
+	ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetProjectHooks")
+	defer span.End()
+	span.SetAttributes(attribute.String("project_name", projectName))
+
+	log.WithField("project_name", projectName).Trace("listing project hooks")
+
+	c.rateLimit(ctx)
+
+	hooks, resp, err := c.Projects.ListProjectHooks(
+		projectName,
+		&goGitlab.ListProjectHooksOptions{},
+		goGitlab.WithContext(ctx),
+	)
+	if err != nil {
+		return
+	}
+
+	c.requestsRemaining(resp)
+
+	return hooks, nil
+}
+
+// AddProjectHook ..
+func (c *Client) AddProjectHook(ctx context.Context, projectName string, options *goGitlab.AddProjectHookOptions) (hook *goGitlab.ProjectHook, err error) {
+	ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:AddProjectHook")
+	defer span.End()
+	span.SetAttributes(attribute.String("project_name", projectName))
+
+	log.WithField("project_name", projectName).Trace("adding project hook")
+
+	c.rateLimit(ctx)
+
+	hook, resp, err := c.Projects.AddProjectHook(
+		projectName,
+		options,
+		goGitlab.WithContext(ctx),
+	)
+	if err != nil {
+		return
+	}
+
+	c.requestsRemaining(resp)
+
+	return hook, nil
+}
+
+// RemoveProjectHook ..
+func (c *Client) RemoveProjectHook(ctx context.Context, projectName string, hookID int) (err error) {
+	ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:RemoveProjectHook")
+	defer span.End()
+	span.SetAttributes(attribute.String("project_name", projectName))
+	span.SetAttributes(attribute.Int("hook_id", hookID))
+
+	log.WithFields(log.Fields{
+		"project_name": projectName,
+		"hook_id":      hookID,
+	}).Trace("removing project hook")
+
+	c.rateLimit(ctx)
+
+	resp, err := c.Projects.DeleteProjectHook(
+		projectName,
+		hookID,
+		goGitlab.WithContext(ctx),
+	)
+	if err != nil {
+		return
+	}
+
+	c.requestsRemaining(resp)
+
+	return nil
+}

pkg/gitlab/webhooks_test.go

@@ -0,0 +1,64 @@
+package gitlab
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/openlyinc/pointy"
+	"github.com/stretchr/testify/assert"
+	goGitlab "github.com/xanzy/go-gitlab"
+)
+
+func TestGetProjectHooks(t *testing.T) {
+	ctx, mux, server, c := getMockedClient()
+	defer server.Close()
+
+	mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/hooks"),
+		func(w http.ResponseWriter, r *http.Request) {
+			assert.Equal(t, "GET", r.Method)
+			expectedQueryParams := url.Values{}
+			assert.Equal(t, expectedQueryParams, r.URL.Query())
+			fmt.Fprint(w, `[{"id":1}]`)
+		})
+
+	hooks, err := c.GetProjectHooks(ctx, "foo")
+	fmt.Println(hooks)
+	assert.NoError(t, err)
+	assert.Len(t, hooks, 1)
+}
+
+func TestAddProjectHook(t *testing.T) {
+	ctx, mux, server, c := getMockedClient()
+	defer server.Close()
+
+	mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/hooks"),
+		func(w http.ResponseWriter, r *http.Request) {
+			assert.Equal(t, "POST", r.Method)
+			expectedQueryParams := url.Values{}
+			assert.Equal(t, expectedQueryParams, r.URL.Query())
+			fmt.Fprint(w, `{"id":1, "url":"www.example.com/webhook", "push_events":false, "pipeline_events": true,  "deployment_events": true,  "enable_ssl_verification": false}`)
+		})
+
+	hook, err := c.AddProjectHook(ctx, "foo", &goGitlab.AddProjectHookOptions{
+		PushEvents:            pointy.Bool(false),
+		PipelineEvents:        pointy.Bool(true),
+		DeploymentEvents:      pointy.Bool(true),
+		EnableSSLVerification: pointy.Bool(false), // add config for this later
+		URL:                   pointy.String("www.example.com/webhook"),
+		Token:                 pointy.String("token"),
+	})
+
+	h := goGitlab.ProjectHook{
+		URL:                   "www.example.com/webhook",
+		ID:                    1,
+		PushEvents:            false,
+		PipelineEvents:        true,
+		DeploymentEvents:      true,
+		EnableSSLVerification: false,
+	}
+
+	assert.NoError(t, err)
+	assert.Equal(t, &h, hook)
+}

pkg/schemas/tasks.go

@@ -42,6 +42,9 @@ const (
 
 	// TaskTypeGarbageCollectMetrics ..
 	TaskTypeGarbageCollectMetrics TaskType = "GarbageCollectMetrics"
+
+	// TaskTypeAddWebhooks ..
+	TaskTypeAddWebhooks TaskType = "AddWebhooks"
 )
 
 // Tasks can be used to keep track of tasks.