fix: use keyset pagination only when it is supported (#755)

* fix: use keyset pagination only when it is supported

#744
switched to keyset pagination for the `/api/v4/projects/:id/jobs` API
endpoint, but this feature is only available in GitLab 15.9 and later
(https://docs.gitlab.com/ee/api/jobs.html#list-project-jobs).

This commit retrieves the instance metadata at startup to
determine whether keyset pagination is available. If the metadata
is not available or returns a version < 15.9, offset pagination will
be used.

* chore: Use golang.org/x/mod/semver for version parsing

go.mod

@@ -93,6 +93,7 @@ require (
 	go.opentelemetry.io/otel/metric v1.21.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	golang.org/x/crypto v0.16.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.15.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect

go.sum

@@ -227,6 +227,8 @@ golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq
 golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No=
 golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=

pkg/controller/metadata.go

@@ -0,0 +1,24 @@
+package controller
+
+import (
+	"context"
+
+	goGitlab "github.com/xanzy/go-gitlab"
+
+	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/gitlab"
+)
+
+func (c *Controller) GetGitLabMetadata(ctx context.Context) error {
+	options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)}
+
+	metadata, _, err := c.Gitlab.Metadata.GetMetadata(options...)
+	if err != nil {
+		return err
+	}
+
+	if metadata.Version != "" {
+		c.Gitlab.UpdateVersion(gitlab.NewGitLabVersion(metadata.Version))
+	}
+
+	return nil
+}

pkg/controller/metadata_test.go

@@ -0,0 +1,61 @@
+package controller
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config"
+	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/gitlab"
+)
+
+func TestGetGitLabMetadataSuccess(t *testing.T) {
+	tests := []struct {
+		name            string
+		data            string
+		expectedVersion gitlab.GitLabVersion
+	}{
+		{
+			name: "successful parse",
+			data: `
+{
+"version":"16.7.0-pre",
+"revision":"3fe364fe754",
+"kas":{
+	"enabled":true,
+	"externalUrl":"wss://kas.gitlab.com",
+	"version":"v16.7.0-rc2"
+},
+"enterprise":true
+}
+`,
+			expectedVersion: gitlab.NewGitLabVersion("v16.7.0-pre"),
+		},
+		{
+			name: "unsuccessful parse",
+			data: `
+{
+"revision":"3fe364fe754"
+}
+`,
+			expectedVersion: gitlab.NewGitLabVersion(""),
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, c, mux, srv := newTestController(config.Config{})
+			defer srv.Close()
+
+			mux.HandleFunc("/api/v4/metadata",
+				func(w http.ResponseWriter, r *http.Request) {
+					fmt.Fprint(w, tc.data)
+				})
+
+			assert.NoError(t, c.GetGitLabMetadata(ctx))
+			assert.Equal(t, tc.expectedVersion, c.Gitlab.Version())
+		})
+	}
+}

pkg/controller/scheduler.go

@@ -307,6 +307,10 @@ func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.G
 	ctx, span := otel.Tracer(tracerName).Start(ctx, "controller:Schedule")
 	defer span.End()
 
+	go func() {
+		c.GetGitLabMetadata(ctx)
+	}()
+
 	for tt, cfg := range map[schemas.TaskType]config.SchedulerConfig{
 		schemas.TaskTypePullProjectsFromWildcards:    config.SchedulerConfig(pull.ProjectsFromWildcards),
 		schemas.TaskTypePullEnvironmentsFromProjects: config.SchedulerConfig(pull.EnvironmentsFromProjects),

pkg/gitlab/client.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -36,6 +37,9 @@ type Client struct {
 	RequestsCounter   atomic.Uint64
 	RequestsLimit     int
 	RequestsRemaining int
+
+	version GitLabVersion
+	mutex   sync.RWMutex
 }
 
 // ClientConfig ..
@@ -140,6 +144,19 @@ func (c *Client) rateLimit(ctx context.Context) {
 	c.RequestsCounter.Add(1)
 }
 
+func (c *Client) UpdateVersion(version GitLabVersion) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	c.version = version
+}
+
+func (c *Client) Version() GitLabVersion {
+	c.mutex.RLock()
+	defer c.mutex.RUnlock()
+
+	return c.version
+}
+
 func (c *Client) requestsRemaining(response *goGitlab.Response) {
 	if response == nil {
 		return

pkg/gitlab/jobs.go

@@ -231,13 +231,24 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
 	var (
 		foundJobs []*goGitlab.Job
 		resp      *goGitlab.Response
+		opt       *goGitlab.ListJobsOptions
 	)
 
-	opt := &goGitlab.ListJobsOptions{
-		ListOptions: goGitlab.ListOptions{
-			Pagination: "keyset",
-			PerPage:    100,
-		},
+	keysetPagination := c.Version().PipelineJobsKeysetPaginationSupported()
+	if keysetPagination {
+		opt = &goGitlab.ListJobsOptions{
+			ListOptions: goGitlab.ListOptions{
+				Pagination: "keyset",
+				PerPage:    100,
+			},
+		}
+	} else {
+		opt = &goGitlab.ListJobsOptions{
+			ListOptions: goGitlab.ListOptions{
+				Page:    1,
+				PerPage: 100,
+			},
+		}
 	}
 
 	options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)}
@@ -274,7 +285,8 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
 			}
 		}
 
-		if resp.NextLink == "" {
+		if keysetPagination && resp.NextLink == "" ||
+			(!keysetPagination && resp.CurrentPage >= resp.NextPage) {
 			var notFoundJobs []string
 
 			for k := range jobsToRefresh {
@@ -295,9 +307,11 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
 			break
 		}
 
-		options = []goGitlab.RequestOptionFunc{
-			goGitlab.WithContext(ctx),
-			goGitlab.WithKeysetPaginationParameters(resp.NextLink),
+		if keysetPagination {
+			options = []goGitlab.RequestOptionFunc{
+				goGitlab.WithContext(ctx),
+				goGitlab.WithKeysetPaginationParameters(resp.NextLink),
+			}
 		}
 	}
 

pkg/gitlab/jobs_test.go

@@ -132,64 +132,93 @@ func TestListPipelineBridges(t *testing.T) {
 }
 
 func TestListRefMostRecentJobs(t *testing.T) {
-	ctx, mux, server, c := getMockedClient()
-	defer server.Close()
-
-	ref := schemas.Ref{
-		Project: schemas.NewProject("foo"),
-		Name:    "yay",
+	tests := []struct {
+		name                string
+		keysetPagination    bool
+		expectedQueryParams url.Values
+	}{
+		{
+			name:             "offset pagination",
+			keysetPagination: false,
+			expectedQueryParams: url.Values{
+				"page":     []string{"1"},
+				"per_page": []string{"100"},
+			},
+		},
+		{
+			name:             "keyset pagination",
+			keysetPagination: true,
+			expectedQueryParams: url.Values{
+				"pagination": []string{"keyset"},
+				"per_page":   []string{"100"},
+			},
+		},
 	}
 
-	jobs, err := c.ListRefMostRecentJobs(ctx, ref)
-	assert.NoError(t, err)
-	assert.Len(t, jobs, 0)
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, mux, server, c := getMockedClient()
+			defer server.Close()
 
-	mux.HandleFunc("/api/v4/projects/foo/jobs",
-		func(w http.ResponseWriter, r *http.Request) {
-			assert.Equal(t, "GET", r.Method)
-			expectedQueryParams := url.Values{
-				"pagination": []string{"keyset"},
-				"per_page":   []string{"100"},
+			if tc.keysetPagination {
+				c.UpdateVersion(NewGitLabVersion("16.0.0"))
+			} else {
+				c.UpdateVersion(NewGitLabVersion("15.0.0"))
 			}
-			assert.Equal(t, expectedQueryParams, r.URL.Query())
-			fmt.Fprint(w, `[{"id":3,"name":"foo","ref":"yay"},{"id":4,"name":"bar","ref":"yay"}]`)
-		})
 
-	mux.HandleFunc(fmt.Sprintf("/api/v4/projects/bar/jobs"),
-		func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(http.StatusNotFound)
-		})
+			ref := schemas.Ref{
+				Project: schemas.NewProject("foo"),
+				Name:    "yay",
+			}
 
-	ref.LatestJobs = schemas.Jobs{
-		"foo": {
-			ID:   1,
-			Name: "foo",
-		},
-		"bar": {
-			ID:   2,
-			Name: "bar",
-		},
-	}
+			jobs, err := c.ListRefMostRecentJobs(ctx, ref)
+			assert.NoError(t, err)
+			assert.Len(t, jobs, 0)
+
+			mux.HandleFunc("/api/v4/projects/foo/jobs",
+				func(w http.ResponseWriter, r *http.Request) {
+					assert.Equal(t, "GET", r.Method)
+					assert.Equal(t, tc.expectedQueryParams, r.URL.Query())
+					fmt.Fprint(w, `[{"id":3,"name":"foo","ref":"yay"},{"id":4,"name":"bar","ref":"yay"}]`)
+				})
+
+			mux.HandleFunc(fmt.Sprintf("/api/v4/projects/bar/jobs"),
+				func(w http.ResponseWriter, r *http.Request) {
+					w.WriteHeader(http.StatusNotFound)
+				})
+
+			ref.LatestJobs = schemas.Jobs{
+				"foo": {
+					ID:   1,
+					Name: "foo",
+				},
+				"bar": {
+					ID:   2,
+					Name: "bar",
+				},
+			}
 
-	jobs, err = c.ListRefMostRecentJobs(ctx, ref)
-	assert.NoError(t, err)
-	assert.Len(t, jobs, 2)
-	assert.Equal(t, 3, jobs[0].ID)
-	assert.Equal(t, 4, jobs[1].ID)
+			jobs, err = c.ListRefMostRecentJobs(ctx, ref)
+			assert.NoError(t, err)
+			assert.Len(t, jobs, 2)
+			assert.Equal(t, 3, jobs[0].ID)
+			assert.Equal(t, 4, jobs[1].ID)
 
-	ref.LatestJobs["baz"] = schemas.Job{
-		ID:   5,
-		Name: "baz",
-	}
+			ref.LatestJobs["baz"] = schemas.Job{
+				ID:   5,
+				Name: "baz",
+			}
 
-	jobs, err = c.ListRefMostRecentJobs(ctx, ref)
-	assert.NoError(t, err)
-	assert.Len(t, jobs, 2)
-	assert.Equal(t, 3, jobs[0].ID)
-	assert.Equal(t, 4, jobs[1].ID)
+			jobs, err = c.ListRefMostRecentJobs(ctx, ref)
+			assert.NoError(t, err)
+			assert.Len(t, jobs, 2)
+			assert.Equal(t, 3, jobs[0].ID)
+			assert.Equal(t, 4, jobs[1].ID)
 
-	// Test invalid project id
-	ref.Project.Name = "bar"
-	_, err = c.ListRefMostRecentJobs(ctx, ref)
-	assert.Error(t, err)
+			// Test invalid project id
+			ref.Project.Name = "bar"
+			_, err = c.ListRefMostRecentJobs(ctx, ref)
+			assert.Error(t, err)
+		})
+	}
 }

pkg/gitlab/version.go

@@ -0,0 +1,32 @@
+package gitlab
+
+import (
+	"strings"
+
+	"golang.org/x/mod/semver"
+)
+
+type GitLabVersion struct {
+	Version string
+}
+
+func NewGitLabVersion(version string) GitLabVersion {
+	ver := ""
+	if strings.HasPrefix(version, "v") {
+		ver = version
+	} else if version != "" {
+		ver = "v" + version
+	}
+
+	return GitLabVersion{Version: ver}
+}
+
+// PipelineJobsKeysetPaginationSupported returns true if the GitLab instance
+// is running 15.9 or later.
+func (v GitLabVersion) PipelineJobsKeysetPaginationSupported() bool {
+	if v.Version == "" {
+		return false
+	}
+
+	return semver.Compare(v.Version, "v15.9.0") >= 0
+}