Add paging

cmd/root.go

@@ -39,6 +39,8 @@ const (
 	softExitFlag       = "soft-exit"
 	orgFlag            = "organization"
 	excludeReposFlag   = "exclude-repos"
+	pageCountFlag      = "page-count"
+	pageIndexFlag      = "page-index"
 )
 
 // ENV keys.
@@ -133,6 +135,8 @@ func init() {
 		orgFlagUsage                 = "used when using organization github app"
 		excludeReposFlagUsage        = "used to exclude repos from gathering on org mode"
 		softExitUsage                = "used on cleanup to exit soft without crashing"
+		pageCountFlagUsage           = "used with pagination per org to specify slice of pages"
+		pageIndexFlagUsage           = "used with pagination per org to specify index of how many slices"
 	)
 
 	const classifierUsageTemplate = "classifier to use when uploading to Dependency Track. Valid values are: %s"
@@ -152,6 +156,9 @@ func init() {
 
 	rootCmd.PersistentFlags().StringP(orgFlag, "g", "", orgFlagUsage)
 	rootCmd.PersistentFlags().StringSliceP(excludeReposFlag, "x", nil, excludeReposFlagUsage)
+
+	rootCmd.PersistentFlags().IntP(pageCountFlag, "r", 0, pageCountFlagUsage)
+	rootCmd.PersistentFlags().IntP(pageIndexFlag, "y", 0, pageIndexFlagUsage)
 }
 
 func initConfig() {

cmd/util.go

@@ -65,6 +65,16 @@ func createAppFromCLI(cmd *cobra.Command, verbose bool) (*app.App, error) {
 		options = append(options, app.WithDependencyTrack(baseURL, apiToken, classifier))
 	}
 
+	pagesCount, err := cmd.Flags().GetInt(pageCountFlag)
+	if err != nil {
+		return nil, fmt.Errorf(errTemplate, pageCountFlag)
+	}
+	pagesIndex, err := cmd.Flags().GetInt(pageIndexFlag)
+	if err != nil {
+		return nil, fmt.Errorf(errTemplate, pageIndexFlag)
+	}
+	options = append(options, app.WithPageSlicing(pagesCount, pagesIndex))
+
 	tags, err := cmd.Flags().GetStringSlice(tagsFlag)
 	if err != nil {
 		return nil, fmt.Errorf(errTemplate, tagsFlag)

internal/app/app.go

@@ -31,6 +31,7 @@ type App struct {
 	githubUsername, githubAPIToken, organization string // TODO Move later on to a separate GitHub client
 	dependencyTrackClient                        *dtrack.DependencyTrackClient
 	purgeCache, softExit                         bool
+	pagesCount, pagesIndex                       int64
 }
 
 type SBOMsFromFilesystemConfig struct {
@@ -44,6 +45,7 @@ type options struct {
 	githubUsername, githubAPIToken, organization string // TODO Move later on to a separate GitHub client
 	dependencyTrackClient                        *dtrack.DependencyTrackClient
 	purgeCache, softExit                         bool
+	pageCount, pageIndex                         int64
 }
 
 type Option func(options *options) error
@@ -90,6 +92,14 @@ func WithGitHubCredentials(username, apiToken string) Option {
 	}
 }
 
+func WithPageSlicing(pageCount, pageIndex int) Option {
+	return func(options *options) error {
+		options.pageCount = int64(pageCount)
+		options.pageIndex = int64(pageIndex)
+		return nil
+	}
+}
+
 func WithCachePurge() Option {
 	return func(options *options) error {
 		options.purgeCache = true
@@ -148,6 +158,8 @@ func New(outputFile string, opts ...Option) (*App, error) {
 	app.purgeCache = options.purgeCache
 	app.softExit = options.softExit
 	app.dependencyTrackClient = options.dependencyTrackClient
+	app.pagesCount = options.pageCount
+	app.pagesIndex = options.pageIndex
 
 	app.organization = options.organization
 
@@ -235,7 +247,7 @@ func (a App) SBOMsFromOrganization(organizationURL string, delayAmount uint16) {
 	}
 
 	c := internal.NewGetRepositoriesConfig(ctx, organizationURL, a.githubUsername, a.githubAPIToken, a.organization)
-	err := internal.WalkRepositories(c, collectSBOMsFromRepositories)
+	err := internal.WalkRepositories(c, collectSBOMsFromRepositories, a.pagesCount, a.pagesIndex)
 
 	if err != nil && !errors.Is(err, context.Canceled) {
 		log.WithError(err).Fatal("Collection failed! Can't recover - exiting")

internal/requests.go

@@ -148,7 +148,7 @@ func GetRepositories(conf GetRepositoriesConfig) ([]repositoryMapping, error) {
 	return exponentialBackoff(getRepositories, conf.BackoffPolicy...)
 }
 
-func WalkRepositories(conf GetRepositoriesConfig, callback func(repositoryURLs []string, apiToken string)) error {
+func WalkRepositories(conf GetRepositoriesConfig, callback func(repositoryURLs []string, apiToken string), pageCount, pageIndex int64) error {
 	var repositories []repositoryMapping
 	var err error
 	regenCount := 0
@@ -159,8 +159,14 @@ func WalkRepositories(conf GetRepositoriesConfig, callback func(repositoryURLs [
 		return fmt.Errorf("can't walk repository with malformed URL - %s: %w", conf.URL, err)
 	}
 
-	page := 1
+	start := (pageIndex * pageCount) + 1
+	end := int(start + pageCount - 1)
+	page := int(start)
 	for {
+		if page > end && pageCount != 0 {
+			log.WithField("request github", endpoint.String()).Infof("returning due to page limit, page: %d", page)
+			return nil
+		}
 		query := endpoint.Query()
 		query.Set("page", strconv.Itoa(page))
 		endpoint.RawQuery = query.Encode()

internal/requests_test.go

@@ -145,7 +145,7 @@ func TestWalkRepositories(t *testing.T) {
 		reqConf := createGetRepositoriesConfig(goodResponseServer.URL)
 		err := WalkRepositories(reqConf, func(repos []string, apiToken string) {
 			collectedRepos = append(collectedRepos, repos...)
-		})
+		}, 0, 0)
 		require.NoError(t, err)
 		assert.Equal(t, 4, hitCounter)
 
@@ -159,7 +159,7 @@ func TestWalkRepositories(t *testing.T) {
 	})
 
 	t.Run("return an error when request config contains invalid URL", func(t *testing.T) {
-		err := WalkRepositories(createGetRepositoriesConfig("http://bad url.com"), nil)
+		err := WalkRepositories(createGetRepositoriesConfig("http://bad url.com"), nil, 0, 0)
 		var e url.InvalidHostError
 		assert.ErrorAs(t, err, &e)
 		assert.Contains(t, err.Error(), repositoryWalkFailed)