devtools/cmd/seeddb: apply -keep_going to versions check

Some module proxies may return invalid responses for versions for some modules.
This change helps skip through those, when the majority of other modules would
succeed.

Concrete example: given a GitHub repo with no tagged releases, when querying
the official Go module proxy for versions at /v@/list, it returns 200 OK and no
content (since there are no versions). The jFrog Artifactory go module proxy's
behaviour, in contrast, is to return 404 NOT FOUND if there are no versions.

That means that if you're using seeddb and any of your seed.txt modules does
not have tagged versions, we fail-fast when we try to list that module's
versions, despite using -keep_going. This change applies -keep_going to that
section of code, so that we skip over the 404 errors.

Original author: Jean Barkhuysen <[email protected]>

Fixes #71140

Change-Id: I2ccbcc356c322deed81860ee92274fba04a079b2
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/641675
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Michael Knyszek <[email protected]>
Reviewed-by: Jonathan Amsterdam <[email protected]>
kokoro-CI: kokoro <[email protected]>
Reviewed-by: Jean Barkhuysen <[email protected]>

devtools/cmd/seeddb/main.go

@@ -89,12 +89,24 @@ func run(ctx context.Context, db *database.DB, proxyURL string) error {
 		return err
 	}
 
+	var (
+		mu     sync.Mutex
+		errors database.MultiErr
+	)
+
 	// Expand versions and group by module path.
+	log.Printf("expanding versions")
 	versionsByPath := map[string][]string{}
 	for _, m := range seedModules {
 		vers, err := versions(ctx, proxyClient, m)
 		if err != nil {
-			return err
+			if *keepGoing {
+				mu.Lock()
+				errors = append(errors, err)
+				mu.Unlock()
+			} else {
+				return err
+			}
 		}
 		versionsByPath[m.Path] = append(versionsByPath[m.Path], vers...)
 	}
@@ -111,10 +123,7 @@ func run(ctx context.Context, db *database.DB, proxyURL string) error {
 		f.DB = postgres.New(db)
 	}
 
-	var (
-		mu     sync.Mutex
-		errors database.MultiErr
-	)
+	log.Printf("fetching")
 	for path, vers := range versionsByPath {
 		path := path
 		vers := vers
@@ -135,9 +144,11 @@ func run(ctx context.Context, db *database.DB, proxyURL string) error {
 		})
 	}
 	if err := g.Wait(); err != nil {
+		log.Printf("Wait failed: %v", err)
 		return err
 	}
 	if len(errors) > 0 {
+		log.Printf("there were errors")
 		return errors
 	}
 	log.Printf("successfully fetched all modules in %s", time.Since(start).Round(time.Millisecond))