From 5cd78ab2f85b85d7a7e15c3977e216f03b94aa54 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Thu, 9 Jan 2025 07:14:01 -0500 Subject: [PATCH] devtools/cmd/seeddb: apply -keep_going to versions check Some module proxies may return invalid responses for versions for some modules. This change helps skip through those, when the majority of other modules would succeed. Concrete example: given a GitHub repo with no tagged releases, when querying the official Go module proxy for versions at /v@/list, it returns 200 OK and no content (since there are no versions). The jFrog Artifactory go module proxy's behaviour, in contrast, is to return 404 NOT FOUND if there are no versions. That means that if you're using seeddb and any of your seed.txt modules does not have tagged versions, we fail-fast when we try to list that module's versions, despite using -keep_going. This change applies -keep_going to that section of code, so that we skip over the 404 errors. Original author: Jean Barkhuysen Fixes #71140 Change-Id: I2ccbcc356c322deed81860ee92274fba04a079b2 Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/641675 LUCI-TryBot-Result: Go LUCI Reviewed-by: Michael Knyszek Reviewed-by: Jonathan Amsterdam kokoro-CI: kokoro Reviewed-by: Jean Barkhuysen --- devtools/cmd/seeddb/main.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/devtools/cmd/seeddb/main.go b/devtools/cmd/seeddb/main.go index 85109da7c..3c2c96b8b 100644 --- a/devtools/cmd/seeddb/main.go +++ b/devtools/cmd/seeddb/main.go @@ -89,12 +89,24 @@ func run(ctx context.Context, db *database.DB, proxyURL string) error { return err } + var ( + mu sync.Mutex + errors database.MultiErr + ) + // Expand versions and group by module path. + log.Printf("expanding versions") versionsByPath := map[string][]string{} for _, m := range seedModules { vers, err := versions(ctx, proxyClient, m) if err != nil { - return err + if *keepGoing { + mu.Lock() + errors = append(errors, err) + mu.Unlock() + } else { + return err + } } versionsByPath[m.Path] = append(versionsByPath[m.Path], vers...) } @@ -111,10 +123,7 @@ func run(ctx context.Context, db *database.DB, proxyURL string) error { f.DB = postgres.New(db) } - var ( - mu sync.Mutex - errors database.MultiErr - ) + log.Printf("fetching") for path, vers := range versionsByPath { path := path vers := vers @@ -135,9 +144,11 @@ func run(ctx context.Context, db *database.DB, proxyURL string) error { }) } if err := g.Wait(); err != nil { + log.Printf("Wait failed: %v", err) return err } if len(errors) > 0 { + log.Printf("there were errors") return errors } log.Printf("successfully fetched all modules in %s", time.Since(start).Round(time.Millisecond))