devtools/cmd/seeddb: support fetching all version of a module

The seeddb script now supports syntax for fetching all versions of a
module, by adding <module>@all to a seed file.

Change-Id: Ia609df5033ac5522d0eb1d17d024453bd3b0e1bc
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/333156
Trust: Julie Qiu <julie@golang.org>
Run-TryBot: Julie Qiu <julie@golang.org>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Jamal Carvalho <jamal@golang.org>

2 files changed, 49 insertions, 31 deletions

diff --git a/devtools/cmd/seeddb/main.go b/devtools/cmd/seeddb/main.go
index 203fca49..5c3d9555 100644
--- a/devtools/cmd/seeddb/main.go
+++ b/devtools/cmd/seeddb/main.go
@@ -28,6 +28,7 @@ import (
 	"golang.org/x/pkgsite/internal/postgres"
 	"golang.org/x/pkgsite/internal/proxy"
 	"golang.org/x/pkgsite/internal/source"
+	"golang.org/x/pkgsite/internal/stdlib"
 	"golang.org/x/pkgsite/internal/worker"
 	"golang.org/x/sync/errgroup"
 )
@@ -74,15 +75,6 @@ func run(ctx context.Context, db *postgres.DB, proxyURL string) error {
 	}
 
 	sourceClient := source.NewClient(config.SourceTimeout)
-	fetchFunc := func(ctx context.Context, modulePath, version string) (int, error) {
-		f := &worker.Fetcher{
-			ProxyClient:  proxyClient,
-			SourceClient: sourceClient,
-			DB:           db,
-		}
-		code, _, err := f.FetchAndUpdateState(ctx, modulePath, version, "")
-		return code, err
-	}
 
 	sf := fmt.Sprintf("devtools/cmd/seeddb/%s", *seedfile)
 	seedModules, err := readSeedFile(ctx, sf)
@@ -92,33 +84,36 @@ func run(ctx context.Context, db *postgres.DB, proxyURL string) error {
 
 	r := results{}
 	g := new(errgroup.Group)
+	f := &worker.Fetcher{
+		ProxyClient:  proxyClient,
+		SourceClient: sourceClient,
+		DB:           db,
+	}
 	for _, m := range seedModules {
 		m := m
-
-		g.Go(func() error {
-			log.Infof(ctx, "Fetch requested: %q %q", m.path, m.version)
-			start := time.Now()
-			defer func() {
-				// Log the duration of this fetch request.
-				r.add(m.path, m.version, start)
-			}()
-
-			fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
-			defer cancel()
-
-			code, err := fetchFunc(fetchCtx, m.path, m.version)
+		vers := []string{m.version}
+		if m.version == "all" {
+			if m.path == stdlib.ModulePath {
+				vers, err = stdlib.Versions()
+			} else {
+				vers, err = proxyClient.Versions(ctx, m.path)
+			}
 			if err != nil {
-				if code == http.StatusNotFound {
-					// We expect
-					// github.com/jackc/pgx/pgxpool@v3.6.2+incompatible
-					// to fail from seed.txt, so that it will redirect to
-					// github.com/jackc/pgx/v4/pgxpool in tests.
-					return nil
-				}
 				return err
 			}
-			return nil
-		})
+		}
+		for _, v := range vers {
+			v := v
+			g.Go(func() error {
+				// Log the duration of this fetch request.
+				start := time.Now()
+				defer func() {
+					r.add(m.path, v, start)
+				}()
+
+				return fetchFunc(ctx, f, m.path, v)
+			})
+		}
 	}
 	if err := g.Wait(); err != nil {
 		return err
@@ -137,6 +132,27 @@ func run(ctx context.Context, db *postgres.DB, proxyURL string) error {
 	return nil
 }
 
+func fetchFunc(ctx context.Context, f *worker.Fetcher, m, v string) (err error) {
+	defer derrors.Wrap(&err, "fetchFunc(ctx, f, %q, %q)", m, v)
+
+	log.Infof(ctx, "Fetch requested: %q %q", m, v)
+	fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+	defer cancel()
+
+	code, _, err := f.FetchAndUpdateState(fetchCtx, m, v, "")
+	if err != nil {
+		if code == http.StatusNotFound {
+			// We expect
+			// github.com/jackc/pgx/pgxpool@v3.6.2+incompatible
+			// to fail from seed.txt, so that it will redirect to
+			// github.com/jackc/pgx/v4/pgxpool in tests.
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+
 type results struct {
 	paths map[string]time.Duration
 	mu    sync.Mutex
diff --git a/devtools/cmd/seeddb/seed.txt b/devtools/cmd/seeddb/seed.txt
index 333850fe..592e1547 100644
--- a/devtools/cmd/seeddb/seed.txt
+++ b/devtools/cmd/seeddb/seed.txt
@@ -3,3 +3,5 @@
 # license that can be found in the LICENSE file.
 
 github.com/julieqiu/api-demo@main
+golang.org/x/tools@all
+std@all