internal/fetch: add two stdlib getters

This breaks out the behavior of getting the stdlib into its own
getter, stdlibZipModuleGetter. Most (but not all) of the special cases
in the fetch code for the stdlib are in the stdlibZipModuleGetter,
which is installed on the frontend. For the local pkgsite command, we
add a new builder for the goPackagesModuleGetter that loads the
packages from the stdlib given a GOROOT. The stdlibZipModuleGetter is
also installed for pkgsite in as a fallback if the
goPackagesModuleGetter doesn't successfully fetch the stdlib module
(I'm not sure if that's possible).

Fixes #60114

Change-Id: Ida6a5367343643cc337c6d05e0d40095f79ca9e5
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/507883
Reviewed-by: Jamal Carvalho <jamal@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Michael Matloob <matloob@golang.org>
kokoro-CI: kokoro <noreply+kokoro@google.com>

5 files changed, 155 insertions, 17 deletions

diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go
index 7432e3ee..635d4293 100644
--- a/internal/fetch/fetch.go
+++ b/internal/fetch/fetch.go
@@ -75,7 +75,10 @@ func fetchModule(ctx context.Context, fr *FetchResult, mg ModuleGetter) error {
 	commitTime := info.Time
 
 	var contentDir fs.FS
-	if fr.ModulePath == stdlib.ModulePath {
+	switch mg.(type) {
+	case *stdlibZipModuleGetter:
+		// Special behavior for stdlibZipModuleGetter because its info doesn't actually
+		// give us the true resolved version.
 		var resolvedVersion string
 		contentDir, resolvedVersion, commitTime, err = stdlib.ContentDir(fr.RequestedVersion)
 		if err != nil {
@@ -84,7 +87,7 @@ func fetchModule(ctx context.Context, fr *FetchResult, mg ModuleGetter) error {
 		// If the requested version is a branch name like "master" or "main", we cannot
 		// determine the right resolved version until we start working with the repo.
 		fr.ResolvedVersion = resolvedVersion
-	} else {
+	default:
 		contentDir, err = mg.ContentDir(ctx, fr.ModulePath, fr.ResolvedVersion)
 		if err != nil {
 			return err
@@ -151,14 +154,6 @@ func fetchModule(ctx context.Context, fr *FetchResult, mg ModuleGetter) error {
 // the modulePath is "std", a request to @master will return an empty
 // commit time.
 func GetInfo(ctx context.Context, modulePath, requestedVersion string, mg ModuleGetter) (_ *proxy.VersionInfo, err error) {
-	if modulePath == stdlib.ModulePath {
-		var resolvedVersion string
-		resolvedVersion, err = stdlib.ZipInfo(requestedVersion)
-		if err != nil {
-			return nil, err
-		}
-		return &proxy.VersionInfo{Version: resolvedVersion}, nil
-	}
 	return mg.Info(ctx, modulePath, requestedVersion)
 }
 
@@ -166,9 +161,6 @@ func GetInfo(ctx context.Context, modulePath, requestedVersion string, mg Module
 // contents of the file obtained from the module getter. If modulePath is the
 // standard library, then the contents will be nil.
 func getGoModPath(ctx context.Context, modulePath, resolvedVersion string, mg ModuleGetter) (string, []byte, error) {
-	if modulePath == stdlib.ModulePath {
-		return stdlib.ModulePath, nil, nil
-	}
 	goModBytes, err := mg.Mod(ctx, modulePath, resolvedVersion)
 	if err != nil {
 		return "", nil, err
diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go
index 237d8973..1d35d7b7 100644
--- a/internal/fetch/fetch_test.go
+++ b/internal/fetch/fetch_test.go
@@ -67,6 +67,7 @@ func TestFetchModule(t *testing.T) {
 		mod          *testModule
 		fetchVersion string
 		proxyOnly    bool
+		stdzip       bool
 	}{
 		{name: "single", mod: moduleOnePackage},
 		{name: "wasm", mod: moduleWasm},
@@ -85,9 +86,9 @@ func TestFetchModule(t *testing.T) {
 		{name: "nonredistributable packages", mod: moduleNonRedist},
 		{name: "generics", mod: moduleGenerics},
 		// Proxy only as stdlib is not accounted for in local mode
-		{name: "stdlib module", mod: moduleStd, proxyOnly: true},
+		{name: "stdlib module", mod: moduleStd, stdzip: true},
 		// Proxy only as version is pre specified in local mode
-		{name: "master version of stdlib module", mod: moduleStdMaster, fetchVersion: "master", proxyOnly: true},
+		{name: "master version of stdlib module", mod: moduleStdMaster, fetchVersion: "master", stdzip: true},
 		// Proxy only as version is pre specified in local mode
 		{name: "master version of module", mod: moduleMaster, fetchVersion: "master", proxyOnly: true},
 		// Proxy only as version is pre specified in local mode
@@ -109,7 +110,12 @@ func TestFetchModule(t *testing.T) {
 			}{
 				{name: "proxy", fetch: proxyFetcher},
 				{name: "local", fetch: localFetcher},
+				{name: "stdlibzip", fetch: stdlibZipFetcher},
 			} {
+				if (!test.stdzip && fetcher.name == "stdlibzip") ||
+					(test.stdzip && fetcher.name != "stdlibzip") {
+					continue
+				}
 				if test.proxyOnly && fetcher.name == "local" {
 					continue
 				}
diff --git a/internal/fetch/fetchdata_test.go b/internal/fetch/fetchdata_test.go
index f3d5268b..fd304125 100644
--- a/internal/fetch/fetchdata_test.go
+++ b/internal/fetch/fetchdata_test.go
@@ -852,7 +852,7 @@ var moduleStdMaster = &testModule{
 				Version:           stdlib.TestMasterVersion,
 				CommitTime:        stdlib.TestCommitTime,
 				HasGoMod:          true,
-				SourceInfo:        source.NewStdlibInfo("master"),
+				SourceInfo:        source.NewStdlibInfoForTest("master"),
 				IsRedistributable: true,
 			},
 			Units: []*internal.Unit{
@@ -927,7 +927,7 @@ var moduleStd = &testModule{
 				Version:           "v1.12.5",
 				CommitTime:        stdlib.TestCommitTime,
 				HasGoMod:          true,
-				SourceInfo:        source.NewStdlibInfo("v1.12.5"),
+				SourceInfo:        source.NewStdlibInfoForTest("v1.12.5"),
 				IsRedistributable: true,
 			},
 			Units: []*internal.Unit{
diff --git a/internal/fetch/getters.go b/internal/fetch/getters.go
index f4b0eb46..ff2e146b 100644
--- a/internal/fetch/getters.go
+++ b/internal/fetch/getters.go
@@ -33,6 +33,7 @@ import (
 	"golang.org/x/pkgsite/internal/log"
 	"golang.org/x/pkgsite/internal/proxy"
 	"golang.org/x/pkgsite/internal/source"
+	"golang.org/x/pkgsite/internal/stdlib"
 	"golang.org/x/pkgsite/internal/version"
 	"golang.org/x/tools/go/packages"
 )
@@ -229,6 +230,7 @@ type goPackagesModuleGetter struct {
 	dir      string              // directory from which go/packages was run
 	packages []*packages.Package // all packages
 	modules  []*packages.Module  // modules references by packagages; sorted by path
+	isStd    bool
 }
 
 // NewGoPackagesModuleGetter returns a ModuleGetter that loads packages using
@@ -275,6 +277,49 @@ func NewGoPackagesModuleGetter(ctx context.Context, dir string, patterns ...stri
 	}, nil
 }
 
+// NewGoPackagesStdlibModuleGetter returns a ModuleGetter that loads stdlib packages using
+// go/packages.Load, from the requested GOROOT.
+func NewGoPackagesStdlibModuleGetter(ctx context.Context, dir string) (*goPackagesModuleGetter, error) {
+	abs, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, err
+	}
+	start := time.Now()
+	env := []string(nil)
+	if dir != "" {
+		env = append(os.Environ(), "GOROOT=", abs)
+	}
+	cfg := &packages.Config{
+		Context: ctx,
+		Dir:     abs,
+		Mode: packages.NeedName |
+			packages.NeedModule |
+			packages.NeedCompiledGoFiles |
+			packages.NeedFiles,
+		Env: env,
+	}
+	pkgs, err := packages.Load(cfg, "std")
+	log.Infof(ctx, "go/packages.Load(std) loaded %d packages from %s in %v", len(pkgs), dir, time.Since(start))
+	if err != nil {
+		return nil, err
+	}
+
+	stdmod := &packages.Module{Path: "std",
+		Dir: filepath.Join(abs, "src"),
+	}
+	modules := []*packages.Module{stdmod}
+	for _, p := range pkgs {
+		p.Module = stdmod
+	}
+
+	return &goPackagesModuleGetter{
+		isStd:    true,
+		dir:      abs,
+		packages: pkgs,
+		modules:  modules,
+	}, nil
+}
+
 // findModule searches known modules for a module matching the provided path.
 func (g *goPackagesModuleGetter) findModule(path string) (*packages.Module, error) {
 	i := sort.Search(len(g.modules), func(i int) bool {
@@ -501,6 +546,71 @@ func (g *goPackagesModuleGetter) HasChanged(ctx context.Context, info internal.M
 	return mtime == nil || mtime.After(info.CommitTime), nil
 }
 
+// A stdlibZipModuleGetter gets the modules for the stdlib by downloading a zip file.
+type stdlibZipModuleGetter struct {
+}
+
+// NewStdlibZipModuleGetter returns a ModuleGetter that loads stdlib packages using stdlib
+// zip files.
+func NewStdlibZipModuleGetter() *stdlibZipModuleGetter {
+	return &stdlibZipModuleGetter{}
+}
+
+// Info returns basic information about the module.
+func (g *stdlibZipModuleGetter) Info(ctx context.Context, path, vers string) (_ *proxy.VersionInfo, err error) {
+	// TODO(matloob) Do we need to call stdlib.ContentDir here and get the resolved version?
+	if path != "std" {
+		return nil, fmt.Errorf("%w: not module std", derrors.NotFound)
+	}
+	var resolvedVersion string
+	resolvedVersion, err = stdlib.ZipInfo(vers)
+	if err != nil {
+		return nil, err
+	}
+	return &proxy.VersionInfo{Version: resolvedVersion}, nil
+}
+
+// Mod returns the contents of the module's go.mod file.
+// We return dummy contents to that include the name expected by the fetcher.
+func (g *stdlibZipModuleGetter) Mod(ctx context.Context, path, version string) ([]byte, error) {
+	if path != "std" {
+		return nil, fmt.Errorf("%w: not module std", derrors.NotFound)
+	}
+	return []byte("module std\n"), nil
+}
+
+// ContentDir uses stdlib.ContentDir to return a fs.FS representing the standard library's
+// contents.
+func (g *stdlibZipModuleGetter) ContentDir(ctx context.Context, path, version string) (fs.FS, error) {
+	// Currently we don't actually use ContentDir and do special behavior for the stdlibZipModuleGetter.
+	// TODO(matloob): stdlib.ContentDir returns information that should be returned by Info.
+	// One alternative is to have Info call stdlib.ContentDir and save the results (like with a
+	// singleflight) But my guess is that Info is expected to be fast, so doing that would
+	// cause an unexpected slowdown.
+	if path != "std" {
+		return nil, fmt.Errorf("%w: not module std", derrors.NotFound)
+	}
+	fs, _, _, err := stdlib.ContentDir(version)
+	return fs, err
+}
+
+// SourceInfo returns a source.Info that will create /files links to modules in
+// the cache.
+func (g *stdlibZipModuleGetter) SourceInfo(ctx context.Context, path, version string) (*source.Info, error) {
+	if path != "std" {
+		return nil, fmt.Errorf("%w: not module std", derrors.NotFound)
+	}
+	return source.NewStdlibInfo(version)
+}
+
+func (g *stdlibZipModuleGetter) SourceFS() (string, fs.FS) {
+	return "", nil
+}
+
+func (g *stdlibZipModuleGetter) String() string {
+	return "stdlib"
+}
+
 // A modCacheModuleGetter gets modules from a directory in the filesystem that
 // is organized like the module cache, with a cache/download directory that has
 // paths that correspond to proxy URLs. An example of such a directory is $(go
diff --git a/internal/fetch/helper_test.go b/internal/fetch/helper_test.go
index 4768088a..72b91673 100644
--- a/internal/fetch/helper_test.go
+++ b/internal/fetch/helper_test.go
@@ -188,6 +188,36 @@ func localFetcher(t *testing.T, withLicenseDetector bool, ctx context.Context, m
 	return got, d
 }
 
+// stdlibZipFetcher is a test helper function that sets up a test proxy, fetches
+// a module using FetchModule, and returns a fetch result and a license detector.
+func stdlibZipFetcher(t *testing.T, withLicenseDetector bool, ctx context.Context, mod *proxytest.Module, fetchVersion string) (*FetchResult, *licenses.Detector) {
+	t.Helper()
+
+	modulePath := mod.ModulePath
+	version := mod.Version
+	if version == "" {
+		version = sample.VersionString
+	}
+	if fetchVersion == "" {
+		fetchVersion = version
+	}
+
+	got := FetchModule(ctx, modulePath, fetchVersion, NewStdlibZipModuleGetter())
+	if !withLicenseDetector {
+		return got, nil
+	}
+
+	fs, _, _, err := stdlib.ContentDir(fetchVersion)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	d := licenses.NewDetectorFS(modulePath, fetchVersion, fs, func(format string, args ...any) {
+		log.Infof(ctx, format, args...)
+	})
+	return got, d
+}
+
 func licenseDetector(ctx context.Context, t *testing.T, modulePath, version string, proxyClient *proxy.Client) *licenses.Detector {
 	t.Helper()
 	var (