internal/fetch: add two stdlib getters

This breaks out the behavior of getting the stdlib into its own
getter, stdlibZipModuleGetter. Most (but not all) of the special cases
in the fetch code for the stdlib are in the stdlibZipModuleGetter,
which is installed on the frontend. For the local pkgsite command, we
add a new builder for the goPackagesModuleGetter that loads the
packages from the stdlib given a GOROOT. The stdlibZipModuleGetter is
also installed for pkgsite in as a fallback if the
goPackagesModuleGetter doesn't successfully fetch the stdlib module
(I'm not sure if that's possible).

Fixes #60114

Change-Id: Ida6a5367343643cc337c6d05e0d40095f79ca9e5
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/507883
Reviewed-by: Jamal Carvalho <jamal@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Michael Matloob <matloob@golang.org>
kokoro-CI: kokoro <noreply+kokoro@google.com>

2 files changed, 49 insertions, 18 deletions

diff --git a/cmd/pkgsite/main.go b/cmd/pkgsite/main.go
index f56993c0..4b08645f 100644
--- a/cmd/pkgsite/main.go
+++ b/cmd/pkgsite/main.go
@@ -60,6 +60,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 	"time"
@@ -92,11 +93,12 @@ var (
 )
 
 type serverConfig struct {
-	paths         []string
-	gopathMode    bool
-	useCache      bool
-	cacheDir      string
-	useListedMods bool
+	paths          []string
+	gopathMode     bool
+	useCache       bool
+	cacheDir       string
+	useListedMods  bool
+	useLocalStdlib bool
 
 	proxy *proxy.Client // client, or nil; controlled by the -proxy flag
 }
@@ -220,6 +222,10 @@ func buildServer(ctx context.Context, serverCfg serverConfig) (*frontend.Server,
 		}
 	}
 
+	if serverCfg.useLocalStdlib {
+		cfg.useLocalStdlib = true
+	}
+
 	getters, err := buildGetters(ctx, cfg)
 	if err != nil {
 		return nil, err
@@ -328,10 +334,11 @@ func getGOPATHModuleDirs(ctx context.Context, modulePaths []string) (map[string]
 // getterConfig defines the set of getters for the server to use.
 // See buildGetters.
 type getterConfig struct {
-	all         bool                              // if set, request "all" instead of ["<modulePath>/..."]
-	dirs        map[string][]frontend.LocalModule // local modules to serve
-	modCacheDir string                            // path to module cache, or ""
-	proxy       *proxy.Client                     // proxy client, or nil
+	all            bool                              // if set, request "all" instead of ["<modulePath>/..."]
+	dirs           map[string][]frontend.LocalModule // local modules to serve
+	modCacheDir    string                            // path to module cache, or ""
+	proxy          *proxy.Client                     // proxy client, or nil
+	useLocalStdlib bool                              // use go/packages for the local stdlib
 }
 
 // buildGetters constructs module getters based on the given configuration.
@@ -377,6 +384,22 @@ func buildGetters(ctx context.Context, cfg getterConfig) ([]fetch.ModuleGetter,
 	if cfg.proxy != nil {
 		getters = append(getters, fetch.NewProxyModuleGetter(cfg.proxy, source.NewClient(time.Second)))
 	}
+
+	if cfg.useLocalStdlib {
+		goRepo := *goRepoPath
+		if goRepo == "" {
+			goRepo = runtime.GOROOT()
+		}
+		mg, err := fetch.NewGoPackagesStdlibModuleGetter(ctx, goRepo)
+		if err != nil {
+			log.Errorf(ctx, "loading packages from stdlib: %v", err)
+		} else {
+			getters = append(getters, mg)
+		}
+	}
+
+	getters = append(getters, fetch.NewStdlibZipModuleGetter())
+
 	return getters, nil
 }
 
diff --git a/cmd/pkgsite/main_test.go b/cmd/pkgsite/main_test.go
index b23f95f6..e17ac699 100644
--- a/cmd/pkgsite/main_test.go
+++ b/cmd/pkgsite/main_test.go
@@ -56,12 +56,13 @@ package a
 
 	cfg := func(modifyDefault func(*serverConfig)) serverConfig {
 		c := serverConfig{
-			paths:         []string{localModule},
-			gopathMode:    false,
-			useListedMods: true,
-			useCache:      true,
-			cacheDir:      cacheDir,
-			proxy:         prox,
+			paths:          []string{localModule},
+			gopathMode:     false,
+			useListedMods:  true,
+			useLocalStdlib: true,
+			useCache:       true,
+			cacheDir:       cacheDir,
+			proxy:          prox,
 		}
 		if modifyDefault != nil {
 			modifyDefault(&c)
@@ -131,7 +132,9 @@ package a
 		},
 		{
 			"search",
-			cfg(nil),
+			cfg(func(c *serverConfig) {
+				c.useLocalStdlib = false
+			}),
 			"search?q=a",
 			http.StatusOK,
 			in(".SearchResults",
@@ -140,7 +143,9 @@ package a
 		},
 		{
 			"no symbol search",
-			cfg(nil),
+			cfg(func(c *serverConfig) {
+				c.useLocalStdlib = false
+			}),
 			"search?q=A", // using a capital letter should not cause symbol search
 			http.StatusOK,
 			in(".SearchResults",
@@ -149,7 +154,9 @@ package a
 		},
 		{
 			"search not found",
-			cfg(nil),
+			cfg(func(c *serverConfig) {
+				c.useLocalStdlib = false
+			}),
 			"search?q=zzz",
 			http.StatusOK,
 			in(".SearchResults",
@@ -169,6 +176,7 @@ package a
 			"search unsupported",
 			cfg(func(c *serverConfig) {
 				c.paths = nil
+				c.useLocalStdlib = false
 			}),
 			"search?q=zzz",
 			http.StatusFailedDependency,