frontend: fix loading stdlib in direct_proxy mode

- internal/fetch: fix stdlib path extraction by removing invalid "std/" prefix.
- internal/fetchdatasource: fix "std" proxy errors in GetLatestInfo.
- cmd/frontend: prioritize stdlib getter and add local GOROOT optimization.

Reproduction:
- `go run ./cmd/frontend -dev -direct_proxy` should utilize the local GOROOT as the source for stdlib.
- `go run ./cmd/frontend -direct_proxy` should enable one to via a stdlib library at `localhost:8080/std` or any other stdlib pkg.

Fixes golang/go#77112

Change-Id: I0686922804178129550d9bb9edc87b2adff51aa6
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/746700
Auto-Submit: Ethan Lee <ethanalee@google.com>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
kokoro-CI: kokoro <noreply+kokoro@google.com>
Reviewed-by: Nicholas Husin <husin@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Nicholas Husin <nsh@golang.org>

4 files changed, 25 insertions, 6 deletions

diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go
index 7ed399fe..b438c8be 100644
--- a/cmd/frontend/main.go
+++ b/cmd/frontend/main.go
@@ -88,10 +88,18 @@ func main() {
 
 	if *directProxy {
 		sourceClient := source.NewClient(&http.Client{Transport: &ochttp.Transport{}, Timeout: 1 * time.Minute})
+		var stdlibGetter fetch.ModuleGetter = fetch.NewStdlibZipModuleGetter()
+		if *devMode {
+			if mg, err := fetch.NewGoPackagesStdlibModuleGetter(ctx, internal.GOROOT()); err == nil {
+				stdlibGetter = mg
+			} else {
+				log.Errorf(ctx, "loading packages from stdlib: %v", err)
+			}
+		}
 		ds := fetchdatasource.Options{
 			Getters: []fetch.ModuleGetter{
+				stdlibGetter,
 				fetch.NewProxyModuleGetter(proxyClient, sourceClient),
-				fetch.NewStdlibZipModuleGetter(),
 			},
 			ProxyClientForLatest: proxyClient,
 			BypassLicenseCheck:   *bypassLicenseCheck,
diff --git a/internal/fetch/getters.go b/internal/fetch/getters.go
index d6c00410..f95dada6 100644
--- a/internal/fetch/getters.go
+++ b/internal/fetch/getters.go
@@ -231,7 +231,6 @@ type goPackagesModuleGetter struct {
 	dir      string              // directory from which go/packages was run
 	packages []*packages.Package // all packages
 	modules  []*packages.Module  // modules references by packagages; sorted by path
-	isStd    bool
 }
 
 // NewGoPackagesModuleGetter returns a ModuleGetter that loads packages using
@@ -316,7 +315,6 @@ func NewGoPackagesStdlibModuleGetter(ctx context.Context, dir string) (*goPackag
 	}
 
 	return &goPackagesModuleGetter{
-		isStd:    true,
 		dir:      abs,
 		packages: pkgs,
 		modules:  modules,
diff --git a/internal/fetch/package.go b/internal/fetch/package.go
index 221f76ce..f8f5594e 100644
--- a/internal/fetch/package.go
+++ b/internal/fetch/package.go
@@ -22,6 +22,7 @@ import (
 	"golang.org/x/pkgsite/internal/licenses"
 	"golang.org/x/pkgsite/internal/log"
 	"golang.org/x/pkgsite/internal/source"
+	"golang.org/x/pkgsite/internal/stdlib"
 	"golang.org/x/pkgsite/internal/trace"
 	"golang.org/x/sync/errgroup"
 )
@@ -216,6 +217,9 @@ func extractPackageMetas(ctx context.Context, modulePath, resolvedVersion string
 			return nil
 		}
 		importPath := path.Join(modulePath, innerPath)
+		if modulePath == stdlib.ModulePath {
+			importPath = innerPath
+		}
 		if ignoredByGoTool(importPath) || isVendored(importPath) {
 			// File is in a directory we're not looking to process at this time, so skip it.
 			return nil
@@ -322,6 +326,9 @@ func extractPackageMetas(ctx context.Context, modulePath, resolvedVersion string
 					status = derrors.PackageBuildContextNotSupported
 				}
 				pkgPath = path.Join(modulePath, innerPath)
+				if modulePath == stdlib.ModulePath {
+					pkgPath = innerPath
+				}
 			} else {
 				mu.Lock()
 				pkgs = append(pkgs, pkg)
diff --git a/internal/fetchdatasource/fetchdatasource.go b/internal/fetchdatasource/fetchdatasource.go
index f36be0c1..6676d89f 100644
--- a/internal/fetchdatasource/fetchdatasource.go
+++ b/internal/fetchdatasource/fetchdatasource.go
@@ -23,6 +23,7 @@ import (
 	"golang.org/x/pkgsite/internal/log"
 	"golang.org/x/pkgsite/internal/lru"
 	"golang.org/x/pkgsite/internal/proxy"
+	"golang.org/x/pkgsite/internal/stdlib"
 	"golang.org/x/pkgsite/internal/version"
 )
 
@@ -285,9 +286,14 @@ func (ds *FetchDataSource) GetLatestInfo(ctx context.Context, unitPath, modulePa
 	latest.MinorVersion = latestUnitMeta.Version
 	latest.MinorModulePath = latestUnitMeta.ModulePath
 
-	latest.MajorModulePath, latest.MajorUnitPath, err = ds.getLatestMajorVersion(ctx, unitPath, modulePath)
-	if err != nil {
-		return latest, err
+	if modulePath == stdlib.ModulePath {
+		latest.MajorModulePath = latest.MinorModulePath
+		latest.MajorUnitPath = unitPath
+	} else {
+		latest.MajorModulePath, latest.MajorUnitPath, err = ds.getLatestMajorVersion(ctx, unitPath, modulePath)
+		if err != nil {
+			return latest, err
+		}
 	}
 	// Do not try to discover whether the unit is in the latest minor version; assume it is.
 	latest.UnitExistsAtMinor = true