cmd: split the cmd/pkgsite server building logic to a new package

This will allow us to create a separate binary (probably called
cmd/pkgdoc) that will be started by "go doc" to serve pkgsite. The
separate binary can have different flags and options.

Change-Id: Ie8c2cdce4c850c9f787db12679c872f279c435ad
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/561340
kokoro-CI: kokoro <noreply+kokoro@google.com>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Robert Findley <rfindley@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>

2 files changed, 15 insertions, 547 deletions

diff --git a/cmd/pkgsite/main.go b/cmd/pkgsite/main.go
index f9eae158..50b69444 100644
--- a/cmd/pkgsite/main.go
+++ b/cmd/pkgsite/main.go
@@ -49,35 +49,21 @@
 package main
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"flag"
 	"fmt"
-	"io/fs"
 	"net"
 	"net/http"
 	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"sort"
 	"strings"
 	"time"
 
-	"github.com/google/safehtml/template"
-	"golang.org/x/pkgsite/internal"
+	"golang.org/x/pkgsite/cmd/internal/pkgsite"
 	"golang.org/x/pkgsite/internal/browser"
-	"golang.org/x/pkgsite/internal/fetch"
-	"golang.org/x/pkgsite/internal/fetchdatasource"
-	"golang.org/x/pkgsite/internal/frontend"
 	"golang.org/x/pkgsite/internal/log"
 	"golang.org/x/pkgsite/internal/middleware/timeout"
 	"golang.org/x/pkgsite/internal/proxy"
-	"golang.org/x/pkgsite/internal/source"
 	"golang.org/x/pkgsite/internal/stdlib"
-	"golang.org/x/pkgsite/static"
-	thirdparty "golang.org/x/pkgsite/third_party"
 )
 
 const defaultAddr = "localhost:8080" // default webserver address
@@ -86,31 +72,21 @@ var (
 	httpAddr   = flag.String("http", defaultAddr, "HTTP service address to listen for incoming requests on")
 	goRepoPath = flag.String("gorepo", "", "path to Go repo on local filesystem")
 	useProxy   = flag.Bool("proxy", false, "fetch from GOPROXY if not found locally")
-	devMode    = flag.Bool("dev", false, "enable developer mode (reload templates on each page load, serve non-minified JS/CSS, etc.)")
-	staticFlag = flag.String("static", "static", "path to folder containing static files served")
 	openFlag   = flag.Bool("open", false, "open a browser window to the server's address")
-	// other flags are bound to serverConfig below
+	// other flags are bound to ServerConfig below
 )
 
-type serverConfig struct {
-	paths          []string
-	gopathMode     bool
-	useCache       bool
-	cacheDir       string
-	useListedMods  bool
-	useLocalStdlib bool
-
-	proxy *proxy.Client // client, or nil; controlled by the -proxy flag
-}
-
 func main() {
-	var serverCfg serverConfig
+	var serverCfg pkgsite.ServerConfig
 
-	flag.BoolVar(&serverCfg.gopathMode, "gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src")
-	flag.BoolVar(&serverCfg.useCache, "cache", false, "fetch from the module cache")
-	flag.StringVar(&serverCfg.cacheDir, "cachedir", "", "module cache directory (defaults to `go env GOMODCACHE`)")
-	flag.BoolVar(&serverCfg.useListedMods, "list", true, "for each path, serve all modules in build list")
-	serverCfg.useLocalStdlib = true
+	flag.BoolVar(&serverCfg.GOPATHMode, "gopath_mode", false, "assume that local modules' Paths are relative to GOPATH/src")
+	flag.BoolVar(&serverCfg.UseCache, "cache", false, "fetch from the module cache")
+	flag.StringVar(&serverCfg.CacheDir, "cachedir", "", "module cache directory (defaults to `go env GOMODCACHE`)")
+	flag.BoolVar(&serverCfg.UseListedMods, "list", true, "for each path, serve all modules in build list")
+	flag.BoolVar(&serverCfg.DevMode, "dev", false, "enable developer mode (reload templates on each page load, serve non-minified JS/CSS, etc.)")
+	flag.StringVar(&serverCfg.DevModeStaticDir, "static", "static", "path to folder containing static files served")
+	serverCfg.UseLocalStdlib = true
+	serverCfg.GoRepoPath = *goRepoPath
 
 	flag.Usage = func() {
 		out := flag.CommandLine.Output()
@@ -120,9 +96,9 @@ func main() {
 		flag.PrintDefaults()
 	}
 	flag.Parse()
-	serverCfg.paths = collectPaths(flag.Args())
+	serverCfg.Paths = collectPaths(flag.Args())
 
-	if serverCfg.useCache || *useProxy {
+	if serverCfg.UseCache || *useProxy {
 		fmt.Fprintf(os.Stderr, "BYPASSING LICENSE CHECKING: MAY DISPLAY NON-REDISTRIBUTABLE INFORMATION\n")
 	}
 
@@ -132,7 +108,7 @@ func main() {
 			die("GOPROXY environment variable is not set")
 		}
 		var err error
-		serverCfg.proxy, err = proxy.New(url, nil)
+		serverCfg.Proxy, err = proxy.New(url, nil)
 		if err != nil {
 			die("connecting to proxy: %s", err)
 		}
@@ -143,7 +119,7 @@ func main() {
 	}
 
 	ctx := context.Background()
-	server, err := buildServer(ctx, serverCfg)
+	server, err := pkgsite.BuildServer(ctx, serverCfg)
 	if err != nil {
 		die(err.Error())
 	}
@@ -182,75 +158,6 @@ func die(format string, args ...any) {
 	os.Exit(1)
 }
 
-func buildServer(ctx context.Context, serverCfg serverConfig) (*frontend.Server, error) {
-	if len(serverCfg.paths) == 0 && !serverCfg.useCache && serverCfg.proxy == nil {
-		serverCfg.paths = []string{"."}
-	}
-
-	cfg := getterConfig{
-		all:   serverCfg.useListedMods,
-		proxy: serverCfg.proxy,
-	}
-
-	// By default, the requested paths are interpreted as directories. However,
-	// if -gopath_mode is set, they are interpreted as relative paths to modules
-	// in a GOPATH directory.
-	if serverCfg.gopathMode {
-		var err error
-		cfg.dirs, err = getGOPATHModuleDirs(ctx, serverCfg.paths)
-		if err != nil {
-			return nil, fmt.Errorf("searching GOPATH: %v", err)
-		}
-	} else {
-		var err error
-		cfg.dirs, err = getModuleDirs(ctx, serverCfg.paths)
-		if err != nil {
-			return nil, fmt.Errorf("searching GOPATH: %v", err)
-		}
-	}
-
-	if serverCfg.useCache {
-		cfg.modCacheDir = serverCfg.cacheDir
-		if cfg.modCacheDir == "" {
-			var err error
-			cfg.modCacheDir, err = defaultCacheDir()
-			if err != nil {
-				return nil, err
-			}
-			if cfg.modCacheDir == "" {
-				return nil, fmt.Errorf("empty value for GOMODCACHE")
-			}
-		}
-	}
-
-	if serverCfg.useLocalStdlib {
-		cfg.useLocalStdlib = true
-	}
-
-	getters, err := buildGetters(ctx, cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	// Collect unique module paths served by this server.
-	seenModules := make(map[frontend.LocalModule]bool)
-	var allModules []frontend.LocalModule
-	for _, modules := range cfg.dirs {
-		for _, m := range modules {
-			if seenModules[m] {
-				continue
-			}
-			seenModules[m] = true
-			allModules = append(allModules, m)
-		}
-	}
-	sort.Slice(allModules, func(i, j int) bool {
-		return allModules[i].ModulePath < allModules[j].ModulePath
-	})
-
-	return newServer(getters, allModules, cfg.proxy)
-}
-
 func collectPaths(args []string) []string {
 	var paths []string
 	for _, arg := range args {
@@ -258,222 +165,3 @@ func collectPaths(args []string) []string {
 	}
 	return paths
 }
-
-// getGOPATHModuleDirs returns the set of workspace modules for each directory,
-// determined by running go list -m.
-//
-// An error is returned if any operations failed unexpectedly, or if no
-// requested directories contain any valid modules.
-func getModuleDirs(ctx context.Context, dirs []string) (map[string][]frontend.LocalModule, error) {
-	dirModules := make(map[string][]frontend.LocalModule)
-	for _, dir := range dirs {
-		output, err := runGo(dir, "list", "-m", "-json")
-		if err != nil {
-			return nil, fmt.Errorf("listing modules in %s: %v", dir, err)
-		}
-		var modules []frontend.LocalModule
-		decoder := json.NewDecoder(bytes.NewBuffer(output))
-		for decoder.More() {
-			var m frontend.LocalModule
-			if err := decoder.Decode(&m); err != nil {
-				return nil, err
-			}
-			if m.ModulePath != "command-line-arguments" {
-				modules = append(modules, m)
-			}
-		}
-		if len(modules) > 0 {
-			dirModules[dir] = modules
-		}
-	}
-	if len(dirs) > 0 && len(dirModules) == 0 {
-		return nil, fmt.Errorf("no modules in any of the requested directories")
-	}
-	return dirModules, nil
-}
-
-// getGOPATHModuleDirs returns local module information for directories in
-// GOPATH corresponding to the requested module paths.
-//
-// An error is returned if any operations failed unexpectedly, or if no modules
-// were resolved. If individual module paths are not found, an error is logged
-// and the path skipped.
-func getGOPATHModuleDirs(ctx context.Context, modulePaths []string) (map[string][]frontend.LocalModule, error) {
-	gopath, err := runGo("", "env", "GOPATH")
-	if err != nil {
-		return nil, err
-	}
-	gopaths := filepath.SplitList(strings.TrimSpace(string(gopath)))
-
-	dirs := make(map[string][]frontend.LocalModule)
-	for _, path := range modulePaths {
-		dir := ""
-		for _, gopath := range gopaths {
-			candidate := filepath.Join(gopath, "src", path)
-			info, err := os.Stat(candidate)
-			if err == nil && info.IsDir() {
-				dir = candidate
-				break
-			}
-			if err != nil && !os.IsNotExist(err) {
-				return nil, err
-			}
-		}
-		if dir == "" {
-			log.Errorf(ctx, "ERROR: no GOPATH directory contains %q", path)
-		} else {
-			dirs[dir] = []frontend.LocalModule{{ModulePath: path, Dir: dir}}
-		}
-	}
-
-	if len(modulePaths) > 0 && len(dirs) == 0 {
-		return nil, fmt.Errorf("no GOPATH directories contain any of the requested module(s)")
-	}
-	return dirs, nil
-}
-
-// getterConfig defines the set of getters for the server to use.
-// See buildGetters.
-type getterConfig struct {
-	all            bool                              // if set, request "all" instead of ["<modulePath>/..."]
-	dirs           map[string][]frontend.LocalModule // local modules to serve
-	modCacheDir    string                            // path to module cache, or ""
-	proxy          *proxy.Client                     // proxy client, or nil
-	useLocalStdlib bool                              // use go/packages for the local stdlib
-}
-
-// buildGetters constructs module getters based on the given configuration.
-//
-// Getters are returned in the following priority order:
-//  1. local getters for cfg.dirs, in the given order
-//  2. a module cache getter, if cfg.modCacheDir != ""
-//  3. a proxy getter, if cfg.proxy != nil
-func buildGetters(ctx context.Context, cfg getterConfig) ([]fetch.ModuleGetter, error) {
-	var getters []fetch.ModuleGetter
-
-	// Load local getters for each directory.
-	for dir, modules := range cfg.dirs {
-		var patterns []string
-		if cfg.all {
-			patterns = append(patterns, "all")
-		} else {
-			for _, m := range modules {
-				patterns = append(patterns, fmt.Sprintf("%s/...", m))
-			}
-		}
-		mg, err := fetch.NewGoPackagesModuleGetter(ctx, dir, patterns...)
-		if err != nil {
-			log.Errorf(ctx, "Loading packages from %s: %v", dir, err)
-		} else {
-			getters = append(getters, mg)
-		}
-	}
-	if len(getters) == 0 && len(cfg.dirs) > 0 {
-		return nil, fmt.Errorf("failed to load any module(s) at %v", cfg.dirs)
-	}
-
-	// Add a getter for the local module cache.
-	if cfg.modCacheDir != "" {
-		g, err := fetch.NewModCacheGetter(cfg.modCacheDir)
-		if err != nil {
-			return nil, err
-		}
-		getters = append(getters, g)
-	}
-
-	if cfg.useLocalStdlib {
-		goRepo := *goRepoPath
-		if goRepo == "" {
-			goRepo = getGOROOT()
-		}
-		if goRepo != "" { // if goRepo == "" we didn't get a *goRepoPath and couldn't find GOROOT. Fall back to the zip files.
-			mg, err := fetch.NewGoPackagesStdlibModuleGetter(ctx, goRepo)
-			if err != nil {
-				log.Errorf(ctx, "loading packages from stdlib: %v", err)
-			} else {
-				getters = append(getters, mg)
-			}
-		}
-	}
-
-	// Add a proxy
-	if cfg.proxy != nil {
-		getters = append(getters, fetch.NewProxyModuleGetter(cfg.proxy, source.NewClient(&http.Client{Timeout: time.Second})))
-	}
-
-	getters = append(getters, fetch.NewStdlibZipModuleGetter())
-
-	return getters, nil
-}
-
-func getGOROOT() string {
-	if rg := runtime.GOROOT(); rg != "" {
-		return rg
-	}
-	b, err := exec.Command("go", "env", "GOROOT").Output()
-	if err != nil {
-		return ""
-	}
-	return strings.TrimSpace(string(b))
-}
-
-func newServer(getters []fetch.ModuleGetter, localModules []frontend.LocalModule, prox *proxy.Client) (*frontend.Server, error) {
-	lds := fetchdatasource.Options{
-		Getters:              getters,
-		ProxyClientForLatest: prox,
-		BypassLicenseCheck:   true,
-	}.New()
-
-	// In dev mode, use a dirFS to pick up template/JS/CSS changes without
-	// restarting the server.
-	var staticFS fs.FS
-	if *devMode {
-		staticFS = os.DirFS(*staticFlag)
-	} else {
-		staticFS = static.FS
-	}
-
-	// Preload local modules to warm the cache.
-	for _, lm := range localModules {
-		go lds.GetUnitMeta(context.Background(), "", lm.ModulePath, fetch.LocalVersion)
-	}
-	go lds.GetUnitMeta(context.Background(), "", "std", "latest")
-
-	server, err := frontend.NewServer(frontend.ServerConfig{
-		DataSourceGetter: func(context.Context) internal.DataSource { return lds },
-		TemplateFS:       template.TrustedFSFromEmbed(static.FS),
-		StaticFS:         staticFS,
-		DevMode:          *devMode,
-		LocalMode:        true,
-		LocalModules:     localModules,
-		ThirdPartyFS:     thirdparty.FS,
-	})
-	if err != nil {
-		return nil, err
-	}
-	for _, g := range getters {
-		p, fsys := g.SourceFS()
-		if p != "" {
-			server.InstallFS(p, fsys)
-		}
-	}
-	return server, nil
-}
-
-func defaultCacheDir() (string, error) {
-	out, err := runGo("", "env", "GOMODCACHE")
-	if err != nil {
-		return "", err
-	}
-	return strings.TrimSpace(string(out)), nil
-}
-
-func runGo(dir string, args ...string) ([]byte, error) {
-	cmd := exec.Command("go", args...)
-	cmd.Dir = dir
-	out, err := cmd.Output()
-	if err != nil {
-		return nil, fmt.Errorf("running go with %q: %v: %s", args, err, out)
-	}
-	return out, nil
-}
diff --git a/cmd/pkgsite/main_test.go b/cmd/pkgsite/main_test.go
index 704a425e..85026c15 100644
--- a/cmd/pkgsite/main_test.go
+++ b/cmd/pkgsite/main_test.go
@@ -5,231 +5,11 @@
 package main
 
 import (
-	"context"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path"
-	"path/filepath"
-	"regexp"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"golang.org/x/net/html"
-	"golang.org/x/pkgsite/internal/proxy/proxytest"
-	"golang.org/x/pkgsite/internal/testenv"
-	"golang.org/x/pkgsite/internal/testing/htmlcheck"
-	"golang.org/x/pkgsite/internal/testing/testhelper"
 )
 
-var (
-	in      = htmlcheck.In
-	hasText = htmlcheck.HasText
-	attr    = htmlcheck.HasAttr
-
-	// href checks for an exact match in an href attribute.
-	href = func(val string) htmlcheck.Checker {
-		return attr("href", "^"+regexp.QuoteMeta(val)+"$")
-	}
-)
-
-func TestServer(t *testing.T) {
-	testenv.MustHaveExecPath(t, "go") // for local modules
-
-	repoPath := func(fn string) string { return filepath.Join("..", "..", fn) }
-
-	abs := func(dir string) string {
-		a, err := filepath.Abs(dir)
-		if err != nil {
-			t.Fatal(err)
-		}
-		return a
-	}
-
-	localModule, _ := testhelper.WriteTxtarToTempDir(t, `
--- go.mod --
-module example.com/testmod
--- a.go --
-package a
-`)
-	cacheDir := repoPath("internal/fetch/testdata/modcache")
-	testModules := proxytest.LoadTestModules(repoPath("internal/proxy/testdata"))
-	prox, teardown := proxytest.SetupTestClient(t, testModules)
-	defer teardown()
-
-	cfg := func(modifyDefault func(*serverConfig)) serverConfig {
-		c := serverConfig{
-			paths:         []string{localModule},
-			gopathMode:    false,
-			useListedMods: true,
-			useCache:      true,
-			cacheDir:      cacheDir,
-			proxy:         prox,
-		}
-		if modifyDefault != nil {
-			modifyDefault(&c)
-		}
-		return c
-	}
-
-	modcacheChecker := in("",
-		in(".Documentation", hasText("var V = 1")),
-		sourceLinks(path.Join(filepath.ToSlash(abs(cacheDir)), "modcache.com@v1.0.0"), "a.go"))
-
-	ctx := context.Background()
-	for _, test := range []struct {
-		name     string
-		cfg      serverConfig
-		url      string
-		wantCode int
-		want     htmlcheck.Checker
-	}{
-		{
-			"local",
-			cfg(nil),
-			"example.com/testmod",
-			http.StatusOK,
-			in("",
-				in(".Documentation", hasText("There is no documentation for this package.")),
-				sourceLinks(path.Join(filepath.ToSlash(abs(localModule)), "example.com/testmod"), "a.go")),
-		},
-		{
-			"modcache",
-			cfg(nil),
-			"modcache.com@v1.0.0",
-			http.StatusOK,
-			modcacheChecker,
-		},
-		{
-			"modcache latest",
-			cfg(nil),
-			"modcache.com",
-			http.StatusOK,
-			modcacheChecker,
-		},
-		{
-			"modcache unsupported",
-			cfg(func(c *serverConfig) {
-				c.useCache = false
-			}),
-			"modcache.com",
-			http.StatusFailedDependency, // TODO(rfindley): should this be 404?
-			hasText("page is not supported"),
-		},
-		{
-			"proxy",
-			cfg(nil),
-			"example.com/single/pkg",
-			http.StatusOK,
-			hasText("G is new in v1.1.0"),
-		},
-		{
-			"proxy unsupported",
-			cfg(func(c *serverConfig) {
-				c.proxy = nil
-			}),
-			"example.com/single/pkg",
-			http.StatusFailedDependency, // TODO(rfindley): should this be 404?
-			hasText("page is not supported"),
-		},
-		{
-			"search",
-			cfg(func(c *serverConfig) {
-				c.useLocalStdlib = false
-			}),
-			"search?q=a",
-			http.StatusOK,
-			in(".SearchResults",
-				hasText("example.com/testmod"),
-			),
-		},
-		{
-			"no symbol search",
-			cfg(func(c *serverConfig) {
-				c.useLocalStdlib = false
-			}),
-			"search?q=A", // using a capital letter should not cause symbol search
-			http.StatusOK,
-			in(".SearchResults",
-				hasText("example.com/testmod"),
-			),
-		},
-		{
-			"search not found",
-			cfg(func(c *serverConfig) {
-				c.useLocalStdlib = false
-			}),
-			"search?q=zzz",
-			http.StatusOK,
-			in(".SearchResults",
-				hasText("no matches"),
-			),
-		},
-		{
-			"search vulns not found",
-			cfg(nil),
-			"search?q=GO-1234-1234",
-			http.StatusOK,
-			in(".SearchResults",
-				hasText("no matches"),
-			),
-		},
-		{
-			"search unsupported",
-			cfg(func(c *serverConfig) {
-				c.paths = nil
-				c.useLocalStdlib = false
-			}),
-			"search?q=zzz",
-			http.StatusFailedDependency,
-			hasText("page is not supported"),
-		},
-		{
-			"vulns unsupported",
-			cfg(nil),
-			"vuln/",
-			http.StatusFailedDependency,
-			hasText("page is not supported"),
-		},
-		// TODO(rfindley): add a test for the standard library once it doesn't go
-		// through the stdlib package.
-		// See also golang/go#58923.
-	} {
-		t.Run(test.name, func(t *testing.T) {
-			server, err := buildServer(ctx, test.cfg)
-			if err != nil {
-				t.Fatal(err)
-			}
-			mux := http.NewServeMux()
-			server.Install(mux.Handle, nil, nil)
-
-			w := httptest.NewRecorder()
-			mux.ServeHTTP(w, httptest.NewRequest("GET", "/"+test.url, nil))
-			if w.Code != test.wantCode {
-				t.Fatalf("got status code = %d, want %d", w.Code, test.wantCode)
-			}
-			doc, err := html.Parse(w.Body)
-			if err != nil {
-				t.Fatal(err)
-			}
-			if err := test.want(doc); err != nil {
-				if testing.Verbose() {
-					html.Render(os.Stdout, doc)
-				}
-				t.Error(err)
-			}
-		})
-	}
-}
-
-func sourceLinks(dir, filename string) htmlcheck.Checker {
-	filesPath := path.Join("/files", dir) + "/"
-	return in("",
-		in(".UnitMeta-repo a", href(filesPath)),
-		in(".UnitFiles-titleLink a", href(filesPath)),
-		in(".UnitFiles-fileList a", href(filesPath+filename)))
-}
-
 func TestCollectPaths(t *testing.T) {
 	got := collectPaths([]string{"a", "b,c2,d3", "e4", "f,g"})
 	want := []string{"a", "b", "c2", "d3", "e4", "f", "g"}