cmd/pkgsite,internal: serve source

- Set up a /files endpoint on the server that can serve
  files from fs.FS implementations.

- Add source.FilesInfo, which returns a source.Info that links to
  /files paths. Use it to implement the SourceInfo method of the local
  ModuleGetters.

- Add a SourceFS method to the local MethodGetters so they can tell
  the server how to serve their files.

- Improve the tests in cmd/pkgsite to verify the source links.

Fixes golang/go#47982

Change-Id: Iff42bad15c4abaf408a364e58b7c0f0dad60b40d
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/347932
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>

2 files changed, 87 insertions, 18 deletions

diff --git a/cmd/pkgsite/main.go b/cmd/pkgsite/main.go
index 3ec5bf8c..31e55c8e 100644
--- a/cmd/pkgsite/main.go
+++ b/cmd/pkgsite/main.go
@@ -81,16 +81,16 @@ func main() {
 		paths = []string{"."}
 	}
 
-	var downloadDir string
+	var modCacheDir string
 	if *useCache {
-		downloadDir = *cacheDir
-		if downloadDir == "" {
+		modCacheDir = *cacheDir
+		if modCacheDir == "" {
 			var err error
-			downloadDir, err = defaultCacheDir()
+			modCacheDir, err = defaultCacheDir()
 			if err != nil {
 				die("%v", err)
 			}
-			if downloadDir == "" {
+			if modCacheDir == "" {
 				die("empty value for GOMODCACHE")
 			}
 		}
@@ -116,7 +116,7 @@ func main() {
 		stdlib.SetGoRepoPath(*goRepoPath)
 	}
 
-	server, err := newServer(ctx, paths, *gopathMode, downloadDir, prox)
+	server, err := newServer(ctx, paths, *gopathMode, modCacheDir, prox)
 	if err != nil {
 		die("%s\nMaybe you need to provide the location of static assets with -static.", err)
 	}
@@ -144,7 +144,11 @@ func collectPaths(args []string) []string {
 func newServer(ctx context.Context, paths []string, gopathMode bool, downloadDir string, prox *proxy.Client) (*frontend.Server, error) {
 	getters := buildGetters(ctx, paths, gopathMode)
 	if downloadDir != "" {
-		getters = append(getters, fetch.NewFSProxyModuleGetter(downloadDir))
+		g, err := fetch.NewFSProxyModuleGetter(downloadDir)
+		if err != nil {
+			return nil, err
+		}
+		getters = append(getters, g)
 	}
 	if prox != nil {
 		getters = append(getters, fetch.NewProxyModuleGetter(prox, source.NewClient(time.Second)))
@@ -161,6 +165,12 @@ func newServer(ctx context.Context, paths []string, gopathMode bool, downloadDir
 	if err != nil {
 		return nil, err
 	}
+	for _, g := range getters {
+		p, fsys := g.SourceFS()
+		if p != "" {
+			server.InstallFS(p, fsys)
+		}
+	}
 	return server, nil
 }
 
diff --git a/cmd/pkgsite/main_test.go b/cmd/pkgsite/main_test.go
index 021a56e2..a938eb3c 100644
--- a/cmd/pkgsite/main_test.go
+++ b/cmd/pkgsite/main_test.go
@@ -9,17 +9,40 @@ import (
 	"flag"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path"
 	"path/filepath"
-	"strings"
+	"regexp"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"golang.org/x/net/html"
 	"golang.org/x/pkgsite/internal/proxy/proxytest"
+	"golang.org/x/pkgsite/internal/testing/htmlcheck"
+)
+
+var (
+	in      = htmlcheck.In
+	hasText = htmlcheck.HasText
+	attr    = htmlcheck.HasAttr
+
+	// href checks for an exact match in an href attribute.
+	href = func(val string) htmlcheck.Checker {
+		return attr("href", "^"+regexp.QuoteMeta(val)+"$")
+	}
 )
 
 func Test(t *testing.T) {
 	repoPath := func(fn string) string { return filepath.Join("..", "..", fn) }
 
+	abs := func(dir string) string {
+		a, err := filepath.Abs(dir)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return a
+	}
+
 	localModule := repoPath("internal/fetch/testdata/has_go_mod")
 	cacheDir := repoPath("internal/fetch/testdata/modcache")
 	flag.Set("static", repoPath("static"))
@@ -34,15 +57,37 @@ func Test(t *testing.T) {
 	mux := http.NewServeMux()
 	server.Install(mux.Handle, nil, nil)
 
+	modcacheChecker := in("",
+		in(".Documentation", hasText("var V = 1")),
+		sourceLinks(path.Join(abs(cacheDir), "modcache.com@v1.0.0"), "a.go"))
+
 	for _, test := range []struct {
-		name       string
-		url        string
-		wantInBody string
+		name string
+		url  string
+		want htmlcheck.Checker
 	}{
-		{"local", "example.com/testmod", "There is no documentation for this package."},
-		{"modcache", "modcache.com@v1.0.0", "var V = 1"},
-		{"modcache2", "modcache.com", "var V = 1"},
-		{"proxy", "example.com/single/pkg", "G is new in v1.1.0"},
+		{
+			"local",
+			"example.com/testmod",
+			in("",
+				in(".Documentation", hasText("There is no documentation for this package.")),
+				sourceLinks(path.Join(abs(localModule), "example.com/testmod"), "a.go")),
+		},
+		{
+			"modcache",
+			"modcache.com@v1.0.0",
+			modcacheChecker,
+		},
+		{
+			"modcache latest",
+			"modcache.com",
+			modcacheChecker,
+		},
+		{
+			"proxy",
+			"example.com/single/pkg",
+			hasText("G is new in v1.1.0"),
+		},
 	} {
 		t.Run(test.name, func(t *testing.T) {
 			w := httptest.NewRecorder()
@@ -50,14 +95,28 @@ func Test(t *testing.T) {
 			if w.Code != http.StatusOK {
 				t.Fatalf("got status code = %d, want %d", w.Code, http.StatusOK)
 			}
-			body := w.Body.String()
-			if !strings.Contains(body, test.wantInBody) {
-				t.Fatalf("body is missing %q\n%s", test.wantInBody, body)
+			doc, err := html.Parse(w.Body)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := test.want(doc); err != nil {
+				if testing.Verbose() {
+					html.Render(os.Stdout, doc)
+				}
+				t.Error(err)
 			}
 		})
 	}
 }
 
+func sourceLinks(dir, filename string) htmlcheck.Checker {
+	filesPath := path.Join("/files", dir) + "/"
+	return in("",
+		in(".UnitMeta-repo a", href(filesPath)),
+		in(".UnitFiles-titleLink a", href(filesPath)),
+		in(".UnitFiles-fileList a", href(filesPath+filename)))
+}
+
 func TestCollectPaths(t *testing.T) {
 	got := collectPaths([]string{"a", "b,c2,d3", "e4", "f,g"})
 	want := []string{"a", "b", "c2", "d3", "e4", "f", "g"}