From 1f39ab54702217a4b686a5902aeec7e5cad36929 Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Wed, 17 Dec 2025 21:10:20 +0000 Subject: internal/frontend: add CodeWiki link support to pkgsite - If a module's source repo exists in CodeWiki, display a link to codewiki.google/repo. - Disable the CodeWiki link from being displayed for screentests by adding it to the hidden elements. Change-Id: Ia5feb913280b8066806e56524bf9d32ce51f0614 Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/730880 Reviewed-by: Jonathan Amsterdam LUCI-TryBot-Result: Go LUCI kokoro-CI: kokoro Auto-Submit: Ethan Lee --- cmd/frontend/main.go | 24 ++-- internal/frontend/depsdev.go | 86 ------------- internal/frontend/links.go | 140 +++++++++++++++++++++ internal/frontend/links_test.go | 85 +++++++++++++ internal/frontend/server.go | 62 ++++----- internal/frontend/unit.go | 10 +- static/frontend/unit/main/_meta.tmpl | 10 ++ static/shared/icon/codewiki-logo.svg | 1 + tests/screentest/testcases.ci.txt | 2 +- tests/screentest/testcases.txt | 5 +- .../testdata/testcases.ci/vuln-no-results.want.png | Bin 92718 -> 92717 bytes .../testdata/testcases.ci/vuln-search.want.png | Bin 150359 -> 150360 bytes tests/screentest/testdata/testcases/badge.want.png | Bin 101232 -> 101225 bytes .../testcases/license-policy-540x1080.want.png | Bin 313433 -> 313426 bytes .../testdata/testcases/nav-submenu.want.png | Bin 133487 -> 129881 bytes 15 files changed, 292 insertions(+), 133 deletions(-) delete mode 100644 internal/frontend/depsdev.go create mode 100644 internal/frontend/links.go create mode 100644 internal/frontend/links_test.go create mode 100644 static/shared/icon/codewiki-logo.svg diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go index 7e070aa6..c3541cd6 100644 --- a/cmd/frontend/main.go +++ b/cmd/frontend/main.go @@ -152,18 +152,18 @@ func main() { TaskIDChangeInterval: config.TaskIDChangeIntervalFrontend, } server, err := frontend.NewServer(frontend.ServerConfig{ - Config: cfg, - FetchServer: fetchServer, - DataSourceGetter: dsg, - Queue: fetchQueue, - TemplateFS: template.TrustedFSFromTrustedSource(staticSource), - StaticFS: os.DirFS(*staticFlag), - ThirdPartyFS: os.DirFS(*thirdPartyPath), - DevMode: *devMode, - LocalMode: *localMode, - Reporter: reporter, - VulndbClient: vc, - DepsDevHTTPClient: &http.Client{Transport: new(ochttp.Transport)}, + Config: cfg, + FetchServer: fetchServer, + DataSourceGetter: dsg, + Queue: fetchQueue, + TemplateFS: template.TrustedFSFromTrustedSource(staticSource), + StaticFS: os.DirFS(*staticFlag), + ThirdPartyFS: os.DirFS(*thirdPartyPath), + DevMode: *devMode, + LocalMode: *localMode, + Reporter: reporter, + VulndbClient: vc, + HTTPClient: &http.Client{Transport: new(ochttp.Transport)}, }) if err != nil { log.Fatalf(ctx, "frontend.NewServer: %v", err) diff --git a/internal/frontend/depsdev.go b/internal/frontend/depsdev.go deleted file mode 100644 index 0b1dbb7d..00000000 --- a/internal/frontend/depsdev.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package frontend - -import ( - "context" - "encoding/json" - "errors" - "net/http" - "net/url" - "time" - - "golang.org/x/pkgsite/internal" - "golang.org/x/pkgsite/internal/log" -) - -const ( - // depsDevBase is the base URL for requests to deps.dev. - // It should not include a trailing slash. - depsDevBase = "https://deps.dev" - // depsDevTimeout is the time budget for making requests to deps.dev. - depsDevTimeout = 250 * time.Millisecond -) - -// depsDevURLGenerator returns a function that will return a URL for the given -// module version on deps.dev. If the URL can't be generated within -// depsDevTimeout then the empty string is returned instead. -func depsDevURLGenerator(ctx context.Context, client *http.Client, um *internal.UnitMeta) func() string { - ctx, cancel := context.WithTimeout(ctx, depsDevTimeout) - url := make(chan string, 1) - go func() { - u, err := fetchDepsDevURL(ctx, client, um.ModulePath, um.Version) - switch { - case errors.Is(err, context.Canceled): - log.Warningf(ctx, "fetching url from deps.dev: %v", err) - case errors.Is(err, context.DeadlineExceeded): - log.Warningf(ctx, "fetching url from deps.dev: %v", err) - case err != nil: - log.Errorf(ctx, "fetching url from deps.dev: %v", err) - } - url <- u - }() - return func() string { - defer cancel() - return <-url - } -} - -// fetchDepsDevURL makes a request to deps.dev to check whether the given -// module version is known there, and if so it returns the link to that module -// version page on deps.dev. -func fetchDepsDevURL(ctx context.Context, client *http.Client, modulePath, version string) (string, error) { - u := depsDevBase + "/_/s/go" + - "/p/" + url.PathEscape(modulePath) + - "/v/" + url.PathEscape(version) + - "/exists" - req, err := http.NewRequestWithContext(ctx, "GET", u, nil) - if err != nil { - return "", err - } - resp, err := client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - switch resp.StatusCode { - case http.StatusNotFound: - return "", nil // No link to return. - case http.StatusOK: - // Handled below. - default: - return "", errors.New(resp.Status) - } - var r struct { - stem, Name, Version string - } - if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { - return "", err - } - if r.Name == "" || r.Version == "" { - return "", errors.New("name or version unset in response") - } - return depsDevBase + "/go/" + url.PathEscape(r.Name) + "/" + url.PathEscape(r.Version), nil -} diff --git a/internal/frontend/links.go b/internal/frontend/links.go new file mode 100644 index 00000000..2edc6804 --- /dev/null +++ b/internal/frontend/links.go @@ -0,0 +1,140 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package frontend + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/pkgsite/internal" + "golang.org/x/pkgsite/internal/log" +) + +const ( + // depsDevBase is the base URL for requests to deps.dev. + // It should not include a trailing slash. + depsDevBase = "https://deps.dev" + // depsDevTimeout is the time budget for making requests to deps.dev. + depsDevTimeout = 250 * time.Millisecond +) + +var ( + codeWikiURLBase = "https://codewiki.google/" + codeWikiExistsURL = "https://codewiki.google/_/exists/" + codeWikiTimeout = 1 * time.Second +) + +type fetcher func(context.Context, *http.Client) (string, error) + +// newURLGenerator returns a function that will return a URL. +// If the URL can't be generated within the timeout then the empty string is returned. +func newURLGenerator(ctx context.Context, client *http.Client, serviceName string, timeout time.Duration, fetch fetcher) func() string { + ctx, cancel := context.WithTimeout(ctx, timeout) + url := make(chan string, 1) + go func() { + u, err := fetch(ctx, client) + switch { + case errors.Is(err, context.Canceled): + log.Warningf(ctx, "fetching url from %s: %v", serviceName, err) + case errors.Is(err, context.DeadlineExceeded): + log.Warningf(ctx, "fetching url from %s: %v", serviceName, err) + case err != nil: + log.Errorf(ctx, "fetching url from %s: %v", serviceName, err) + } + url <- u + }() + return func() string { + defer cancel() + return <-url + } +} + +// depsDevURLGenerator returns a function that will return a URL for the given +// module version on deps.dev. If the URL can't be generated within +// depsDevTimeout then the empty string is returned instead. +func depsDevURLGenerator(ctx context.Context, client *http.Client, um *internal.UnitMeta) func() string { + fetch := func(ctx context.Context, client *http.Client) (string, error) { + return fetchDepsDevURL(ctx, client, um.ModulePath, um.Version) + } + return newURLGenerator(ctx, client, "deps.dev", depsDevTimeout, fetch) +} + +// fetchDepsDevURL makes a request to deps.dev to check whether the given +// module version is known there, and if so it returns the link to that module +// version page on deps.dev. +func fetchDepsDevURL(ctx context.Context, client *http.Client, modulePath, version string) (string, error) { + u := depsDevBase + "/_/s/go" + + "/p/" + url.PathEscape(modulePath) + + "/v/" + url.PathEscape(version) + + "/exists" + req, err := http.NewRequestWithContext(ctx, "GET", u, nil) + if err != nil { + return "", err + } + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusNotFound: + return "", nil // No link to return. + case http.StatusOK: + // Handled below. + default: + return "", errors.New(resp.Status) + } + var r struct { + stem, Name, Version string + } + if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { + return "", err + } + if r.Name == "" || r.Version == "" { + return "", errors.New("name or version unset in response") + } + return depsDevBase + "/go/" + url.PathEscape(r.Name) + "/" + url.PathEscape(r.Version), nil +} + +// codeWikiURLGenerator returns a function that will return a URL for the given +// module version on codewiki. If the URL can't be generated within +// codeWikiTimeout then the empty string is returned instead. +func codeWikiURLGenerator(ctx context.Context, client *http.Client, um *internal.UnitMeta) func() string { + fetch := func(ctx context.Context, client *http.Client) (string, error) { + return fetchCodeWikiURL(ctx, client, um.ModulePath) + } + return newURLGenerator(ctx, client, "codewiki.google", codeWikiTimeout, fetch) +} + +// fetchCodeWikiURL makes a request to codewiki to check whether the given +// path is known there, and if so it returns the link to that page. +func fetchCodeWikiURL(ctx context.Context, client *http.Client, path string) (string, error) { + if strings.HasPrefix(path, "golang.org/x/") { + path = strings.Replace(path, "golang.org/x/", "github.com/golang/", 1) + } + // TODO: Add support for other hosts as needed. + if !strings.HasPrefix(path, "github.com/") { + return "", nil + } + req, err := http.NewRequestWithContext(ctx, "GET", codeWikiExistsURL+path, nil) + if err != nil { + return "", err + } + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return fmt.Sprintf("%s%s", codeWikiURLBase, path), nil + } + return "", errors.New(resp.Status) +} diff --git a/internal/frontend/links_test.go b/internal/frontend/links_test.go new file mode 100644 index 00000000..b5417e9d --- /dev/null +++ b/internal/frontend/links_test.go @@ -0,0 +1,85 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package frontend + +import ( + "context" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "testing" + + "golang.org/x/pkgsite/internal" +) + +func TestCodeWikiURLGenerator(t *testing.T) { + // The log package is periodically used to log warnings on a + // separate goroutine, which can pollute test output. + // For this test, we can discard all of that output. + log.SetOutput(io.Discard) + t.Cleanup(func() { + log.SetOutput(os.Stderr) + }) + mux := http.NewServeMux() + mux.HandleFunc("/_/exists/github.com/owner/repo", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + mux.HandleFunc("/_/exists/github.com/golang/glog", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + + oldCodeWikiURLBase := codeWikiURLBase + oldCodeWikiExistsURL := codeWikiExistsURL + codeWikiURLBase = server.URL + "/" + codeWikiExistsURL = server.URL + "/_/exists/" + t.Cleanup(func() { + codeWikiURLBase = oldCodeWikiURLBase + codeWikiExistsURL = oldCodeWikiExistsURL + }) + + testCases := []struct { + name, modulePath, path string + want string + }{ + { + name: "github repo", + modulePath: "github.com/owner/repo", + want: server.URL + "/github.com/owner/repo", + }, + { + name: "github repo subpackage", + modulePath: "github.com/owner/repo", + want: server.URL + "/github.com/owner/repo", + }, + { + name: "github repo not found", + modulePath: "github.com/owner/repo-not-found", + want: "", + }, + { + name: "non-github repo", + modulePath: "example.com/owner/repo", + want: "", + }, + { + name: "golang.org/x/ repo", + modulePath: "golang.org/x/glog", + want: server.URL + "/github.com/golang/glog", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + um := &internal.UnitMeta{ModuleInfo: internal.ModuleInfo{ModulePath: tc.modulePath}} + url := codeWikiURLGenerator(context.Background(), server.Client(), um)() + if url != tc.want { + t.Errorf("codeWikiURLGenerator(ctx, client, %q) = %q, want %q, got %q", tc.path, url, tc.want, url) + } + }) + } +} diff --git a/internal/frontend/server.go b/internal/frontend/server.go index 5f621fdf..e15f9298 100644 --- a/internal/frontend/server.go +++ b/internal/frontend/server.go @@ -60,7 +60,7 @@ type Server struct { vulnClient *vuln.Client versionID string instanceID string - depsDevHTTPClient *http.Client + HTTPClient *http.Client mu sync.Mutex // Protects all fields below templates map[string]*template.Template @@ -84,18 +84,18 @@ type ServerConfig struct { FetchServer FetchServerInterface // DataSourceGetter should return a DataSource on each call. // It should be goroutine-safe. - DataSourceGetter func(context.Context) internal.DataSource - Queue queue.Queue - TemplateFS template.TrustedFS // for loading templates safely - StaticFS fs.FS // for static/ directory - ThirdPartyFS fs.FS // for third_party/ directory - DevMode bool - LocalMode bool - GoDocMode bool - LocalModules []LocalModule - Reporter derrors.Reporter - VulndbClient *vuln.Client - DepsDevHTTPClient *http.Client + DataSourceGetter func(context.Context) internal.DataSource + Queue queue.Queue + TemplateFS template.TrustedFS // for loading templates safely + StaticFS fs.FS // for static/ directory + ThirdPartyFS fs.FS // for third_party/ directory + DevMode bool + LocalMode bool + GoDocMode bool + LocalModules []LocalModule + Reporter derrors.Reporter + VulndbClient *vuln.Client + HTTPClient *http.Client } // NewServer creates a new Server for the given database and template directory. @@ -107,24 +107,24 @@ func NewServer(scfg ServerConfig) (_ *Server, err error) { } dochtml.LoadTemplates(scfg.TemplateFS) s := &Server{ - fetchServer: scfg.FetchServer, - getDataSource: scfg.DataSourceGetter, - queue: scfg.Queue, - templateFS: scfg.TemplateFS, - staticFS: scfg.StaticFS, - thirdPartyFS: scfg.ThirdPartyFS, - devMode: scfg.DevMode, - localMode: scfg.LocalMode, - goDocMode: scfg.GoDocMode, - localModules: scfg.LocalModules, - templates: ts, - reporter: scfg.Reporter, - fileMux: http.NewServeMux(), - vulnClient: scfg.VulndbClient, - depsDevHTTPClient: scfg.DepsDevHTTPClient, - } - if s.depsDevHTTPClient == nil { - s.depsDevHTTPClient = http.DefaultClient + fetchServer: scfg.FetchServer, + getDataSource: scfg.DataSourceGetter, + queue: scfg.Queue, + templateFS: scfg.TemplateFS, + staticFS: scfg.StaticFS, + thirdPartyFS: scfg.ThirdPartyFS, + devMode: scfg.DevMode, + localMode: scfg.LocalMode, + goDocMode: scfg.GoDocMode, + localModules: scfg.LocalModules, + templates: ts, + reporter: scfg.Reporter, + fileMux: http.NewServeMux(), + vulnClient: scfg.VulndbClient, + HTTPClient: scfg.HTTPClient, + } + if s.HTTPClient == nil { + s.HTTPClient = http.DefaultClient } if scfg.Config != nil { s.appVersionLabel = scfg.Config.AppVersionLabel() diff --git a/internal/frontend/unit.go b/internal/frontend/unit.go index 96aafa9b..55471797 100644 --- a/internal/frontend/unit.go +++ b/internal/frontend/unit.go @@ -97,7 +97,7 @@ type UnitPage struct { Details any // GoDocMode indicates whether to suppress the unit header and right hand unit metadata. - // If set to true, the page will also not have Vulns or a DepsDevURL. + // If set to true, the page will also not have Vulns, DepsDevURL or a CodeWikiURL. GoDocMode bool // Vulns holds vulnerability information. @@ -106,6 +106,9 @@ type UnitPage struct { // DepsDevURL holds the full URL to this module version on deps.dev. DepsDevURL string + // CodeWikiURL holds the full URL to this module's repo on codewiki.google. + CodeWikiURL string + // IsGoProject is true if the package is from the standard library or a // golang.org sub-repository. IsGoProject bool @@ -141,8 +144,10 @@ func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *ht } makeDepsDevURL := func() string { return "" } + makeCodeWikiURL := func() string { return "" } if !s.goDocMode { - makeDepsDevURL = depsDevURLGenerator(ctx, s.depsDevHTTPClient, um) + makeDepsDevURL = depsDevURLGenerator(ctx, s.HTTPClient, um) + makeCodeWikiURL = codeWikiURLGenerator(ctx, s.HTTPClient, um) } // Use GOOS and GOARCH query parameters to create a build context, which @@ -239,6 +244,7 @@ func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *ht if !s.goDocMode { page.DepsDevURL = makeDepsDevURL() + page.CodeWikiURL = makeCodeWikiURL() } // Show the banner if there was no error getting the latest major version, diff --git a/static/frontend/unit/main/_meta.tmpl b/static/frontend/unit/main/_meta.tmpl index 649d0d8b..8e32eb23 100644 --- a/static/frontend/unit/main/_meta.tmpl +++ b/static/frontend/unit/main/_meta.tmpl @@ -40,6 +40,16 @@ {{end}} + {{with .CodeWikiURL}} +
  • + + Code Wiki Logo + Code Wiki + +
    • + {{end}} {{template "unit-meta-links" .Details.ReadmeLinks}} {{template "unit-meta-links" .Details.DocLinks}} {{template "unit-meta-links" .Details.ModuleReadmeLinks}} diff --git a/static/shared/icon/codewiki-logo.svg b/static/shared/icon/codewiki-logo.svg new file mode 100644 index 00000000..28bd44ac --- /dev/null +++ b/static/shared/icon/codewiki-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/screentest/testcases.ci.txt b/tests/screentest/testcases.ci.txt index 0020685f..d37bcdfc 100644 --- a/tests/screentest/testcases.ci.txt +++ b/tests/screentest/testcases.ci.txt @@ -3,7 +3,7 @@ windowsize 1536x960 block https://codecov.io/* https://travis-ci.com/* {{$ready := "[role='treeitem'][aria-selected='true']"}} -{{$hideElements := "document.querySelector(\"[data-test-id='UnitHeader-importedby']\")?.remove();document.querySelector(\"[data-test-id='meta-link-depsdev']\")?.remove();"}} +{{$hideElements := "document.querySelector(\"[data-test-id='UnitHeader-importedby']\")?.remove();document.querySelector(\"[data-test-id='meta-link-depsdev']\")?.remove();document.querySelector(\"[data-test-id='meta-link-codewiki']\")?.remove();"}} {{$scrollTop := "window.scrollTo({top:0});"}} test vuln diff --git a/tests/screentest/testcases.txt b/tests/screentest/testcases.txt index ed5c7474..4f94269e 100644 --- a/tests/screentest/testcases.txt +++ b/tests/screentest/testcases.txt @@ -11,7 +11,7 @@ block https://codecov.io/* https://travis-ci.com/* # The aria-selected attribute is added by the last piece of JS to run. {{$ready := "[role='treeitem'][aria-selected='true']"}} -{{$hideElements := "document.querySelector(\"[data-test-id='UnitHeader-importedby']\")?.remove();document.querySelector(\"[data-test-id='meta-link-depsdev']\")?.remove();"}} +{{$hideElements := "document.querySelector(\"[data-test-id='UnitHeader-importedby']\")?.remove();document.querySelector(\"[data-test-id='meta-link-depsdev']\")?.remove();document.querySelector(\"[data-test-id='meta-link-codewiki']\")?.remove();"}} # JS for hiding the "Links" heading. We are already hiding the deps.dev link because # its presence depends on a timeout, but for some modules, if the link isn't @@ -231,5 +231,8 @@ capture fullscreen 540x1080 test nav submenu path /github.com/jba/bit +wait {{$ready}} +eval {{$hideElements}} +sleep 3s click .js-desktop-menu-hover capture viewport diff --git a/tests/screentest/testdata/testcases.ci/vuln-no-results.want.png b/tests/screentest/testdata/testcases.ci/vuln-no-results.want.png index 5c4c0be8..fa1fc823 100644 Binary files a/tests/screentest/testdata/testcases.ci/vuln-no-results.want.png and b/tests/screentest/testdata/testcases.ci/vuln-no-results.want.png differ diff --git a/tests/screentest/testdata/testcases.ci/vuln-search.want.png b/tests/screentest/testdata/testcases.ci/vuln-search.want.png index d50d15b2..4662b137 100644 Binary files a/tests/screentest/testdata/testcases.ci/vuln-search.want.png and b/tests/screentest/testdata/testcases.ci/vuln-search.want.png differ diff --git a/tests/screentest/testdata/testcases/badge.want.png b/tests/screentest/testdata/testcases/badge.want.png index c41c3c59..af178292 100644 Binary files a/tests/screentest/testdata/testcases/badge.want.png and b/tests/screentest/testdata/testcases/badge.want.png differ diff --git a/tests/screentest/testdata/testcases/license-policy-540x1080.want.png b/tests/screentest/testdata/testcases/license-policy-540x1080.want.png index 74bc24cc..1b0c1b67 100644 Binary files a/tests/screentest/testdata/testcases/license-policy-540x1080.want.png and b/tests/screentest/testdata/testcases/license-policy-540x1080.want.png differ diff --git a/tests/screentest/testdata/testcases/nav-submenu.want.png b/tests/screentest/testdata/testcases/nav-submenu.want.png index 75aed572..429d0898 100644 Binary files a/tests/screentest/testdata/testcases/nav-submenu.want.png and b/tests/screentest/testdata/testcases/nav-submenu.want.png differ -- cgit v1.3