aboutsummaryrefslogtreecommitdiff
path: root/internal/postgres
diff options
context:
space:
mode:
authorJonathan Amsterdam <jba@google.com>2021-07-14 07:07:26 -0400
committerJonathan Amsterdam <jba@google.com>2021-07-14 13:59:21 +0000
commitbcdc85aa4c7bb05e8b3ffe7eaef7e6f64bab4081 (patch)
treefad66b5f8164f014f337b30972d58809ee1b035e /internal/postgres
parented966fad5adc568bac42880c0e5ae834f43c773f (diff)
downloadgo-x-pkgsite-bcdc85aa4c7bb05e8b3ffe7eaef7e6f64bab4081.tar.xz
internal: move SearchResult to postgres package
Change-Id: I73f0c7c35a5221302c27012c9118fa3bed5fdd62 Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/334529 Trust: Jonathan Amsterdam <jba@google.com> Run-TryBot: Jonathan Amsterdam <jba@google.com> TryBot-Result: kokoro <noreply+kokoro@google.com> Reviewed-by: Julie Qiu <julie@golang.org>
Diffstat (limited to 'internal/postgres')
-rw-r--r--internal/postgres/benchmarks_test.go3
-rw-r--r--internal/postgres/search.go76
-rw-r--r--internal/postgres/search_test.go36
-rw-r--r--internal/postgres/symbolsearch.go4
-rw-r--r--internal/postgres/symbolsearch_test.go8
5 files changed, 87 insertions, 40 deletions
diff --git a/internal/postgres/benchmarks_test.go b/internal/postgres/benchmarks_test.go
index ca4531cb..57f2cf25 100644
--- a/internal/postgres/benchmarks_test.go
+++ b/internal/postgres/benchmarks_test.go
@@ -8,7 +8,6 @@ import (
"context"
"testing"
- "golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/config"
"golang.org/x/pkgsite/internal/database"
)
@@ -43,7 +42,7 @@ func BenchmarkSearch(b *testing.B) {
b.Fatal(err)
}
db := New(ddb)
- searchers := map[string]func(context.Context, string, SearchOptions) ([]*internal.SearchResult, error){
+ searchers := map[string]func(context.Context, string, SearchOptions) ([]*SearchResult, error){
"db.Search": db.Search,
}
for name, search := range searchers {
diff --git a/internal/postgres/search.go b/internal/postgres/search.go
index d6330007..01227045 100644
--- a/internal/postgres/search.go
+++ b/internal/postgres/search.go
@@ -65,7 +65,7 @@ type searchResponse struct {
// 'popular'), to be used in logging and reporting.
source string
// results are partially filled out from only the search_documents table.
- results []*internal.SearchResult
+ results []*SearchResult
// err indicates a technical failure of the search query, or that results are
// not provably complete.
err error
@@ -109,6 +109,54 @@ type SearchOptions struct {
SearchSymbols bool
}
+// SearchResult represents a single search result from SearchDocuments.
+type SearchResult struct {
+ Name string
+ PackagePath string
+ ModulePath string
+ Version string
+ Synopsis string
+ Licenses []string
+
+ CommitTime time.Time
+
+ // Score is used to sort items in an array of SearchResult.
+ Score float64
+
+ // NumImportedBy is the number of packages that import PackagePath.
+ NumImportedBy uint64
+
+ // SameModule is a list of SearchResults from the same module as this one,
+ // with lower scores.
+ SameModule []*SearchResult
+
+ // OtherMajor is a set of module paths with the same series path but at
+ // different major versions of this module.
+ OtherMajor map[string]bool
+
+ // NumResults is the total number of packages that were returned for this
+ // search.
+ NumResults uint64
+
+ // Approximate reports whether NumResults is an approximate count. NumResults
+ // can be approximate if search scanned only a subset of documents, and
+ // result count is estimated using the hyperloglog algorithm.
+ Approximate bool
+
+ // Symbol information returned by a search request.
+ // Only populated for symbol search mode.
+ SymbolName string
+ SymbolKind internal.SymbolKind
+ SymbolSynopsis string
+ SymbolGOOS string
+ SymbolGOARCH string
+
+ // Offset is the 0-based number of this row in the DB query results, which
+ // is the value to use in a SQL OFFSET clause to have this row be the first
+ // one returned.
+ Offset int
+}
+
// Search executes two search requests concurrently:
// - a sequential scan of packages in descending order of popularity.
// - all packages ("deep" search) using an inverted index to filter to search
@@ -132,7 +180,7 @@ type SearchOptions struct {
// The gap in this optimization is search terms that are very frequent, but
// rarely relevant: "int" or "package", for example. In these cases we'll pay
// the penalty of a deep search that scans nearly every package.
-func (db *DB) Search(ctx context.Context, q string, opts SearchOptions) (_ []*internal.SearchResult, err error) {
+func (db *DB) Search(ctx context.Context, q string, opts SearchOptions) (_ []*SearchResult, err error) {
defer derrors.WrapStack(&err, "DB.Search(ctx, %q, %+v)", q, opts)
if opts.Limit == 0 {
opts.Limit = opts.MaxResults
@@ -151,7 +199,7 @@ func (db *DB) Search(ctx context.Context, q string, opts SearchOptions) (_ []*in
return nil, err
}
// Filter out excluded paths.
- var results []*internal.SearchResult
+ var results []*SearchResult
for _, r := range resp.results {
ex, err := db.IsExcluded(ctx, r.PackagePath)
if err != nil {
@@ -286,7 +334,7 @@ func (db *DB) deepSearch(ctx context.Context, q string, limit, offset, maxResult
OFFSET $3`, scoreExpr)
var (
- results []*internal.SearchResult
+ results []*SearchResult
err error
)
if experiment.IsActive(ctx, internal.ExperimentSearchIncrementally) {
@@ -294,7 +342,7 @@ func (db *DB) deepSearch(ctx context.Context, q string, limit, offset, maxResult
const pageSize = 10 // TODO(jba): get from elsewhere
additionalRows := 10 // after reaching pageSize module paths
collect := func(rows *sql.Rows) error {
- var r internal.SearchResult
+ var r SearchResult
if err := rows.Scan(&r.PackagePath, &r.Version, &r.ModulePath, &r.CommitTime,
&r.NumImportedBy, &r.Score, &r.NumResults); err != nil {
return fmt.Errorf("rows.Scan(): %v", err)
@@ -314,7 +362,7 @@ func (db *DB) deepSearch(ctx context.Context, q string, limit, offset, maxResult
err = db.db.RunQueryIncrementally(ctx, query, fetchSize, collect, q, limit, offset)
} else {
collect := func(rows *sql.Rows) error {
- var r internal.SearchResult
+ var r SearchResult
if err := rows.Scan(&r.PackagePath, &r.Version, &r.ModulePath, &r.CommitTime,
&r.NumImportedBy, &r.Score, &r.NumResults); err != nil {
return fmt.Errorf("rows.Scan(): %v", err)
@@ -352,9 +400,9 @@ func (db *DB) popularSearch(ctx context.Context, searchQuery string, limit, offs
imported_by_count,
score
FROM popular_search($1, $2, $3, $4, $5)`
- var results []*internal.SearchResult
+ var results []*SearchResult
collect := func(rows *sql.Rows) error {
- var r internal.SearchResult
+ var r SearchResult
if err := rows.Scan(&r.PackagePath, &r.Version, &r.ModulePath, &r.CommitTime,
&r.NumImportedBy, &r.Score); err != nil {
return fmt.Errorf("rows.Scan(): %v", err)
@@ -387,7 +435,7 @@ func (db *DB) popularSearch(ctx context.Context, searchQuery string, limit, offs
// addPackageDataToSearchResults adds package information to SearchResults that is not stored
// in the search_documents table.
-func (db *DB) addPackageDataToSearchResults(ctx context.Context, results []*internal.SearchResult) (err error) {
+func (db *DB) addPackageDataToSearchResults(ctx context.Context, results []*SearchResult) (err error) {
defer derrors.WrapStack(&err, "DB.addPackageDataToSearchResults(results)")
if len(results) == 0 {
return nil
@@ -396,7 +444,7 @@ func (db *DB) addPackageDataToSearchResults(ctx context.Context, results []*inte
keys []string
// resultMap tracks PackagePath->SearchResult, to allow joining with the
// returned package data.
- resultMap = make(map[string]*internal.SearchResult)
+ resultMap = make(map[string]*SearchResult)
)
for _, r := range results {
resultMap[r.PackagePath] = r
@@ -476,10 +524,10 @@ func sortAndDedup(s []string) []string {
// Packages from lower major versions of the module are grouped under the first
// package of the highest major version. But they are not removed from the
// top-level list.
-func groupSearchResults(rs []*internal.SearchResult) []*internal.SearchResult {
- modules := map[string]*internal.SearchResult{} // module path to first result
- series := map[string]*internal.SearchResult{} // series path to result with max major version
- var results []*internal.SearchResult
+func groupSearchResults(rs []*SearchResult) []*SearchResult {
+ modules := map[string]*SearchResult{} // module path to first result
+ series := map[string]*SearchResult{} // series path to result with max major version
+ var results []*SearchResult
for _, r := range rs {
f := modules[r.ModulePath]
if f == nil {
diff --git a/internal/postgres/search_test.go b/internal/postgres/search_test.go
index 83cf524e..91cb734b 100644
--- a/internal/postgres/search_test.go
+++ b/internal/postgres/search_test.go
@@ -590,8 +590,8 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) {
}},
}
- kubeResult = func(score float64, numResults uint64) *internal.SearchResult {
- return &internal.SearchResult{
+ kubeResult = func(score float64, numResults uint64) *SearchResult {
+ return &SearchResult{
Name: pkgKube.Name,
PackagePath: pkgKube.Path,
Synopsis: pkgKube.Documentation[0].Synopsis,
@@ -604,8 +604,8 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) {
}
}
- goCDKResult = func(score float64, numResults uint64) *internal.SearchResult {
- return &internal.SearchResult{
+ goCDKResult = func(score float64, numResults uint64) *SearchResult {
+ return &SearchResult{
Name: pkgGoCDK.Name,
PackagePath: pkgGoCDK.Path,
Synopsis: pkgGoCDK.Documentation[0].Synopsis,
@@ -618,7 +618,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) {
}
}
- offset = func(r *internal.SearchResult, n int) *internal.SearchResult {
+ offset = func(r *SearchResult, n int) *SearchResult {
r.Offset = n
return r
}
@@ -635,7 +635,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) {
packages map[string]*internal.Unit
limit, offset int
searchQuery string
- want []*internal.SearchResult
+ want []*SearchResult
}{
{
name: "two documents, single term search",
@@ -644,7 +644,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) {
modGoCDK: pkgGoCDK,
modKube: pkgKube,
},
- want: []*internal.SearchResult{
+ want: []*SearchResult{
goCDKResult(packageScore, 2),
offset(kubeResult(packageScore, 2), 1),
},
@@ -658,7 +658,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) {
modKube: pkgKube,
modGoCDK: pkgGoCDK,
},
- want: []*internal.SearchResult{
+ want: []*SearchResult{
goCDKResult(packageScore, 2),
},
},
@@ -671,7 +671,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) {
modGoCDK: pkgGoCDK,
modKube: pkgKube,
},
- want: []*internal.SearchResult{
+ want: []*SearchResult{
offset(kubeResult(packageScore, 2), 1),
},
},
@@ -682,7 +682,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) {
modGoCDK: pkgGoCDK,
modKube: pkgKube,
},
- want: []*internal.SearchResult{
+ want: []*SearchResult{
goCDKResult(goAndCDKScore, 1),
},
},
@@ -692,7 +692,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) {
packages: map[string]*internal.Unit{
modGoCDK: pkgGoCDK,
},
- want: []*internal.SearchResult{
+ want: []*SearchResult{
goCDKResult(cloudScore, 1),
},
},
@@ -726,7 +726,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) {
}
// The searchers differ in these two fields.
- opt := cmpopts.IgnoreFields(internal.SearchResult{}, "Approximate", "NumResults")
+ opt := cmpopts.IgnoreFields(SearchResult{}, "Approximate", "NumResults")
if diff := cmp.Diff(test.want, got.results, opt); diff != "" {
t.Errorf("testDB.Search(%v, %d, %d) mismatch (-want +got):\n%s", test.searchQuery, test.limit, test.offset, diff)
}
@@ -1353,7 +1353,7 @@ func TestDeleteOlderVersionFromSearch(t *testing.T) {
}
func TestGroupSearchResults(t *testing.T) {
- rs := []*internal.SearchResult{
+ rs := []*SearchResult{
{PackagePath: "m.com/p", ModulePath: "m.com", Score: 10},
{PackagePath: "m.com/p2", ModulePath: "m.com", Score: 8},
{PackagePath: "a.com/p", ModulePath: "a.com", Score: 7},
@@ -1361,23 +1361,23 @@ func TestGroupSearchResults(t *testing.T) {
{PackagePath: "m.com/v2/p2", ModulePath: "m.com/v2", Score: 4},
}
got := groupSearchResults(rs)
- sp2 := &internal.SearchResult{
+ sp2 := &SearchResult{
PackagePath: "m.com/p2",
ModulePath: "m.com",
Score: 8,
}
- sp := &internal.SearchResult{
+ sp := &SearchResult{
PackagePath: "m.com/p",
ModulePath: "m.com",
Score: 10,
- SameModule: []*internal.SearchResult{sp2},
+ SameModule: []*SearchResult{sp2},
}
- want := []*internal.SearchResult{
+ want := []*SearchResult{
{
PackagePath: "m.com/v2/p",
ModulePath: "m.com/v2",
Score: 10.00001,
- SameModule: []*internal.SearchResult{
+ SameModule: []*SearchResult{
{PackagePath: "m.com/v2/p2", ModulePath: "m.com/v2", Score: 4},
},
OtherMajor: map[string]bool{"m.com": true},
diff --git a/internal/postgres/symbolsearch.go b/internal/postgres/symbolsearch.go
index 5f6c6170..80ccecd5 100644
--- a/internal/postgres/symbolsearch.go
+++ b/internal/postgres/symbolsearch.go
@@ -141,9 +141,9 @@ func (db *DB) symbolSearch(ctx context.Context, q string, limit, offset, maxResu
LIMIT $2
OFFSET $3`, symbolScoreExpr)
- var results []*internal.SearchResult
+ var results []*SearchResult
collect := func(rows *sql.Rows) error {
- var r internal.SearchResult
+ var r SearchResult
if err := rows.Scan(
&r.PackagePath,
&r.ModulePath,
diff --git a/internal/postgres/symbolsearch_test.go b/internal/postgres/symbolsearch_test.go
index 39333f49..2e85aa11 100644
--- a/internal/postgres/symbolsearch_test.go
+++ b/internal/postgres/symbolsearch_test.go
@@ -25,11 +25,11 @@ func TestSymbolSearch(t *testing.T) {
m.Packages()[0].Documentation[0].API = sample.API
MustInsertModule(ctx, t, testDB, m)
- checkResult := func(metas ...internal.SymbolMeta) []*internal.SearchResult {
- var results []*internal.SearchResult
+ checkResult := func(metas ...internal.SymbolMeta) []*SearchResult {
+ var results []*SearchResult
for _, sm := range metas {
results = append(results,
- &internal.SearchResult{
+ &SearchResult{
Name: sample.PackageName,
PackagePath: sample.PackagePath,
ModulePath: sample.ModulePath,
@@ -50,7 +50,7 @@ func TestSymbolSearch(t *testing.T) {
for _, test := range []struct {
name string
q string
- want []*internal.SearchResult
+ want []*SearchResult
}{
{
name: "test search by <package>.<identifier>",