diff options
| author | Jonathan Amsterdam <jba@google.com> | 2021-07-14 07:07:26 -0400 |
|---|---|---|
| committer | Jonathan Amsterdam <jba@google.com> | 2021-07-14 13:59:21 +0000 |
| commit | bcdc85aa4c7bb05e8b3ffe7eaef7e6f64bab4081 (patch) | |
| tree | fad66b5f8164f014f337b30972d58809ee1b035e /internal/postgres | |
| parent | ed966fad5adc568bac42880c0e5ae834f43c773f (diff) | |
| download | go-x-pkgsite-bcdc85aa4c7bb05e8b3ffe7eaef7e6f64bab4081.tar.xz | |
internal: move SearchResult to postgres package
Change-Id: I73f0c7c35a5221302c27012c9118fa3bed5fdd62
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/334529
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
Diffstat (limited to 'internal/postgres')
| -rw-r--r-- | internal/postgres/benchmarks_test.go | 3 | ||||
| -rw-r--r-- | internal/postgres/search.go | 76 | ||||
| -rw-r--r-- | internal/postgres/search_test.go | 36 | ||||
| -rw-r--r-- | internal/postgres/symbolsearch.go | 4 | ||||
| -rw-r--r-- | internal/postgres/symbolsearch_test.go | 8 |
5 files changed, 87 insertions, 40 deletions
diff --git a/internal/postgres/benchmarks_test.go b/internal/postgres/benchmarks_test.go index ca4531cb..57f2cf25 100644 --- a/internal/postgres/benchmarks_test.go +++ b/internal/postgres/benchmarks_test.go @@ -8,7 +8,6 @@ import ( "context" "testing" - "golang.org/x/pkgsite/internal" "golang.org/x/pkgsite/internal/config" "golang.org/x/pkgsite/internal/database" ) @@ -43,7 +42,7 @@ func BenchmarkSearch(b *testing.B) { b.Fatal(err) } db := New(ddb) - searchers := map[string]func(context.Context, string, SearchOptions) ([]*internal.SearchResult, error){ + searchers := map[string]func(context.Context, string, SearchOptions) ([]*SearchResult, error){ "db.Search": db.Search, } for name, search := range searchers { diff --git a/internal/postgres/search.go b/internal/postgres/search.go index d6330007..01227045 100644 --- a/internal/postgres/search.go +++ b/internal/postgres/search.go @@ -65,7 +65,7 @@ type searchResponse struct { // 'popular'), to be used in logging and reporting. source string // results are partially filled out from only the search_documents table. - results []*internal.SearchResult + results []*SearchResult // err indicates a technical failure of the search query, or that results are // not provably complete. err error @@ -109,6 +109,54 @@ type SearchOptions struct { SearchSymbols bool } +// SearchResult represents a single search result from SearchDocuments. +type SearchResult struct { + Name string + PackagePath string + ModulePath string + Version string + Synopsis string + Licenses []string + + CommitTime time.Time + + // Score is used to sort items in an array of SearchResult. + Score float64 + + // NumImportedBy is the number of packages that import PackagePath. + NumImportedBy uint64 + + // SameModule is a list of SearchResults from the same module as this one, + // with lower scores. + SameModule []*SearchResult + + // OtherMajor is a set of module paths with the same series path but at + // different major versions of this module. + OtherMajor map[string]bool + + // NumResults is the total number of packages that were returned for this + // search. + NumResults uint64 + + // Approximate reports whether NumResults is an approximate count. NumResults + // can be approximate if search scanned only a subset of documents, and + // result count is estimated using the hyperloglog algorithm. + Approximate bool + + // Symbol information returned by a search request. + // Only populated for symbol search mode. + SymbolName string + SymbolKind internal.SymbolKind + SymbolSynopsis string + SymbolGOOS string + SymbolGOARCH string + + // Offset is the 0-based number of this row in the DB query results, which + // is the value to use in a SQL OFFSET clause to have this row be the first + // one returned. + Offset int +} + // Search executes two search requests concurrently: // - a sequential scan of packages in descending order of popularity. // - all packages ("deep" search) using an inverted index to filter to search @@ -132,7 +180,7 @@ type SearchOptions struct { // The gap in this optimization is search terms that are very frequent, but // rarely relevant: "int" or "package", for example. In these cases we'll pay // the penalty of a deep search that scans nearly every package. -func (db *DB) Search(ctx context.Context, q string, opts SearchOptions) (_ []*internal.SearchResult, err error) { +func (db *DB) Search(ctx context.Context, q string, opts SearchOptions) (_ []*SearchResult, err error) { defer derrors.WrapStack(&err, "DB.Search(ctx, %q, %+v)", q, opts) if opts.Limit == 0 { opts.Limit = opts.MaxResults @@ -151,7 +199,7 @@ func (db *DB) Search(ctx context.Context, q string, opts SearchOptions) (_ []*in return nil, err } // Filter out excluded paths. - var results []*internal.SearchResult + var results []*SearchResult for _, r := range resp.results { ex, err := db.IsExcluded(ctx, r.PackagePath) if err != nil { @@ -286,7 +334,7 @@ func (db *DB) deepSearch(ctx context.Context, q string, limit, offset, maxResult OFFSET $3`, scoreExpr) var ( - results []*internal.SearchResult + results []*SearchResult err error ) if experiment.IsActive(ctx, internal.ExperimentSearchIncrementally) { @@ -294,7 +342,7 @@ func (db *DB) deepSearch(ctx context.Context, q string, limit, offset, maxResult const pageSize = 10 // TODO(jba): get from elsewhere additionalRows := 10 // after reaching pageSize module paths collect := func(rows *sql.Rows) error { - var r internal.SearchResult + var r SearchResult if err := rows.Scan(&r.PackagePath, &r.Version, &r.ModulePath, &r.CommitTime, &r.NumImportedBy, &r.Score, &r.NumResults); err != nil { return fmt.Errorf("rows.Scan(): %v", err) @@ -314,7 +362,7 @@ func (db *DB) deepSearch(ctx context.Context, q string, limit, offset, maxResult err = db.db.RunQueryIncrementally(ctx, query, fetchSize, collect, q, limit, offset) } else { collect := func(rows *sql.Rows) error { - var r internal.SearchResult + var r SearchResult if err := rows.Scan(&r.PackagePath, &r.Version, &r.ModulePath, &r.CommitTime, &r.NumImportedBy, &r.Score, &r.NumResults); err != nil { return fmt.Errorf("rows.Scan(): %v", err) @@ -352,9 +400,9 @@ func (db *DB) popularSearch(ctx context.Context, searchQuery string, limit, offs imported_by_count, score FROM popular_search($1, $2, $3, $4, $5)` - var results []*internal.SearchResult + var results []*SearchResult collect := func(rows *sql.Rows) error { - var r internal.SearchResult + var r SearchResult if err := rows.Scan(&r.PackagePath, &r.Version, &r.ModulePath, &r.CommitTime, &r.NumImportedBy, &r.Score); err != nil { return fmt.Errorf("rows.Scan(): %v", err) @@ -387,7 +435,7 @@ func (db *DB) popularSearch(ctx context.Context, searchQuery string, limit, offs // addPackageDataToSearchResults adds package information to SearchResults that is not stored // in the search_documents table. -func (db *DB) addPackageDataToSearchResults(ctx context.Context, results []*internal.SearchResult) (err error) { +func (db *DB) addPackageDataToSearchResults(ctx context.Context, results []*SearchResult) (err error) { defer derrors.WrapStack(&err, "DB.addPackageDataToSearchResults(results)") if len(results) == 0 { return nil @@ -396,7 +444,7 @@ func (db *DB) addPackageDataToSearchResults(ctx context.Context, results []*inte keys []string // resultMap tracks PackagePath->SearchResult, to allow joining with the // returned package data. - resultMap = make(map[string]*internal.SearchResult) + resultMap = make(map[string]*SearchResult) ) for _, r := range results { resultMap[r.PackagePath] = r @@ -476,10 +524,10 @@ func sortAndDedup(s []string) []string { // Packages from lower major versions of the module are grouped under the first // package of the highest major version. But they are not removed from the // top-level list. -func groupSearchResults(rs []*internal.SearchResult) []*internal.SearchResult { - modules := map[string]*internal.SearchResult{} // module path to first result - series := map[string]*internal.SearchResult{} // series path to result with max major version - var results []*internal.SearchResult +func groupSearchResults(rs []*SearchResult) []*SearchResult { + modules := map[string]*SearchResult{} // module path to first result + series := map[string]*SearchResult{} // series path to result with max major version + var results []*SearchResult for _, r := range rs { f := modules[r.ModulePath] if f == nil { diff --git a/internal/postgres/search_test.go b/internal/postgres/search_test.go index 83cf524e..91cb734b 100644 --- a/internal/postgres/search_test.go +++ b/internal/postgres/search_test.go @@ -590,8 +590,8 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) { }}, } - kubeResult = func(score float64, numResults uint64) *internal.SearchResult { - return &internal.SearchResult{ + kubeResult = func(score float64, numResults uint64) *SearchResult { + return &SearchResult{ Name: pkgKube.Name, PackagePath: pkgKube.Path, Synopsis: pkgKube.Documentation[0].Synopsis, @@ -604,8 +604,8 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) { } } - goCDKResult = func(score float64, numResults uint64) *internal.SearchResult { - return &internal.SearchResult{ + goCDKResult = func(score float64, numResults uint64) *SearchResult { + return &SearchResult{ Name: pkgGoCDK.Name, PackagePath: pkgGoCDK.Path, Synopsis: pkgGoCDK.Documentation[0].Synopsis, @@ -618,7 +618,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) { } } - offset = func(r *internal.SearchResult, n int) *internal.SearchResult { + offset = func(r *SearchResult, n int) *SearchResult { r.Offset = n return r } @@ -635,7 +635,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) { packages map[string]*internal.Unit limit, offset int searchQuery string - want []*internal.SearchResult + want []*SearchResult }{ { name: "two documents, single term search", @@ -644,7 +644,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) { modGoCDK: pkgGoCDK, modKube: pkgKube, }, - want: []*internal.SearchResult{ + want: []*SearchResult{ goCDKResult(packageScore, 2), offset(kubeResult(packageScore, 2), 1), }, @@ -658,7 +658,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) { modKube: pkgKube, modGoCDK: pkgGoCDK, }, - want: []*internal.SearchResult{ + want: []*SearchResult{ goCDKResult(packageScore, 2), }, }, @@ -671,7 +671,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) { modGoCDK: pkgGoCDK, modKube: pkgKube, }, - want: []*internal.SearchResult{ + want: []*SearchResult{ offset(kubeResult(packageScore, 2), 1), }, }, @@ -682,7 +682,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) { modGoCDK: pkgGoCDK, modKube: pkgKube, }, - want: []*internal.SearchResult{ + want: []*SearchResult{ goCDKResult(goAndCDKScore, 1), }, }, @@ -692,7 +692,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) { packages: map[string]*internal.Unit{ modGoCDK: pkgGoCDK, }, - want: []*internal.SearchResult{ + want: []*SearchResult{ goCDKResult(cloudScore, 1), }, }, @@ -726,7 +726,7 @@ func TestInsertSearchDocumentAndSearch(t *testing.T) { } // The searchers differ in these two fields. - opt := cmpopts.IgnoreFields(internal.SearchResult{}, "Approximate", "NumResults") + opt := cmpopts.IgnoreFields(SearchResult{}, "Approximate", "NumResults") if diff := cmp.Diff(test.want, got.results, opt); diff != "" { t.Errorf("testDB.Search(%v, %d, %d) mismatch (-want +got):\n%s", test.searchQuery, test.limit, test.offset, diff) } @@ -1353,7 +1353,7 @@ func TestDeleteOlderVersionFromSearch(t *testing.T) { } func TestGroupSearchResults(t *testing.T) { - rs := []*internal.SearchResult{ + rs := []*SearchResult{ {PackagePath: "m.com/p", ModulePath: "m.com", Score: 10}, {PackagePath: "m.com/p2", ModulePath: "m.com", Score: 8}, {PackagePath: "a.com/p", ModulePath: "a.com", Score: 7}, @@ -1361,23 +1361,23 @@ func TestGroupSearchResults(t *testing.T) { {PackagePath: "m.com/v2/p2", ModulePath: "m.com/v2", Score: 4}, } got := groupSearchResults(rs) - sp2 := &internal.SearchResult{ + sp2 := &SearchResult{ PackagePath: "m.com/p2", ModulePath: "m.com", Score: 8, } - sp := &internal.SearchResult{ + sp := &SearchResult{ PackagePath: "m.com/p", ModulePath: "m.com", Score: 10, - SameModule: []*internal.SearchResult{sp2}, + SameModule: []*SearchResult{sp2}, } - want := []*internal.SearchResult{ + want := []*SearchResult{ { PackagePath: "m.com/v2/p", ModulePath: "m.com/v2", Score: 10.00001, - SameModule: []*internal.SearchResult{ + SameModule: []*SearchResult{ {PackagePath: "m.com/v2/p2", ModulePath: "m.com/v2", Score: 4}, }, OtherMajor: map[string]bool{"m.com": true}, diff --git a/internal/postgres/symbolsearch.go b/internal/postgres/symbolsearch.go index 5f6c6170..80ccecd5 100644 --- a/internal/postgres/symbolsearch.go +++ b/internal/postgres/symbolsearch.go @@ -141,9 +141,9 @@ func (db *DB) symbolSearch(ctx context.Context, q string, limit, offset, maxResu LIMIT $2 OFFSET $3`, symbolScoreExpr) - var results []*internal.SearchResult + var results []*SearchResult collect := func(rows *sql.Rows) error { - var r internal.SearchResult + var r SearchResult if err := rows.Scan( &r.PackagePath, &r.ModulePath, diff --git a/internal/postgres/symbolsearch_test.go b/internal/postgres/symbolsearch_test.go index 39333f49..2e85aa11 100644 --- a/internal/postgres/symbolsearch_test.go +++ b/internal/postgres/symbolsearch_test.go @@ -25,11 +25,11 @@ func TestSymbolSearch(t *testing.T) { m.Packages()[0].Documentation[0].API = sample.API MustInsertModule(ctx, t, testDB, m) - checkResult := func(metas ...internal.SymbolMeta) []*internal.SearchResult { - var results []*internal.SearchResult + checkResult := func(metas ...internal.SymbolMeta) []*SearchResult { + var results []*SearchResult for _, sm := range metas { results = append(results, - &internal.SearchResult{ + &SearchResult{ Name: sample.PackageName, PackagePath: sample.PackagePath, ModulePath: sample.ModulePath, @@ -50,7 +50,7 @@ func TestSymbolSearch(t *testing.T) { for _, test := range []struct { name string q string - want []*internal.SearchResult + want []*SearchResult }{ { name: "test search by <package>.<identifier>", |
