internal/postgres: add skip-symbols list to saveModule

- Modules in the skip-symbols list will not have symbol information inserted into postgres.

Change-Id: Ia52b9eb2d7cab662b967fa4a74c40cced0e44be4
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/713060
Reviewed-by: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Peter Weinberger <pjw@google.com>
kokoro-CI: kokoro <noreply+kokoro@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

2 files changed, 75 insertions, 2 deletions

diff --git a/internal/postgres/insert_module.go b/internal/postgres/insert_module.go
index a87f42d5..05e10375 100644
--- a/internal/postgres/insert_module.go
+++ b/internal/postgres/insert_module.go
@@ -61,6 +61,18 @@ func (db *DB) InsertModule(ctx context.Context, m *internal.Module, lmv *interna
 	return db.saveModule(ctx, m, lmv)
 }
 
+// TODO: Convert this to a txt file and investigate ahead of line blocking root cause.
+// If a module is removed from this list, consider backfilling the symbols for all
+// existing versions of that module.
+var skipSymbols = map[string]bool{
+	"github.com/citusdata/azure-sdk-for-go":                  true,
+	"github.com/seveas/azure-sdk-for-go":                     true,
+	"github.com/cdktf/cdktf-provider-azurerm-go/azurerm/v14": true,
+	"github.com/gardener/gardener":                           true,
+	"github.com/gyliu513/okd-origin":                         true,
+	"github.com/tombuildsstuff/azure-sdk-for-go":             true,
+}
+
 // saveModule inserts a Module into the database along with its packages,
 // imports, and licenses.  If any of these rows already exist, the module and
 // corresponding will be deleted and reinserted.
@@ -136,8 +148,10 @@ func (db *DB) saveModule(ctx context.Context, m *internal.Module, lmv *internal.
 			return err
 		}
 		isLatest = m.Version == latest
-		if err := insertSymbols(ctx, tx, m.ModulePath, m.Version, isLatest, pathToID, pathToUnitID, pathToDocs); err != nil {
-			return err
+		if !skipSymbols[m.ModulePath] {
+			if err := insertSymbols(ctx, tx, m.ModulePath, m.Version, isLatest, pathToID, pathToUnitID, pathToDocs); err != nil {
+				return err
+			}
 		}
 		if !isLatest {
 			return nil
diff --git a/internal/postgres/insert_module_test.go b/internal/postgres/insert_module_test.go
index 95df69bc..37dd8b7a 100644
--- a/internal/postgres/insert_module_test.go
+++ b/internal/postgres/insert_module_test.go
@@ -786,3 +786,62 @@ func TestPathsToInsert(t *testing.T) {
 		})
 	}
 }
+
+func TestInsertModuleSkipSymbols(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	checkSymbolsInserted := func(t *testing.T, db *DB, m *internal.Module) bool {
+		t.Helper()
+		var count int
+		err := db.db.QueryRow(ctx, `
+			SELECT count(*) FROM documentation_symbols ds
+			INNER JOIN documentation d ON d.id = ds.documentation_id
+			INNER JOIN units u ON u.id = d.unit_id
+			INNER JOIN modules mod ON mod.id = u.module_id
+			WHERE mod.module_path = $1 AND mod.version = $2`,
+			m.ModulePath, m.Version).Scan(&count)
+		if err != nil && err != sql.ErrNoRows {
+			t.Fatalf("error when checking for symbols: %v", err)
+		}
+		return count > 0
+	}
+
+	for _, test := range []struct {
+		name         string
+		modulePath   string
+		wantInserted bool
+	}{
+		{
+			name:         "skipped",
+			modulePath:   "github.com/gardener/gardener",
+			wantInserted: false,
+		},
+		{
+			name:         "not skipped",
+			modulePath:   "example.com/test-module",
+			wantInserted: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			testDB, release := acquire(t)
+			defer release()
+
+			m := sample.Module(test.modulePath, "v1.2.3", "pkg")
+			m.Units[1].Documentation[0].API = []*internal.Symbol{
+				{
+					SymbolMeta: internal.SymbolMeta{
+						Name:    "Foo",
+						Section: internal.SymbolSectionConstants,
+						Kind:    internal.SymbolKindVariable,
+					},
+				},
+			}
+			MustInsertModule(ctx, t, testDB, m)
+
+			if got := checkSymbolsInserted(t, testDB, m); got != test.wantInserted {
+				t.Errorf("symbols inserted: got %t; want %t", got, test.wantInserted)
+			}
+		})
+	}
+}