internal/postgres,etc.: support bypassing license restrictions

Add a mode to postgres.DB where data is saved even if the module or
package is not redistributable.  This "bypassing license restrictions"
mode is off by default.

Add a flag to the worker binary to turn it on. Used a flag rather than
an environment variable so there is no way for ambient state to affect
the binary for this important decision.

Also:

- Add a test to verify that license data is removed/not removed
  depending on the bypass setting.

- Fix a bug where the new model insertion wasn't omitting
  documentation for non-redistributable modules.

- Fix a bug in internal/sample where the top-level readme wasn't
  getting populated.

A later CL will change the frontend to bypass the check.

For golang/go#39602
For golang/go#39629

Change-Id: I67a6d24c18f3b93cfbfc9ec2a20159c07a84e077
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/246957
Reviewed-by: Julie Qiu <julie@golang.org>
Run-TryBot: Julie Qiu <julie@golang.org>
TryBot-Result: kokoro <noreply+kokoro@google.com>

1 files changed, 64 insertions, 0 deletions

diff --git a/internal/postgres/insert_module_test.go b/internal/postgres/insert_module_test.go
index d4cb7438..db7b9c01 100644
--- a/internal/postgres/insert_module_test.go
+++ b/internal/postgres/insert_module_test.go
@@ -8,6 +8,7 @@ import (
 	"context"
 	"database/sql"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
 	"sync"
@@ -135,6 +136,69 @@ func checkModule(ctx context.Context, t *testing.T, want *internal.Module) {
 	}
 }
 
+func TestInsertModuleLicenseCheck(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+	defer cancel()
+
+	for _, bypass := range []bool{false, true} {
+		t.Run(fmt.Sprintf("bypass=%t", bypass), func(t *testing.T) {
+			defer ResetTestDB(testDB, t)
+			var db *DB
+			if bypass {
+				db = NewBypassingLicenseCheck(testDB.db)
+			} else {
+				db = testDB
+			}
+			checkHasRedistData := func(readme string, doc safehtml.HTML, want bool) {
+				t.Helper()
+				if got := readme != ""; got != want {
+					t.Errorf("readme: got %t, want %t", got, want)
+				}
+				if got := doc.String() != ""; got != want {
+					t.Errorf("doc: got %t, want %t", got, want)
+				}
+			}
+
+			mod := sample.Module(sample.ModulePath, sample.VersionString, "")
+			checkHasRedistData(mod.LegacyReadmeContents, mod.LegacyPackages[0].DocumentationHTML, true)
+			checkHasRedistData(mod.Directories[0].Readme.Contents, mod.Directories[0].Package.Documentation.HTML, true)
+			mod.IsRedistributable = false
+			mod.LegacyPackages[0].IsRedistributable = false
+			mod.Directories[0].IsRedistributable = false
+
+			if err := db.InsertModule(ctx, mod); err != nil {
+				t.Fatal(err)
+			}
+
+			// Legacy model
+			mi, err := db.LegacyGetModuleInfo(ctx, mod.ModulePath, mod.Version)
+			if err != nil {
+				t.Fatal(err)
+			}
+			pkg, err := db.LegacyGetPackage(ctx, mod.ModulePath, mod.ModulePath, mod.Version)
+			if err != nil {
+				t.Fatal(err)
+			}
+			checkHasRedistData(mi.LegacyReadmeContents, pkg.DocumentationHTML, bypass)
+
+			// New model
+			dir, err := db.GetDirectory(ctx, mod.ModulePath, mod.ModulePath, mod.Version)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var readme string
+			if dir.Readme != nil {
+				readme = dir.Readme.Contents
+			}
+			var doc safehtml.HTML
+			if dir.Package != nil && dir.Package.Documentation != nil {
+				doc = dir.Package.Documentation.HTML
+			}
+			checkHasRedistData(readme, doc, bypass)
+		})
+	}
+}
+
 func TestUpsertModule(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
 	defer cancel()