discovery: add handling for non-redistributable packages

This CL implements admittedly complicated logic for handling license
redistributability. It does this in three main steps, which could
conceivably be split into separate CLs if desirable.

1. Add an internal.Package.IsRedistributable method, which reports a
package as redistributable if and only if there is at least one license
permitting redistribution at the module root, and if every directory
containing a detected license file contains at least one license file
permitting redistribution. It is very possible that this logic is will
change in the future.

2. Add behavior in the postgres handler to prune package and version
data that should not be stored for non-redistributable content. As a
precaution it was decided to do this in the data layer rather than at
fetch time.

3. Add handling in the frontend to make it clear when content is not
redistributable.

Because we now have a scenario where we can't show license details, but
still want to surface the license disclaimer, move it to a separate
page.

Also:
 + modify license extraction to store the license path relative to the
   module@version subdirectory of the module zip.
 + fix a bug where VersionType was not set on in postgres.GetPackage.

Fixes b/127335320
Updates b/124309095

Change-Id: I36194b78651ccff91b0e06d0d3d4fc639a884565
Reviewed-on: https://team-review.git.corp.google.com/c/golang/discovery/+/455069
Reviewed-by: Julie Qiu <julieqiu@google.com>

1 files changed, 45 insertions, 7 deletions

diff --git a/internal/postgres/postgres.go b/internal/postgres/postgres.go
index e36c3174..30ea5383 100644
--- a/internal/postgres/postgres.go
+++ b/internal/postgres/postgres.go
@@ -298,10 +298,10 @@ func (db *DB) GetPackage(ctx context.Context, path string, version string) (*int
 	}
 
 	var (
-		commitTime                                                     time.Time
-		name, synopsis, seriesPath, modulePath, suffix, readmeFilePath string
-		readmeContents                                                 []byte
-		licenseTypes, licensePaths                                     []string
+		commitTime                                                                  time.Time
+		name, synopsis, seriesPath, modulePath, suffix, readmeFilePath, versionType string
+		readmeContents                                                              []byte
+		licenseTypes, licensePaths                                                  []string
 	)
 	query := `
 		SELECT
@@ -314,7 +314,8 @@ func (db *DB) GetPackage(ctx context.Context, path string, version string) (*int
 			v.module_path,
 			p.name,
 			p.synopsis,
-			p.suffix
+			p.suffix,
+			v.version_type
 		FROM
 			versions v
 		INNER JOIN
@@ -333,7 +334,8 @@ func (db *DB) GetPackage(ctx context.Context, path string, version string) (*int
 
 	row := db.QueryRowContext(ctx, query, path, version)
 	if err := row.Scan(&commitTime, pq.Array(&licenseTypes),
-		pq.Array(&licensePaths), &readmeFilePath, &readmeContents, &seriesPath, &modulePath, &name, &synopsis, &suffix); err != nil {
+		pq.Array(&licensePaths), &readmeFilePath, &readmeContents, &seriesPath, &modulePath,
+		&name, &synopsis, &suffix, &versionType); err != nil {
 		if err == sql.ErrNoRows {
 			return nil, derrors.NotFound(fmt.Sprintf("package %s@%s not found", path, version))
 		}
@@ -360,6 +362,7 @@ func (db *DB) GetPackage(ctx context.Context, path string, version string) (*int
 			CommitTime:     commitTime,
 			ReadmeFilePath: readmeFilePath,
 			ReadmeContents: readmeContents,
+			VersionType:    internal.VersionType(versionType),
 		},
 	}, nil
 }
@@ -797,6 +800,30 @@ func padPrerelease(v string) (string, error) {
 	return strings.Join(pre, "."), nil
 }
 
+// removeNonDistributableData removes any information from the version payload,
+// after checking licenses.
+func removeNonDistributableData(v *internal.Version) {
+	hasRedistributablePackage := false
+	for _, p := range v.Packages {
+		if p.IsRedistributable() {
+			hasRedistributablePackage = true
+		} else {
+			// Not redistributable, so prune information that can't be stored. In the
+			// future this should also include documentation.
+			p.Synopsis = ""
+		}
+	}
+
+	// If no packages are redistributable, we have no need for the readme
+	// contents, so drop them. Note that if a single package is redistributable,
+	// it must be true by definition that the module itself it redistributable,
+	// so capturing the README contents is OK.
+	if !hasRedistributablePackage {
+		v.ReadmeFilePath = ""
+		v.ReadmeContents = nil
+	}
+}
+
 // InsertVersion inserts a Version into the database along with any necessary
 // series, modules and packages. If any of these rows already exist, they will
 // not be updated. The version string is also parsed into major, minor, patch
@@ -812,6 +839,8 @@ func (db *DB) InsertVersion(ctx context.Context, version *internal.Version, lice
 		return derrors.InvalidArgument(fmt.Sprintf("validateVersion: %v", err))
 	}
 
+	removeNonDistributableData(version)
+
 	return db.Transact(func(tx *sql.Tx) error {
 		if _, err := tx.ExecContext(ctx,
 			`INSERT INTO series(path)
@@ -888,7 +917,15 @@ func (db *DB) InsertVersion(ctx context.Context, version *internal.Version, lice
 		var importValues []interface{}
 		var pkgLicenseValues []interface{}
 		for _, p := range version.Packages {
-			pkgValues = append(pkgValues, p.Path, p.Synopsis, p.Name, version.Version, version.ModulePath, p.Suffix)
+			pkgValues = append(pkgValues,
+				p.Path,
+				p.Synopsis,
+				p.Name,
+				version.Version,
+				version.ModulePath,
+				p.Suffix,
+				p.IsRedistributable(),
+			)
 
 			for _, l := range p.Licenses {
 				pkgLicenseValues = append(pkgLicenseValues, version.ModulePath, version.Version, l.FilePath, p.Path)
@@ -906,6 +943,7 @@ func (db *DB) InsertVersion(ctx context.Context, version *internal.Version, lice
 				"version",
 				"module_path",
 				"suffix",
+				"redistributable",
 			}
 			table := "packages"
 			if err := bulkInsert(ctx, tx, table, pkgCols, pkgValues, true); err != nil {