internal/worker: add metric for new unprocessed modules

We have a metric for total unprocessed modules, including new ones and
those we are reprocessing.

Add a metric that tracks only the new ones: those with status 0 or 500
(the latter to count the number of failed new modules).

This metric is a better choice for an alert, since during reprocessing
we expect a very large total backlog, but we never want a large
backlog of new modules.

Change-Id: Ibc3cbee1d867f6a454748237352a70cf9eb500c0
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/326290
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>

1 files changed, 13 insertions, 6 deletions

diff --git a/internal/postgres/postgres.go b/internal/postgres/postgres.go
index 4e1dafe7..b8ef1925 100644
--- a/internal/postgres/postgres.go
+++ b/internal/postgres/postgres.go
@@ -113,15 +113,22 @@ func (db *DB) StalenessTimestamp(ctx context.Context) (time.Time, error) {
 }
 
 // NumUnprocessedModules returns the number of modules that need to be processed.
-func (db *DB) NumUnprocessedModules(ctx context.Context) (int, error) {
-	var n int
-	err := db.db.QueryRow(ctx, `
+func (db *DB) NumUnprocessedModules(ctx context.Context) (total, new int, err error) {
+	defer derrors.Wrap(&err, "NumUnprocessedModules()")
+
+	err = db.db.QueryRow(ctx, `
 		SELECT COUNT(*) FROM module_version_states WHERE status = 0 OR status >= 500
-	`).Scan(&n)
+	`).Scan(&total)
+	if err != nil {
+		return 0, 0, err
+	}
+	err = db.db.QueryRow(ctx, `
+		SELECT COUNT(*) FROM module_version_states WHERE status = 0 OR status = 500
+	`).Scan(&new)
 	if err != nil {
-		return 0, err
+		return 0, 0, err
 	}
-	return n, nil
+	return total, new, nil
 }
 
 // collectStrings runs the query, which must select for a single string column, and returns