middleware: report error to GCP on experiment load failure

If experiments can't be loaded, send the error to GCP's error
reporting service. This can be used to alert that there is a
problem with the experiment configuration.

For golang/go#41483

Change-Id: Ia4fd9047dafd2ddd3d1bce46d1acd26ee00b7113
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/256238
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>

6 files changed, 21 insertions, 8 deletions

diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go
index e0c93b52..453cb315 100644
--- a/cmd/frontend/main.go
+++ b/cmd/frontend/main.go
@@ -176,12 +176,13 @@ func main() {
 		log.Fatal(ctx, err)
 	}
 	requestLogger := getLogger(ctx, cfg)
-	experimenter, err := middleware.NewExperimenter(ctx, 1*time.Minute, expg)
+	rc := reportingClient(ctx, cfg)
+	experimenter, err := middleware.NewExperimenter(ctx, 1*time.Minute, expg, rc)
 	if err != nil {
 		log.Fatal(ctx, err)
 	}
 	ermw := middleware.Identity()
-	if rc := reportingClient(ctx, cfg); rc != nil {
+	if rc != nil {
 		ermw = middleware.ErrorReporting(rc.Report)
 	}
 	mw := middleware.Chain(
diff --git a/cmd/worker/main.go b/cmd/worker/main.go
index 2f503249..c042366b 100644
--- a/cmd/worker/main.go
+++ b/cmd/worker/main.go
@@ -143,8 +143,7 @@ func main() {
 		log.Fatalf(ctx, "strconv.Atoi(%q): %v", timeout, err)
 	}
 	requestLogger := logger(ctx, cfg)
-
-	experimenter, err := middleware.NewExperimenter(ctx, 1*time.Minute, func(context.Context) internal.ExperimentSource { return db })
+	experimenter, err := middleware.NewExperimenter(ctx, 1*time.Minute, func(context.Context) internal.ExperimentSource { return db }, reportingClient)
 	if err != nil {
 		log.Fatal(ctx, err)
 	}
diff --git a/internal/frontend/server_test.go b/internal/frontend/server_test.go
index 808d5927..50c56b7c 100644
--- a/internal/frontend/server_test.go
+++ b/internal/frontend/server_test.go
@@ -1087,7 +1087,7 @@ func newTestServer(t *testing.T, proxyModules []*proxy.Module, experimentNames .
 		exps = append(exps, &internal.Experiment{Name: n, Rollout: 100})
 	}
 	esrc := internal.NewLocalExperimentSource(exps)
-	exp, err := middleware.NewExperimenter(ctx, time.Hour, func(context.Context) internal.ExperimentSource { return esrc })
+	exp, err := middleware.NewExperimenter(ctx, time.Hour, func(context.Context) internal.ExperimentSource { return esrc }, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/internal/middleware/experiment.go b/internal/middleware/experiment.go
index b57e74aa..ad39020e 100644
--- a/internal/middleware/experiment.go
+++ b/internal/middleware/experiment.go
@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"
 
+	"cloud.google.com/go/errorreporting"
 	"golang.org/x/pkgsite/internal"
 	"golang.org/x/pkgsite/internal/derrors"
 	"golang.org/x/pkgsite/internal/experiment"
@@ -21,10 +22,16 @@ import (
 
 const experimentQueryParamKey = "experiment"
 
+// A Reporter sends errors to the Error-Reporting service.
+type Reporter interface {
+	Report(errorreporting.Entry)
+}
+
 // An Experimenter contains information about active experiments from the
 // experiment source.
 type Experimenter struct {
 	getExperimentSource func(context.Context) internal.ExperimentSource
+	reporter            Reporter
 	pollEvery           time.Duration
 	mu                  sync.Mutex
 	snapshot            []*internal.Experiment
@@ -32,10 +39,11 @@ type Experimenter struct {
 
 // NewExperimenter returns an Experimenter for use in the middleware. The
 // experimenter regularly polls for updates to the snapshot in the background.
-func NewExperimenter(ctx context.Context, pollEvery time.Duration, esf func(context.Context) internal.ExperimentSource) (_ *Experimenter, err error) {
+func NewExperimenter(ctx context.Context, pollEvery time.Duration, esf func(context.Context) internal.ExperimentSource, rep Reporter) (_ *Experimenter, err error) {
 	defer derrors.Wrap(&err, "middleware.NewExperimenter")
 	e := &Experimenter{
 		getExperimentSource: esf,
+		reporter:            rep,
 		pollEvery:           pollEvery,
 	}
 	if err := e.loadNextSnapshot(ctx); err != nil {
@@ -86,6 +94,11 @@ func (e *Experimenter) pollUpdates(ctx context.Context) {
 			ctx2, cancel := context.WithTimeout(ctx, e.pollEvery)
 			if err := e.loadNextSnapshot(ctx2); err != nil {
 				log.Error(ctx, err)
+				if e.reporter != nil {
+					e.reporter.Report(errorreporting.Entry{
+						Error: fmt.Errorf("loading experiments: %v", err),
+					})
+				}
 			}
 			cancel()
 		}
diff --git a/internal/middleware/experiment_test.go b/internal/middleware/experiment_test.go
index 0a2f5287..d89f1af1 100644
--- a/internal/middleware/experiment_test.go
+++ b/internal/middleware/experiment_test.go
@@ -45,7 +45,7 @@ func TestSetAndLoadExperiments(t *testing.T) {
 			{Name: testFeature, Rollout: 100},
 		},
 	}
-	experimenter, err := NewExperimenter(ctx, 10*time.Millisecond, func(context.Context) internal.ExperimentSource { return source })
+	experimenter, err := NewExperimenter(ctx, 10*time.Millisecond, func(context.Context) internal.ExperimentSource { return source }, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/internal/testing/integration/frontend_test.go b/internal/testing/integration/frontend_test.go
index c4a8b678..fe9d9d0e 100644
--- a/internal/testing/integration/frontend_test.go
+++ b/internal/testing/integration/frontend_test.go
@@ -182,7 +182,7 @@ func setupFrontend(ctx context.Context, t *testing.T, q queue.Queue) *httptest.S
 	mux := http.NewServeMux()
 	s.Install(mux.Handle, nil, nil)
 
-	experimenter, err := middleware.NewExperimenter(ctx, 1*time.Minute, func(context.Context) internal.ExperimentSource { return testDB })
+	experimenter, err := middleware.NewExperimenter(ctx, 1*time.Minute, func(context.Context) internal.ExperimentSource { return testDB }, nil)
 	if err != nil {
 		t.Fatal(err)
 	}