From 9695d5bffab0a61a8a77f6d0f0fb0fa7e9fa96a4 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Mon, 16 Nov 2020 05:23:58 -0500 Subject: internal/queue: use time-independent task IDs The task names we generated for Cloud Tasks included time information, so that we could control how freqeuently tasks were de-duped. We've been using a 3-hour window, meaning that if a module's task finished or was deleted, another task for that module couldn't be added to the queue (would be de-duped) for 3 hours. This scheme has the unintended consequence that if a module is still on the queue after 3 hours, then another task for the same module could be enqueued (since its task ID would be different). That means that if the queue ever gets 3 hours behind, it could get filled with duplicate modules. For example, this morning the queue had about 25,000 tasks, of which about 7,000 were duplicates. This CL sets a task's name to a function of the module path and version. That will prevent a task on the queue from being duplicated. A repeat task can be added about 1 hour after the previous task finishes or is deleted (see the Task De-duplication section of https://cloud.google.com/tasks/docs/reference/rpc/google.cloud.tasks.v2#task), which is fine. It is still possible to add a suffix to the task name to requeue sooner. Change-Id: I34ffce5ea67b9e00b88ca4cf38182e34a6ba8657 Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/270337 Trust: Jonathan Amsterdam Run-TryBot: Jonathan Amsterdam TryBot-Result: kokoro Reviewed-by: Julie Qiu --- internal/queue/queue_test.go | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) (limited to 'internal/queue/queue_test.go') diff --git a/internal/queue/queue_test.go b/internal/queue/queue_test.go index e713ae85..9f12af77 100644 --- a/internal/queue/queue_test.go +++ b/internal/queue/queue_test.go @@ -6,7 +6,6 @@ package queue import ( "testing" - "time" "github.com/golang/protobuf/ptypes" "github.com/google/go-cmp/cmp" @@ -16,22 +15,18 @@ import ( ) func TestNewTaskID(t *testing.T) { - // Verify that the task ID is the same within taskIDChangeInterval and changes - // afterwards. - var ( - module = "mod" - version = "ver" - taskIDChangeInterval = 3 * time.Hour - ) - tm := time.Now().Truncate(taskIDChangeInterval) - id1 := newTaskID(module, version, tm, taskIDChangeInterval) - id2 := newTaskID(module, version, tm.Add(taskIDChangeInterval/2), taskIDChangeInterval) - if id1 != id2 { - t.Error("wanted same task ID, got different") - } - id3 := newTaskID(module, version, tm.Add(taskIDChangeInterval+1), taskIDChangeInterval) - if id1 == id3 { - t.Error("wanted different task ID, got same") + for _, test := range []struct { + modulePath, version string + want string + }{ + {"m-1", "v2", "acc5-m-1_vv2"}, + {"my_module", "v1.2.3", "0cb9-my__module_vv1_o2_o3"}, + {"µπΩ/github.com", "v2.3.4-ß", "a49c-_00b5_03c0_03a9_-github_ocom_vv2_o3_o4-_00df"}, + } { + got := newTaskID(test.modulePath, test.version) + if got != test.want { + t.Errorf("%s@%s: got %s, want %s", test.modulePath, test.version, got, test.want) + } } } @@ -98,7 +93,7 @@ func TestNewTaskRequest(t *testing.T) { if err != nil { t.Fatal(err) } - got := gcp.newTaskRequest("mod", "v1.2.3", "suf", time.Minute) + got := gcp.newTaskRequest("mod", "v1.2.3", "suf") test.want.Task.Name = got.Task.Name if diff := cmp.Diff(test.want, got, cmp.Comparer(proto.Equal)); diff != "" { t.Errorf("mismatch (-want, +got):\n%s", diff) -- cgit v1.3-5-g45d5