diff options
| author | Jonathan Amsterdam <jba@google.com> | 2024-11-11 20:23:05 -0500 |
|---|---|---|
| committer | Jonathan Amsterdam <jba@google.com> | 2024-11-18 16:26:43 +0000 |
| commit | e07a3b7f84c28146f737f55fb2e100e603f7bfbc (patch) | |
| tree | edf95e9e39bb3074d306ba0b9abc9615f7e48e0d /cmd | |
| parent | d5b6a06e5ef0dadbf37df80f73c407550b8162b3 (diff) | |
| download | go-x-website-e07a3b7f84c28146f737f55fb2e100e603f7bfbc.tar.xz | |
cmd/screentest: replace caching with got/want model
Remove the '::caching' suffix from URLs. Instead of caching one
or both arguments to the filesystem or GCS bucket, a user can specify
a directory or bucket as one of the URL arguments. Most commonly,
the testURL will refer to a server and the wantURL will refer
to a set of "golden" files. The update flag will then overwrite
the goldens with the test results, or record them if there are
no goldens.
A useful side benefit is the ability to separate the location
of the goldens from the location where the output of failed tests is
written. For example, the former can be a directory and the latter
a GCS bucket.
This change also simplifies the code, since for the most part
locations, both test, golden and output, are treated uniformly.
There is still a distinction between servers and storage because
servers use a URL path to identify requires while storage uses a
path constructed from the test name, but reads and writes from
storage can be treated similarly.
Change-Id: I1136fa3ea8dbe3a3019594c260f6469cc7021891
Reviewed-on: https://go-review.googlesource.com/c/website/+/627118
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Hyang-Ah Hana Kim <hyangah@gmail.com>
Diffstat (limited to 'cmd')
| -rw-r--r-- | cmd/screentest/main.go | 30 | ||||
| -rw-r--r-- | cmd/screentest/screentest.go | 559 | ||||
| -rw-r--r-- | cmd/screentest/screentest_test.go | 343 | ||||
| -rw-r--r-- | cmd/screentest/testdata/screenshots/cached/homepage.a.png | bin | 155077 -> 0 bytes | |||
| -rw-r--r-- | cmd/screentest/testdata/screenshots/cached/homepage.b.png | bin | 155077 -> 0 bytes | |||
| -rw-r--r-- | cmd/screentest/testdata/screenshots/cached/homepage.want.png | bin | 0 -> 155117 bytes |
6 files changed, 435 insertions, 497 deletions
diff --git a/cmd/screentest/main.go b/cmd/screentest/main.go index 8536f6bf..8c145be2 100644 --- a/cmd/screentest/main.go +++ b/cmd/screentest/main.go @@ -4,15 +4,18 @@ /* Screentest compares images of rendered web pages. -It compares images obtained from two sources, one to test and one for the expected result. -The comparisons are driven by a script file in a format described below. +It compares images obtained from two sources, one to test and one for the expected +result. The comparisons are driven by a script file in a format described below. # Usage screentest [flags] testURL wantURL file ... -The first two arguments are the URLs being tested and the URL of the desired result, respectively. -The remaining arguments are script file pathnames. +The first two arguments are the URLs being tested and the URL of the desired result, +respectively. The remaining arguments are script file pathnames. + +The URLs can be actual URLs for http:, https:, file: or gs: schemes, or they +can be slash-separated file paths (even on Windows). The flags are: @@ -25,12 +28,17 @@ The flags are: HTTP(S) headers to send with each request, as a comma-separated list of name:value. -run REGEXP Run only tests matching regexp. - -o - URL or path for output files. If omitted, files are written to a subdirectory of the - user's cache directory. - -u - Update cached screenshots. - -v + -o + URL or slash-separated path for output files. If omitted, files are written + to a subdirectory of the user's cache directory. Each test file is given + its own directory, so test names in two files can be identical. But the directory + name is the basename of the test file with the extension removed. Conflicting + file names will overwrite each other. + -u + Instead of comparing screenshots, use the test screenshots to update the + want screenshots. This only makes sense if wantURL is a storage location + like a file path or GCS bucket. + -v Variables provided to script templates as comma separated KEY:VALUE pairs. # Scripts @@ -139,7 +147,7 @@ import ( var flags options func init() { - flag.BoolVar(&flags.update, "u", false, "update cached screenshots") + flag.BoolVar(&flags.update, "u", false, "update want with test") flag.StringVar(&flags.vars, "v", "", "variables provided to script templates as comma separated KEY:VALUE pairs") flag.IntVar(&flags.maxConcurrency, "c", (runtime.NumCPU()+1)/2, "number of test cases to run concurrently") flag.StringVar(&flags.debuggerURL, "d", "", "chrome debugger URL") diff --git a/cmd/screentest/screentest.go b/cmd/screentest/screentest.go index d5cf3e6c..fca3f717 100644 --- a/cmd/screentest/screentest.go +++ b/cmd/screentest/screentest.go @@ -5,6 +5,9 @@ // TODO(jba): sleep directive // TODO(jba): specify percent of image that may differ // TODO(jba): remove ints function in template (see cmd/golangorg/testdata/screentest/relnotes.txt) +// TODO(jba): update only tests matching a regexp +// TODO(jba): write index.html to outdir with a nice view of all the failures (and remember +// to clean it up) package main @@ -17,12 +20,11 @@ import ( "fmt" "image" "image/png" - "io" - "io/fs" "log" "net/http" "net/url" "os" + "path" "path/filepath" "regexp" "slices" @@ -37,7 +39,6 @@ import ( "github.com/chromedp/chromedp" "github.com/n7olkachev/imgdiff/pkg/imgdiff" "golang.org/x/sync/errgroup" - "google.golang.org/api/iterator" ) // run compares testURL and wantURL using the test scripts in files and the options in opts. @@ -77,9 +78,7 @@ func run(ctx context.Context, testURL, wantURL string, files []string, opts opti if len(tests) == 0 && opts.run == "" { return fmt.Errorf("no tests found in %q", file) } - if err := cleanOutput(ctx, tests); err != nil { - return fmt.Errorf("cleanOutput(...): %w", err) - } + // TODO(jba): clean output directory ctx, cancel = chromedp.NewContext(ctx, chromedp.WithLogf(log.Printf)) defer cancel() var hdr bool @@ -91,7 +90,7 @@ func run(ctx context.Context, testURL, wantURL string, files []string, opts opti hdr = true } fmt.Fprintf(&buf, "%v\n", err) - fmt.Fprintf(&buf, "inspect diff at %s\n\n", tc.outDiff) + fmt.Fprintf(&buf, "inspect diff at %s\n\n", path.Join(tc.failImageWriter.path(), tc.diffPath)) } fmt.Println(tc.output.String()) }) @@ -103,117 +102,9 @@ func run(ctx context.Context, testURL, wantURL string, files []string, opts opti return nil } -// cleanOutput clears the output locations of images not cached -// as part of a testcase, including diff output from previous test -// runs and obsolete screenshots. It ensures local directories exist -// for test output. GCS buckets must already exist prior to test run. -func cleanOutput(ctx context.Context, tests []*testcase) error { - keepFiles := make(map[string]bool) - bkts := make(map[string]bool) - dirs := make(map[string]bool) - // The extensions of files that are safe to delete - safeExts := map[string]bool{ - "a.png": true, - "b.png": true, - "diff.png": true, - } - for _, t := range tests { - if t.cacheA { - keepFiles[t.outImgA] = true - } - if t.cacheB { - keepFiles[t.outImgB] = true - } - if t.gcsBucket { - bkt, _ := gcsParts(t.outDiff) - bkts[bkt] = true - } else { - dirs[filepath.Dir(t.outDiff)] = true - } - } - if err := cleanBkts(ctx, bkts, keepFiles, safeExts); err != nil { - return fmt.Errorf("cleanBkts(...): %w", err) - } - if err := cleanDirs(dirs, keepFiles, safeExts); err != nil { - return fmt.Errorf("cleanDirs(...): %w", err) - } - return nil -} - -// cleanBkts clears all the GCS buckets in bkts of all objects not included -// in the set of keepFiles. Buckets that do not exist will cause an error. -func cleanBkts(ctx context.Context, bkts, keepFiles, safeExts map[string]bool) error { - if len(bkts) == 0 { - return nil - } - client, err := storage.NewClient(ctx) - if err != nil { - return fmt.Errorf("storage.NewClient(ctx): %w", err) - } - defer client.Close() - for bkt := range bkts { - it := client.Bucket(bkt).Objects(ctx, nil) - for { - attrs, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - return fmt.Errorf("it.Next(): %w", err) - } - filename := "gs://" + attrs.Bucket + "/" + attrs.Name - if !keepFiles[filename] && safeExts[ext(filename)] { - if err := client.Bucket(attrs.Bucket).Object(attrs.Name).Delete(ctx); err != nil && - !errors.Is(err, storage.ErrObjectNotExist) { - return fmt.Errorf("Object(%q).Delete: %v", attrs.Name, err) - } - } - } - } - return client.Close() -} - -// cleanDirs ensures the set of directories in dirs exists and -// clears dirs of all files not included in the set of keepFiles. -func cleanDirs(dirs, keepFiles, safeExts map[string]bool) error { - for dir := range dirs { - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return fmt.Errorf("os.MkdirAll(%q): %w", dir, err) - } - } - for dir := range dirs { - files, err := os.ReadDir(dir) - if err != nil { - return fmt.Errorf("os.ReadDir(%q): %w", dir, err) - } - for _, f := range files { - filename := dir + "/" + f.Name() - if !keepFiles[filename] && safeExts[ext(filename)] { - if err := os.Remove(filename); err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("os.Remove(%q): %w", filename, err) - } - } - } - } - return nil -} - -func ext(filename string) string { - // If the filename has multiple dots use the first one as - // the split for the extension. - if strings.Count(filename, ".") > 1 { - base := filepath.Base(filename) - parts := strings.SplitN(base, ".", 2) - return parts[1] - } - return filepath.Ext(filename) -} - const ( browserWidth = 1536 browserHeight = 960 - cacheSuffix = "::cache" - gcsScheme = "gs://" ) type screenshotType int @@ -225,20 +116,22 @@ const ( ) type testcase struct { - name string - tasks chromedp.Tasks - urlA, urlB string - headers map[string]any // to match chromedp arg - status int - cacheA, cacheB bool - gcsBucket bool - outImgA, outImgB, outDiff string - viewportWidth int - viewportHeight int - screenshotType screenshotType - screenshotElement string - blockedURLs []string - output bytes.Buffer + name string // name of the test (arg to 'test' directive) + tasks chromedp.Tasks + testURL, wantURL string // URL to visit if the command-line arg is http/https + testPath, wantPath string // slash-separated path to use if the command-line arg is file, gs or a path + diffPath string // output path for failed tests + headers map[string]any // to match chromedp arg + status int + viewportWidth int + viewportHeight int + screenshotType screenshotType + screenshotElement string + blockedURLs []string + output bytes.Buffer + testImageReader imageReader // read images for comparison or update + wantImageReadWriter imageReadWriter // read images for comparison, write them for update + failImageWriter imageWriter // write images for failed tests } func (t *testcase) String() string { @@ -247,6 +140,8 @@ func (t *testcase) String() string { // readTests parses the testcases from a text file. func readTests(file, testURL, wantURL string, opts options) ([]*testcase, error) { + ctx := context.Background() + tmpl := template.New(filepath.Base(file)).Funcs(template.FuncMap{ "ints": func(start, end int) []int { var out []int @@ -277,29 +172,28 @@ func readTests(file, testURL, wantURL string, opts options) ([]*testcase, error) tasks chromedp.Tasks originA, originB string status int = http.StatusOK - cacheA, cacheB bool - gcsBucket bool width, height int lineNo int blockedURLs []string ) - cache, err := os.UserCacheDir() + + // The test/want image readers/writers are relative to the test/want URLs, so + // they are common to all files. See test/wantPath for the file- and test-relative components. + // They may be nil if a URL has an http or https scheme. + testImageReader, err := newImageReadWriter(ctx, testURL) if err != nil { - return nil, fmt.Errorf("os.UserCacheDir(): %w", err) + return nil, err } - originA = testURL - if strings.HasSuffix(originA, cacheSuffix) { - originA = strings.TrimSuffix(originA, cacheSuffix) - cacheA = true + wantImageReadWriter, err := newImageReadWriter(ctx, wantURL) + if err != nil { + return nil, err } + + originA = testURL if _, err := url.Parse(originA); err != nil { return nil, fmt.Errorf("url.Parse(%q): %w", originA, err) } originB = wantURL - if strings.HasSuffix(originB, cacheSuffix) { - originB = strings.TrimSuffix(originB, cacheSuffix) - cacheB = true - } if _, err := url.Parse(originB); err != nil { return nil, fmt.Errorf("url.Parse(%q): %w", originB, err) } @@ -312,13 +206,21 @@ func readTests(file, testURL, wantURL string, opts options) ([]*testcase, error) for k, v := range hs { headers[k] = v } - dir := cmp.Or(opts.outputURL, filepath.Join(cache, "screentest")) - out, err := outDir(dir, file) + + outPath := opts.outputURL + if outPath == "" { + cache, err := os.UserCacheDir() + if err != nil { + return nil, fmt.Errorf("os.UserCacheDir(): %w", err) + } + outPath = path.Join(filepath.ToSlash(cache), "screentest") + } + failImageWriter, err := newImageReadWriter(ctx, outPath) if err != nil { return nil, err } - if strings.HasPrefix(out, gcsScheme) { - gcsBucket = true + if failImageWriter == nil { + return nil, fmt.Errorf("cannot write images to %q", outPath) } filter := func(string) bool { return true } @@ -395,13 +297,21 @@ func readTests(file, testURL, wantURL string, opts options) ([]*testcase, error) if pathname == "" { return nil, fmt.Errorf("missing pathname for capture on line %d", lineNo) } - urlA, err := url.Parse(originA + pathname) - if err != nil { - return nil, fmt.Errorf("url.Parse(%q): %w", originA+pathname, err) + var urlA, urlB string + if testImageReader == nil { + u, err := url.Parse(originA + pathname) + if err != nil { + return nil, fmt.Errorf("url.Parse(%q): %w", originA+pathname, err) + } + urlA = u.String() + } - urlB, err := url.Parse(originB + pathname) - if err != nil { - return nil, fmt.Errorf("url.Parse(%q): %w", originB+pathname, err) + if wantImageReadWriter == nil { + u, err := url.Parse(originB + pathname) + if err != nil { + return nil, fmt.Errorf("url.Parse(%q): %w", originB+pathname, err) + } + urlB = u.String() } if !filter(testName) { continue @@ -409,18 +319,18 @@ func readTests(file, testURL, wantURL string, opts options) ([]*testcase, error) test := &testcase{ name: testName, tasks: tasks, - urlA: urlA.String(), - urlB: urlB.String(), + testURL: urlA, + wantURL: urlB, headers: headers, status: status, blockedURLs: blockedURLs, // Default to viewportScreenshot - screenshotType: viewportScreenshot, - viewportWidth: width, - viewportHeight: height, - cacheA: cacheA, - cacheB: cacheB, - gcsBucket: gcsBucket, + screenshotType: viewportScreenshot, + viewportWidth: width, + viewportHeight: height, + failImageWriter: failImageWriter, + testImageReader: testImageReader, + wantImageReadWriter: wantImageReadWriter, } tests = append(tests, test) field, args := splitOneField(args) @@ -448,13 +358,13 @@ func readTests(file, testURL, wantURL string, opts options) ([]*testcase, error) default: return nil, fmt.Errorf("first argument to capture must be 'fullscreen', 'viewport' or 'element'") } - outfile := filepath.Join(out, sanitize(test.name)) - if gcsBucket { - outfile, err = url.JoinPath(out, sanitize(test.name)) - } - test.outImgA = outfile + ".a.png" - test.outImgB = outfile + ".b.png" - test.outDiff = outfile + ".diff.png" + + fileBase := path.Base(filepath.ToSlash(file)) + filePath := strings.TrimSuffix(fileBase, path.Ext(fileBase)) + fnPath := path.Join(filePath, sanitize(test.name)) + test.testPath = fnPath + ".got.png" + test.wantPath = fnPath + ".want.png" + test.diffPath = fnPath + ".diff.png" default: // We should never reach this error. return nil, fmt.Errorf("invalid syntax on line %d: %q", lineNo, line) @@ -487,17 +397,6 @@ func splitList(s string) (map[string]string, error) { return m, nil } -// outDir gets a diff output directory for a given testfile. -// If dir points to a GCS bucket or testfile is empty it just -// returns dir. -func outDir(dir, testfile string) (string, error) { - tf := sanitize(filepath.Base(testfile)) - if strings.HasPrefix(dir, gcsScheme) { - return url.JoinPath(dir, tf) - } - return filepath.Clean(filepath.Join(dir, tf)), nil -} - // splitOneField splits text at the first space or tab // and returns that first field and the remaining text. func splitOneField(text string) (field, rest string) { @@ -540,37 +439,41 @@ func splitDimensions(text string) (width, height int, err error) { func (tc *testcase) run(ctx context.Context, update bool) (err error) { now := time.Now() fmt.Fprintf(&tc.output, "test %s ", tc.name) - var screenA, screenB *image.Image + var testScreen, wantScreen image.Image g, ctx := errgroup.WithContext(ctx) // If the hosts are the same, chrome (or chromedp) does not handle concurrent requests well. // This wouldn't make sense in an actual test, but it does happen in this package's tests. - urla, erra := url.Parse(tc.urlA) - urlb, errb := url.Parse(tc.urlA) + urla, erra := url.Parse(tc.testURL) + urlb, errb := url.Parse(tc.wantURL) if err := cmp.Or(erra, errb); err != nil { return err } if urla.Host == urlb.Host { g.SetLimit(1) } + g.Go(func() error { - screenA, err = tc.screenshot(ctx, tc.urlA, tc.outImgA, tc.cacheA, update) - if err != nil { - return fmt.Errorf("screenshot(ctx, %q, %q, %q, %v): %w", tc, tc.urlA, tc.outImgA, tc.cacheA, err) - } - return nil - }) - g.Go(func() error { - screenB, err = tc.screenshot(ctx, tc.urlB, tc.outImgB, tc.cacheB, update) - if err != nil { - return fmt.Errorf("screenshot(ctx, %q, %q, %q, %v): %w", tc, tc.urlB, tc.outImgB, tc.cacheB, err) - } - return nil + testScreen, err = tc.screenshot(ctx, tc.testURL, tc.testPath, tc.testImageReader) + return err }) + if !update { + g.Go(func() error { + wantScreen, err = tc.screenshot(ctx, tc.wantURL, tc.wantPath, tc.wantImageReadWriter) + return err + }) + } if err := g.Wait(); err != nil { fmt.Fprint(&tc.output, "\n", err) return err } - result := imgdiff.Diff(*screenA, *screenB, &imgdiff.Options{ + + // Update means overwrite the golden with the test result. + if update { + fmt.Fprintf(&tc.output, "updating %s\n", tc.wantURL) + return tc.wantImageReadWriter.writeImage(ctx, tc.wantPath, testScreen) + } + + result := imgdiff.Diff(testScreen, wantScreen, &imgdiff.Options{ Threshold: 0.1, DiffImage: true, }) @@ -579,81 +482,31 @@ func (tc *testcase) run(ctx context.Context, update bool) (err error) { fmt.Fprintf(&tc.output, "(%s)\n", since) return nil } - fmt.Fprintf(&tc.output, "(%s)\nFAIL %s != %s (%d pixels differ)\n", since, tc.urlA, tc.urlB, result.DiffPixelsCount) - g = &errgroup.Group{} - g.Go(func() error { - return writePNG(&result.Image, tc.outDiff) - }) - // Only write screenshots if they haven't already been written to the cache. - if !tc.cacheA { - g.Go(func() error { - return writePNG(screenA, tc.outImgA) - }) - } - if !tc.cacheB { - g.Go(func() error { - return writePNG(screenB, tc.outImgB) - }) - } + fmt.Fprintf(&tc.output, "(%s)\nFAIL %s != %s (%d pixels differ)\n", since, tc.testURL, tc.wantURL, result.DiffPixelsCount) + g, gctx := errgroup.WithContext(ctx) + g.Go(func() error { return tc.failImageWriter.writeImage(gctx, tc.testPath, testScreen) }) + g.Go(func() error { return tc.failImageWriter.writeImage(gctx, tc.wantPath, wantScreen) }) + g.Go(func() error { return tc.failImageWriter.writeImage(gctx, tc.diffPath, result.Image) }) if err := g.Wait(); err != nil { - return fmt.Errorf("writePNG(...): %w", err) + return err } - fmt.Fprintf(&tc.output, "wrote diff to %s\n", tc.outDiff) - return fmt.Errorf("%s != %s", tc.urlA, tc.urlB) + fmt.Fprintf(&tc.output, "wrote diff to %s\n", path.Join(tc.failImageWriter.path(), tc.diffPath)) + return fmt.Errorf("%s != %s", tc.testURL, tc.wantURL) } -// screenshot gets a screenshot for a testcase url. When cache is true it will -// attempt to read the screenshot from a cache or capture a new screenshot -// and write it to the cache if it does not exist. -func (tc *testcase) screenshot(ctx context.Context, url, file string, - cache, update bool, -) (_ *image.Image, err error) { - var data []byte - // If cache is enabled, try to read the file from the cache. - if cache && tc.gcsBucket { - client, err := storage.NewClient(ctx) - if err != nil { - return nil, fmt.Errorf("storage.NewClient(err): %w", err) - } - defer client.Close() - bkt, obj := gcsParts(file) - r, err := client.Bucket(bkt).Object(obj).NewReader(ctx) - if err != nil && !errors.Is(err, storage.ErrObjectNotExist) { - return nil, fmt.Errorf("object.NewReader(ctx): %w", err) - } else if err == nil { - defer r.Close() - data, err = io.ReadAll(r) - if err != nil { - return nil, fmt.Errorf("io.ReadAll(...): %w", err) - } - } - } else if cache { - data, err = os.ReadFile(file) - if err != nil && !errors.Is(err, fs.ErrNotExist) { - return nil, fmt.Errorf("os.ReadFile(...): %w", err) - } - } - // If cache is false, an update is requested, or this is the first test run - // we capture a new screenshot from a live URL. - if !cache || update || data == nil { - update = true - data, err = tc.captureScreenshot(ctx, url) +// screenshot gets a screenshot for a testcase url. If reader is non-nil +// it reads the pathname from reader. Otherwise it captures a new screenshot from url. +func (tc *testcase) screenshot(ctx context.Context, url, pathname string, reader imageReader) (image.Image, error) { + if reader != nil { + return reader.readImage(ctx, pathname) + } else { + data, err := tc.captureScreenshot(ctx, url) if err != nil { return nil, fmt.Errorf("captureScreenshot(ctx, %q, %q): %w", url, tc, err) } + img, _, err := image.Decode(bytes.NewReader(data)) + return img, err } - img, _, err := image.Decode(bytes.NewReader(data)) - if err != nil { - return nil, fmt.Errorf("image.Decode(...): %w", err) - } - // Write to the cache. - if cache && update { - if err := writePNG(&img, file); err != nil { - return nil, fmt.Errorf("os.WriteFile(...): %w", err) - } - fmt.Fprintf(&tc.output, "updated %s\n", file) - } - return &img, nil } type response struct { @@ -722,36 +575,6 @@ func reduceMotion() chromedp.Action { return chromedp.Evaluate(script, nil) } -// writePNG writes image data to a png file. -func writePNG(i *image.Image, filename string) error { - var f io.WriteCloser - if strings.HasPrefix(filename, gcsScheme) { - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - return fmt.Errorf("storage.NewClient(ctx): %w", err) - } - defer client.Close() - bkt, obj := gcsParts(filename) - f = client.Bucket(bkt).Object(obj).NewWriter(ctx) - } else { - var err error - f, err = os.Create(filename) - if err != nil { - return err - } - } - if err := png.Encode(f, *i); err != nil { - // Ignore f.Close() error, since png.Encode returned an error. - _ = f.Close() - return fmt.Errorf("png.Encode(...): %w", err) - } - if err := f.Close(); err != nil { - return fmt.Errorf("f.Close(): %w", err) - } - return nil -} - var sanitizeRegexp = regexp.MustCompile("[.*<>?`'|/\\: ]") // sanitize transforms text into a string suitable for use in a @@ -760,16 +583,6 @@ func sanitize(text string) string { return sanitizeRegexp.ReplaceAllString(text, "-") } -// gcsParts splits a Cloud Storage filename into bucket name and -// object name parts. -func gcsParts(filename string) (bucket, object string) { - filename = strings.TrimPrefix(filename, gcsScheme) - n := strings.Index(filename, "/") - bucket = filename[:n] - object = filename[n+1:] - return bucket, object -} - // waitForEvent waits for browser lifecycle events. This is useful for // ensuring the page is fully loaded before capturing screenshots. func waitForEvent(eventName string) chromedp.ActionFunc { @@ -827,6 +640,145 @@ func checkResponse(tc *testcase, res *response) chromedp.ActionFunc { } } +// An imageReader reads images from slash-separated paths. +type imageReader interface { + readImage(ctx context.Context, path string) (image.Image, error) // get an image with the given name +} + +// An imageWriter writes images to slash-separated paths. +type imageWriter interface { + writeImage(ctx context.Context, path string, img image.Image) error + path() string // return the slash-separated path that this was created with +} + +type imageReadWriter interface { + imageReader + imageWriter +} + +var validSchemes = []string{"file", "gs", "http", "https"} + +// newImageReadWriter returns an imageReadWriter for loc. +// loc can be a URL with a scheme or a slash-separated file path. +func newImageReadWriter(ctx context.Context, loc string) (imageReadWriter, error) { + scheme, _, _ := strings.Cut(loc, ":") + scheme = strings.ToLower(scheme) + switch scheme { + case "http", "https": + return nil, nil + case "file", "gs": + u, err := url.Parse(loc) + if err != nil { + return nil, err + } + if scheme == "file" { + return &dirImageReadWriter{dir: path.Clean(u.Path)}, nil + } + return newGCSImageReadWriter(ctx, loc) + default: + // Assume a file path; Windows paths can start with a drive letter. + return &dirImageReadWriter{dir: path.Clean(loc)}, nil + } +} + +// A dirImageReadWriter reads and writes images to a filesystem directory. +// dir should be slash-separated. +type dirImageReadWriter struct { + dir string +} + +func (rw *dirImageReadWriter) readImage(_ context.Context, path string) (_ image.Image, err error) { + path = rw.nativePathname(path) + defer wrapf(&err, "reading image from %s", path) + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + img, _, err := image.Decode(f) + return img, err +} + +func (rw *dirImageReadWriter) writeImage(_ context.Context, path string, img image.Image) (err error) { + path = rw.nativePathname(path) + defer wrapf(&err, "writing %s", path) + + if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { + return err + } + f, err := os.Create(path) + if err != nil { + return err + } + defer func() { + err = errors.Join(err, f.Close()) + }() + return png.Encode(f, img) +} + +func (rw *dirImageReadWriter) nativePathname(pth string) string { + spath := path.Join(rw.dir, pth) + return filepath.FromSlash(spath) +} + +func (rw *dirImageReadWriter) path() string { + return rw.dir +} + +type gcsImageReadWriter struct { + url string // URL with scheme "gs" referring to bucket and prefix + bucket *storage.BucketHandle + prefix string // initial path of objects; effectively a directory +} + +func newGCSImageReadWriter(ctx context.Context, urlstr string) (*gcsImageReadWriter, error) { + c, err := storage.NewClient(ctx) + if err != nil { + return nil, err + } + u, err := url.Parse(urlstr) + if err != nil { + return nil, err + } + return &gcsImageReadWriter{ + url: urlstr, + bucket: c.Bucket(u.Host), + prefix: u.Path[1:], //remove initial slash + }, nil +} + +func (rw *gcsImageReadWriter) readImage(ctx context.Context, pth string) (_ image.Image, err error) { + defer wrapf(&err, "reading %s", path.Join(rw.url, pth)) + + r, err := rw.bucket.Object(rw.objectName(pth)).NewReader(ctx) + if err != nil { + return nil, err + } + defer r.Close() + img, _, err := image.Decode(r) + return img, err +} + +func (rw *gcsImageReadWriter) writeImage(ctx context.Context, pth string, img image.Image) (err error) { + defer wrapf(&err, "writing %s", path.Join(rw.url, pth)) + + cctx, cancel := context.WithCancel(ctx) + defer cancel() + w := rw.bucket.Object(rw.objectName(pth)).NewWriter(cctx) + if err := png.Encode(w, img); err != nil { + cancel() + _ = w.Close() + return err + } + return w.Close() +} + +func (rw *gcsImageReadWriter) path() string { return rw.url } + +func (rw *gcsImageReadWriter) objectName(pth string) string { + return path.Join(rw.prefix, sanitize(pth)) +} + // runConcurrently calls f on each integer from 0 to n-1, // with at most max invocations active at once. // It waits for all invocations to complete. @@ -845,3 +797,10 @@ func runConcurrently(n, max int, f func(int)) { tokens <- struct{}{} } } + +// wrapf prepends a non-nil *errp with the given message, formatted by fmt.Sprintf. +func wrapf(errp *error, format string, args ...any) { + if *errp != nil { + *errp = fmt.Errorf("%s: %w", fmt.Sprintf(format, args...), *errp) + } +} diff --git a/cmd/screentest/screentest_test.go b/cmd/screentest/screentest_test.go index 2a096a6d..6fc2a17f 100644 --- a/cmd/screentest/screentest_test.go +++ b/cmd/screentest/screentest_test.go @@ -6,7 +6,9 @@ package main import ( "context" + "flag" "fmt" + "image" "net/http" "os" "os/exec" @@ -16,18 +18,16 @@ import ( "github.com/chromedp/chromedp" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/n7olkachev/imgdiff/pkg/imgdiff" ) +var bucketPath = flag.String("bucketpath", "", "bucket/prefix to test GCS I/O") + func TestReadTests(t *testing.T) { type args struct { testURL, wantURL string filename string } - d, err := os.UserCacheDir() - if err != nil { - t.Errorf("os.UserCacheDir(): %v", err) - } - cache := filepath.Join(d, "screentest") tests := []struct { name string args args @@ -48,12 +48,12 @@ func TestReadTests(t *testing.T) { want: []*testcase{ { name: "go.dev homepage", - urlA: "https://go.dev/", - urlB: "http://localhost:6060/", + testURL: "https://go.dev/", + wantURL: "http://localhost:6060/", status: 200, - outImgA: filepath.Join(cache, "readtests-txt", "go-dev-homepage.a.png"), - outImgB: filepath.Join(cache, "readtests-txt", "go-dev-homepage.b.png"), - outDiff: filepath.Join(cache, "readtests-txt", "go-dev-homepage.diff.png"), + testPath: "readtests/go-dev-homepage.got.png", + wantPath: "readtests/go-dev-homepage.want.png", + diffPath: "readtests/go-dev-homepage.diff.png", viewportWidth: 1536, viewportHeight: 960, screenshotType: fullScreenshot, @@ -61,12 +61,12 @@ func TestReadTests(t *testing.T) { }, { name: "go.dev homepage 540x1080", - urlA: "https://go.dev/", - urlB: "http://localhost:6060/", + testURL: "https://go.dev/", + wantURL: "http://localhost:6060/", status: 200, - outImgA: filepath.Join(cache, "readtests-txt", "go-dev-homepage-540x1080.a.png"), - outImgB: filepath.Join(cache, "readtests-txt", "go-dev-homepage-540x1080.b.png"), - outDiff: filepath.Join(cache, "readtests-txt", "go-dev-homepage-540x1080.diff.png"), + testPath: "readtests/go-dev-homepage-540x1080.got.png", + wantPath: "readtests/go-dev-homepage-540x1080.want.png", + diffPath: "readtests/go-dev-homepage-540x1080.diff.png", viewportWidth: 540, viewportHeight: 1080, screenshotType: fullScreenshot, @@ -74,12 +74,12 @@ func TestReadTests(t *testing.T) { }, { name: "about page", - urlA: "https://go.dev/about", - urlB: "http://localhost:6060/about", + testURL: "https://go.dev/about", + wantURL: "http://localhost:6060/about", status: 200, - outImgA: filepath.Join(cache, "readtests-txt", "about-page.a.png"), - outImgB: filepath.Join(cache, "readtests-txt", "about-page.b.png"), - outDiff: filepath.Join(cache, "readtests-txt", "about-page.diff.png"), + testPath: "readtests/about-page.got.png", + wantPath: "readtests/about-page.want.png", + diffPath: "readtests/about-page.diff.png", screenshotType: fullScreenshot, viewportWidth: 1536, viewportHeight: 960, @@ -87,12 +87,12 @@ func TestReadTests(t *testing.T) { }, { name: "homepage element .go-Carousel", - urlA: "https://go.dev/", - urlB: "http://localhost:6060/", + testURL: "https://go.dev/", + wantURL: "http://localhost:6060/", status: 200, - outImgA: filepath.Join(cache, "readtests-txt", "homepage-element--go-Carousel.a.png"), - outImgB: filepath.Join(cache, "readtests-txt", "homepage-element--go-Carousel.b.png"), - outDiff: filepath.Join(cache, "readtests-txt", "homepage-element--go-Carousel.diff.png"), + testPath: "readtests/homepage-element--go-Carousel.got.png", + wantPath: "readtests/homepage-element--go-Carousel.want.png", + diffPath: "readtests/homepage-element--go-Carousel.diff.png", screenshotType: elementScreenshot, screenshotElement: ".go-Carousel", viewportWidth: 1536, @@ -104,12 +104,12 @@ func TestReadTests(t *testing.T) { }, { name: "net package doc", - urlA: "https://go.dev/net", - urlB: "http://localhost:6060/net", + testURL: "https://go.dev/net", + wantURL: "http://localhost:6060/net", status: 200, - outImgA: filepath.Join(cache, "readtests-txt", "net-package-doc.a.png"), - outImgB: filepath.Join(cache, "readtests-txt", "net-package-doc.b.png"), - outDiff: filepath.Join(cache, "readtests-txt", "net-package-doc.diff.png"), + testPath: "readtests/net-package-doc.got.png", + wantPath: "readtests/net-package-doc.want.png", + diffPath: "readtests/net-package-doc.diff.png", screenshotType: viewportScreenshot, viewportWidth: 1536, viewportHeight: 960, @@ -120,12 +120,12 @@ func TestReadTests(t *testing.T) { }, { name: "net package doc 540x1080", - urlA: "https://go.dev/net", - urlB: "http://localhost:6060/net", + testURL: "https://go.dev/net", + wantURL: "http://localhost:6060/net", status: 200, - outImgA: filepath.Join(cache, "readtests-txt", "net-package-doc-540x1080.a.png"), - outImgB: filepath.Join(cache, "readtests-txt", "net-package-doc-540x1080.b.png"), - outDiff: filepath.Join(cache, "readtests-txt", "net-package-doc-540x1080.diff.png"), + testPath: "readtests/net-package-doc-540x1080.got.png", + wantPath: "readtests/net-package-doc-540x1080.want.png", + diffPath: "readtests/net-package-doc-540x1080.diff.png", screenshotType: viewportScreenshot, viewportWidth: 540, viewportHeight: 1080, @@ -140,7 +140,7 @@ func TestReadTests(t *testing.T) { { name: "readtests2", args: args{ - testURL: "https://pkg.go.dev::cache", + testURL: "some/directory", wantURL: "http://localhost:8080", filename: "testdata/readtests2.txt", }, @@ -150,37 +150,33 @@ func TestReadTests(t *testing.T) { }, want: []*testcase{ { - name: "about", - urlA: "https://pkg.go.dev/about", - cacheA: true, - urlB: "http://localhost:8080/about", - headers: map[string]any{"Authorization": "Bearer token"}, - status: 200, - gcsBucket: true, - outImgA: "gs://bucket/prefix/readtests2-txt/about.a.png", - outImgB: "gs://bucket/prefix/readtests2-txt/about.b.png", - outDiff: "gs://bucket/prefix/readtests2-txt/about.diff.png", - screenshotType: viewportScreenshot, - viewportWidth: 100, - viewportHeight: 200, + name: "about", + wantURL: "http://localhost:8080/about", + headers: map[string]any{"Authorization": "Bearer token"}, + status: 200, + testPath: "readtests2/about.got.png", + wantPath: "readtests2/about.want.png", + diffPath: "readtests2/about.diff.png", + screenshotType: viewportScreenshot, + viewportWidth: 100, + viewportHeight: 200, + testImageReader: &dirImageReadWriter{dir: "some/directory"}, }, { name: "eval", - urlA: "https://pkg.go.dev/eval", - cacheA: true, - urlB: "http://localhost:8080/eval", + wantURL: "http://localhost:8080/eval", headers: map[string]interface{}{"Authorization": "Bearer token"}, status: 200, - gcsBucket: true, - outImgA: "gs://bucket/prefix/readtests2-txt/eval.a.png", - outImgB: "gs://bucket/prefix/readtests2-txt/eval.b.png", - outDiff: "gs://bucket/prefix/readtests2-txt/eval.diff.png", + testPath: "readtests2/eval.got.png", + wantPath: "readtests2/eval.want.png", + diffPath: "readtests2/eval.diff.png", screenshotType: viewportScreenshot, viewportWidth: 100, viewportHeight: 200, tasks: chromedp.Tasks{ chromedp.Evaluate("console.log('Hello, world!')", nil), }, + testImageReader: &dirImageReadWriter{dir: "some/directory"}, }, }, }, @@ -194,9 +190,10 @@ func TestReadTests(t *testing.T) { } if diff := cmp.Diff(tt.want, got, cmp.AllowUnexported(testcase{}), - cmpopts.IgnoreFields(testcase{}, "output", "tasks"), + cmpopts.IgnoreFields(testcase{}, "output", "tasks", "failImageWriter"), cmp.AllowUnexported(chromedp.Selector{}), cmpopts.IgnoreFields(chromedp.Selector{}, "by", "wait", "after"), + cmp.AllowUnexported(dirImageReadWriter{}), ); diff != "" { t.Errorf("readTests() mismatch (-want +got):\n%s", diff) } @@ -243,29 +240,26 @@ func TestRun(t *testing.T) { testURL: "https://go.dev", wantURL: "https://pkg.go.dev", file: "testdata/fail.txt", - output: filepath.Join(cache, "fail-txt"), + output: filepath.Join(cache, "fail"), }, wantErr: true, wantFiles: []string{ - filepath.Join(cache, "fail-txt", "homepage.a.png"), - filepath.Join(cache, "fail-txt", "homepage.b.png"), - filepath.Join(cache, "fail-txt", "homepage.diff.png"), + filepath.Join(cache, "fail", "homepage.diff.png"), + filepath.Join(cache, "fail", "homepage.got.png"), + filepath.Join(cache, "fail", "homepage.want.png"), }, }, { - name: "cached", + name: "want stored", args: args{ - testURL: "https://go.dev::cache", - wantURL: "https://go.dev::cache", + testURL: "https://go.dev", + wantURL: "testdata/screenshots", file: "testdata/cached.txt", output: "testdata/screenshots/cached", }, - opts: options{ - outputURL: "testdata/screenshots/cached", - }, + opts: options{update: true}, wantFiles: []string{ - filepath.Join("testdata", "screenshots", "cached", "homepage.a.png"), - filepath.Join("testdata", "screenshots", "cached", "homepage.b.png"), + filepath.Join("testdata", "screenshots", "cached", "homepage.want.png"), }, }, } @@ -297,13 +291,12 @@ func TestHeaders(t *testing.T) { go headerServer() tc := &testcase{ name: "go.dev homepage", - urlA: "http://localhost:6061", - cacheA: true, - urlB: "http://localhost:6061", + testURL: "http://localhost:6061", + wantURL: "http://localhost:6061", headers: map[string]interface{}{"Authorization": "Bearer token"}, - outImgA: filepath.Join("testdata", "screenshots", "headers", "headers-test.a.png"), - outImgB: filepath.Join("testdata", "screenshots", "headers", "headers-test.b.png"), - outDiff: filepath.Join("testdata", "screenshots", "headers", "headers-test.diff.png"), + testPath: filepath.Join("testdata", "screenshots", "headers", "headers-test.got.png"), + wantPath: filepath.Join("testdata", "screenshots", "headers", "headers-test.want.png"), + diffPath: filepath.Join("testdata", "screenshots", "headers", "headers-test.diff.png"), viewportWidth: 1536, viewportHeight: 960, screenshotType: elementScreenshot, @@ -327,117 +320,6 @@ func headerServer() error { return http.ListenAndServe(fmt.Sprintf(":%d", 6061), mux) } -func Test_gcsParts(t *testing.T) { - type args struct { - filename string - } - tests := []struct { - name string - args args - wantBucket string - wantObject string - }{ - { - args: args{ - filename: "gs://bucket-name/object-name", - }, - wantBucket: "bucket-name", - wantObject: "object-name", - }, - { - args: args{ - filename: "gs://bucket-name/subdir/object-name", - }, - wantBucket: "bucket-name", - wantObject: "subdir/object-name", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotBucket, gotObject := gcsParts(tt.args.filename) - if gotBucket != tt.wantBucket { - t.Errorf("gcsParts() gotBucket = %v, want %v", gotBucket, tt.wantBucket) - } - if gotObject != tt.wantObject { - t.Errorf("gcsParts() gotObject = %v, want %v", gotObject, tt.wantObject) - } - }) - } -} - -func Test_cleanDirs(t *testing.T) { - f, err := os.Create("testdata/screenshots/cached/should-delete.a.png") - if err != nil { - t.Fatal(err) - } - if err := f.Close(); err != nil { - t.Fatal(err) - } - type args struct { - dirs map[string]bool - keepFiles map[string]bool - safeExts map[string]bool - } - tests := []struct { - name string - args args - wantFiles map[string]bool - }{ - { - name: "keeps files in keepFiles", - args: args{ - dirs: map[string]bool{ - "testdata/screenshots/cached": true, - "testdata/screenshots/headers": true, - "testdata": true, - }, - keepFiles: map[string]bool{ - "testdata/screenshots/cached/homepage.a.png": true, - "testdata/screenshots/cached/homepage.b.png": true, - "testdata/screenshots/headers/headers-test.a.png": true, - }, - safeExts: map[string]bool{ - "a.png": true, - "b.png": true, - }, - }, - wantFiles: map[string]bool{ - "testdata/screenshots/cached/homepage.a.png": true, - "testdata/screenshots/headers/headers-test.a.png": true, - }, - }, - { - name: "keeps files without matching extension", - args: args{ - dirs: map[string]bool{ - "testdata": true, - }, - safeExts: map[string]bool{ - "a.png": true, - }, - }, - wantFiles: map[string]bool{ - "testdata/cached.txt": true, - "testdata/fail.txt": true, - "testdata/pass.txt": true, - "testdata/readtests.txt": true, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := cleanDirs(tt.args.dirs, tt.args.keepFiles, tt.args.safeExts); err != nil { - t.Fatal(err) - } - for file := range tt.wantFiles { - if _, err := os.Stat(file); err != nil { - t.Errorf("cleanDirs() error = %v, wantErr %v", err, nil) - } - } - }) - } -} - func TestSplitDimensions(t *testing.T) { for _, tc := range []struct { in string @@ -463,3 +345,92 @@ func TestSplitDimensions(t *testing.T) { } } } + +func TestReadWriters(t *testing.T) { + img := image.NewGray(image.Rect(0, 0, 10, 10)) + ctx := context.Background() + path := "sub/file.png" + + test := func(t *testing.T, rw imageReadWriter) { + if err := rw.writeImage(ctx, path, img); err != nil { + t.Fatal(err) + } + got, err := rw.readImage(ctx, path) + if err != nil { + t.Fatal(err) + } + result := imgdiff.Diff(img, got, &imgdiff.Options{}) + if !result.Equal { + t.Error("images not equal") + } + } + + t.Run("dir", func(t *testing.T) { + test(t, &dirImageReadWriter{t.TempDir()}) + }) + t.Run("gcs", func(t *testing.T) { + if *bucketPath == "" { + t.Skip("missing -bucketpath") + } + rw, err := newGCSImageReadWriter(ctx, "gs://"+*bucketPath) + if err != nil { + t.Fatal(err) + } + fmt.Printf("%+v\n", rw) + test(t, rw) + }) +} + +func TestNewImageReadWriter(t *testing.T) { + for _, tc := range []struct { + in string + wantDir string // implies dirImageReadWriter + wantPrefix string // implies gcsImageReadWriter + }{ + { + in: "unix/path", + wantDir: "unix/path", + }, + { + in: "unix/path/../dir", + wantDir: "unix/dir", + }, + { + in: "c:/windows/path", + wantDir: "c:/windows/path", + }, + { + in: "c:/windows/../dir", + wantDir: "c:/dir", + }, + { + in: "file:///file/path", + wantDir: "/file/path", + }, + { + in: "gs://bucket/prefix", + wantPrefix: "prefix", + }, + { + in: "http://example.com", + }, + } { + got, err := newImageReadWriter(context.Background(), tc.in) + if err != nil { + t.Fatal(err) + } + if tc.wantDir != "" { + d, ok := got.(*dirImageReadWriter) + if !ok || d.dir != tc.wantDir { + t.Errorf("%s: got %+v, want dirImageReadWriter{dir: %q}", tc.in, got, tc.wantDir) + } + } else if tc.wantPrefix != "" { + g, ok := got.(*gcsImageReadWriter) + if !ok || g.prefix != tc.wantPrefix { + t.Errorf("%s: got %+v, want gcsImageReadWriter{prefix: %q}", tc.in, got, tc.wantPrefix) + } + } else if got != nil { + t.Errorf("%s: got %+v, want nil", tc.in, got) + } + } +} diff --git a/cmd/screentest/testdata/screenshots/cached/homepage.a.png b/cmd/screentest/testdata/screenshots/cached/homepage.a.png Binary files differdeleted file mode 100644 index e7b4e34a..00000000 --- a/cmd/screentest/testdata/screenshots/cached/homepage.a.png +++ /dev/null diff --git a/cmd/screentest/testdata/screenshots/cached/homepage.b.png b/cmd/screentest/testdata/screenshots/cached/homepage.b.png Binary files differdeleted file mode 100644 index e7b4e34a..00000000 --- a/cmd/screentest/testdata/screenshots/cached/homepage.b.png +++ /dev/null diff --git a/cmd/screentest/testdata/screenshots/cached/homepage.want.png b/cmd/screentest/testdata/screenshots/cached/homepage.want.png Binary files differnew file mode 100644 index 00000000..9933492f --- /dev/null +++ b/cmd/screentest/testdata/screenshots/cached/homepage.want.png |
