From 90c924ff88a8b5ab65538ccc16d160922b1b4003 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 6 Jul 2020 10:58:05 -0400 Subject: testing/fstest: new package for testing file system code This change adds basic test helpers for file system code. The type MapFS is a simple map-based file system for use when exercising general file system code. The func TestFS tests a file system implementation. For #41190. Change-Id: I5a2036f57e733915ad508651ad7317749794423c Reviewed-on: https://go-review.googlesource.com/c/go/+/243910 Trust: Russ Cox Reviewed-by: Rob Pike --- src/testing/fstest/mapfs.go | 204 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 src/testing/fstest/mapfs.go (limited to 'src/testing/fstest/mapfs.go') diff --git a/src/testing/fstest/mapfs.go b/src/testing/fstest/mapfs.go new file mode 100644 index 0000000000..84a943f409 --- /dev/null +++ b/src/testing/fstest/mapfs.go @@ -0,0 +1,204 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fstest + +import ( + "io" + "io/fs" + "path" + "sort" + "strings" + "time" +) + +// A MapFS is a simple in-memory file system for use in tests, +// represented as a map from path names (arguments to Open) +// to information about the files or directories they represent. +// +// The map need not include parent directories for files contained +// in the map; those will be synthesized if needed. +// But a directory can still be included by setting the MapFile.Mode's ModeDir bit; +// this may be necessary for detailed control over the directory's FileInfo +// or to create an empty directory. +// +// File system operations read directly from the map, +// so that the file system can be changed by editing the map as needed. +// An implication is that file system operations must not run concurrently +// with changes to the map, which would be a race. +// Another implication is that opening or reading a directory requires +// iterating over the entire map, so a MapFS should typically be used with not more +// than a few hundred entries or directory reads. +type MapFS map[string]*MapFile + +// A MapFile describes a single file in a MapFS. +type MapFile struct { + Data []byte // file content + Mode fs.FileMode // FileInfo.Mode + ModTime time.Time // FileInfo.ModTime + Sys interface{} // FileInfo.Sys +} + +var _ fs.FS = MapFS(nil) +var _ fs.File = (*openMapFile)(nil) + +// Open opens the named file. +func (fsys MapFS) Open(name string) (fs.File, error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } + file := fsys[name] + if file != nil && file.Mode&fs.ModeDir == 0 { + // Ordinary file + return &openMapFile{name, mapFileInfo{path.Base(name), file}, 0}, nil + } + + // Directory, possibly synthesized. + // Note that file can be nil here: the map need not contain explicit parent directories for all its files. + // But file can also be non-nil, in case the user wants to set metadata for the directory explicitly. + // Either way, we need to construct the list of children of this directory. + var list []mapFileInfo + var elem string + var need = make(map[string]bool) + if name == "." { + elem = "." + for fname, f := range fsys { + i := strings.Index(fname, "/") + if i < 0 { + list = append(list, mapFileInfo{fname, f}) + } else { + need[fname[:i]] = true + } + } + } else { + elem = name[strings.LastIndex(name, "/")+1:] + prefix := name + "/" + for fname, f := range fsys { + if strings.HasPrefix(fname, prefix) { + felem := fname[len(prefix):] + i := strings.Index(felem, "/") + if i < 0 { + list = append(list, mapFileInfo{felem, f}) + } else { + need[fname[len(prefix):len(prefix)+i]] = true + } + } + } + // If the directory name is not in the map, + // and there are no children of the name in the map, + // then the directory is treated as not existing. + if file == nil && list == nil && len(need) == 0 { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } + } + for _, fi := range list { + delete(need, fi.name) + } + for name := range need { + list = append(list, mapFileInfo{name, &MapFile{Mode: fs.ModeDir}}) + } + sort.Slice(list, func(i, j int) bool { + return list[i].name < list[j].name + }) + + if file == nil { + file = &MapFile{Mode: fs.ModeDir} + } + return &mapDir{name, mapFileInfo{elem, file}, list, 0}, nil +} + +// A mapFileInfo implements fs.FileInfo and fs.DirEntry for a given map file. +type mapFileInfo struct { + name string + f *MapFile +} + +func (i *mapFileInfo) Name() string { return i.name } +func (i *mapFileInfo) Size() int64 { return int64(len(i.f.Data)) } +func (i *mapFileInfo) Mode() fs.FileMode { return i.f.Mode } +func (i *mapFileInfo) Type() fs.FileMode { return i.f.Mode.Type() } +func (i *mapFileInfo) ModTime() time.Time { return i.f.ModTime } +func (i *mapFileInfo) IsDir() bool { return i.f.Mode&fs.ModeDir != 0 } +func (i *mapFileInfo) Sys() interface{} { return i.f.Sys } +func (i *mapFileInfo) Info() (fs.FileInfo, error) { return i, nil } + +// An openMapFile is a regular (non-directory) fs.File open for reading. +type openMapFile struct { + path string + mapFileInfo + offset int64 +} + +func (f *openMapFile) Stat() (fs.FileInfo, error) { return &f.mapFileInfo, nil } + +func (f *openMapFile) Close() error { return nil } + +func (f *openMapFile) Read(b []byte) (int, error) { + if f.offset >= int64(len(f.f.Data)) { + return 0, io.EOF + } + if f.offset < 0 { + return 0, &fs.PathError{Op: "read", Path: f.path, Err: fs.ErrInvalid} + } + n := copy(b, f.f.Data[f.offset:]) + f.offset += int64(n) + return n, nil +} + +func (f *openMapFile) Seek(offset int64, whence int) (int64, error) { + switch whence { + case 0: + // offset += 0 + case 1: + offset += f.offset + case 2: + offset += int64(len(f.f.Data)) + } + if offset < 0 || offset > int64(len(f.f.Data)) { + return 0, &fs.PathError{Op: "seek", Path: f.path, Err: fs.ErrInvalid} + } + f.offset = offset + return offset, nil +} + +func (f *openMapFile) ReadAt(b []byte, offset int64) (int, error) { + if offset < 0 || offset > int64(len(f.f.Data)) { + return 0, &fs.PathError{Op: "read", Path: f.path, Err: fs.ErrInvalid} + } + n := copy(b, f.f.Data[offset:]) + if n < len(b) { + return n, io.EOF + } + return n, nil +} + +// A mapDir is a directory fs.File (so also an fs.ReadDirFile) open for reading. +type mapDir struct { + path string + mapFileInfo + entry []mapFileInfo + offset int +} + +func (d *mapDir) Stat() (fs.FileInfo, error) { return &d.mapFileInfo, nil } +func (d *mapDir) Close() error { return nil } +func (d *mapDir) Read(b []byte) (int, error) { + return 0, &fs.PathError{Op: "read", Path: d.path, Err: fs.ErrInvalid} +} + +func (d *mapDir) ReadDir(count int) ([]fs.DirEntry, error) { + n := len(d.entry) - d.offset + if count > 0 && n > count { + n = count + } + if n == 0 && count > 0 { + return nil, io.EOF + } + list := make([]fs.DirEntry, n) + for i := range list { + list[i] = &d.entry[d.offset+i] + } + d.offset += n + return list, nil +} -- cgit v1.3 From f098ccf04a33e2e4d2dffa2e90fe77ca8a0fcbb4 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 6 Jul 2020 11:26:26 -0400 Subject: io/fs: add ReadFile and ReadFileFS Add ReadFile helper function, ReadFileFS interface, and test. Add ReadFile method to fstest.MapFS. Add testing of ReadFile method to fstest.TestFS. For #41190. Change-Id: I5b6a41e2e582824e570463b698b635abaa436c32 Reviewed-on: https://go-review.googlesource.com/c/go/+/243912 Trust: Russ Cox Reviewed-by: Rob Pike --- src/go/build/deps_test.go | 4 ++- src/io/fs/readfile.go | 63 ++++++++++++++++++++++++++++++++++++++++++++ src/io/fs/readfile_test.go | 43 ++++++++++++++++++++++++++++++ src/testing/fstest/mapfs.go | 12 +++++++++ src/testing/fstest/testfs.go | 39 ++++++++++++++++++++++++--- 5 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 src/io/fs/readfile.go create mode 100644 src/io/fs/readfile_test.go (limited to 'src/testing/fstest/mapfs.go') diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index 4867a5031a..ccee539086 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -165,7 +165,9 @@ var depsRules = ` os/signal, STR < path/filepath - < io/ioutil, os/exec + < io/ioutil, os/exec; + + io/ioutil, os/exec, os/signal < OS; reflect !< OS; diff --git a/src/io/fs/readfile.go b/src/io/fs/readfile.go new file mode 100644 index 0000000000..7ee9eadac4 --- /dev/null +++ b/src/io/fs/readfile.go @@ -0,0 +1,63 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs + +import "io" + +// ReadFileFS is the interface implemented by a file system +// that provides an optimized implementation of ReadFile. +type ReadFileFS interface { + FS + + // ReadFile reads the named file and returns its contents. + // A successful call returns a nil error, not io.EOF. + // (Because ReadFile reads the whole file, the expected EOF + // from the final Read is not treated as an error to be reported.) + ReadFile(name string) ([]byte, error) +} + +// ReadFile reads the named file from the file system fs and returns its contents. +// A successful call returns a nil error, not io.EOF. +// (Because ReadFile reads the whole file, the expected EOF +// from the final Read is not treated as an error to be reported.) +// +// If fs implements ReadFileFS, ReadFile calls fs.ReadFile. +// Otherwise ReadFile calls fs.Open and uses Read and Close +// on the returned file. +func ReadFile(fsys FS, name string) ([]byte, error) { + if fsys, ok := fsys.(ReadFileFS); ok { + return fsys.ReadFile(name) + } + + file, err := fsys.Open(name) + if err != nil { + return nil, err + } + defer file.Close() + + var size int + if info, err := file.Stat(); err == nil { + size64 := info.Size() + if int64(int(size64)) == size64 { + size = int(size64) + } + } + + data := make([]byte, 0, size+1) + for { + if len(data) >= cap(data) { + d := append(data[:cap(data)], 0) + data = d[:len(data)] + } + n, err := file.Read(data[len(data):cap(data)]) + data = data[:len(data)+n] + if err != nil { + if err == io.EOF { + err = nil + } + return data, err + } + } +} diff --git a/src/io/fs/readfile_test.go b/src/io/fs/readfile_test.go new file mode 100644 index 0000000000..0afa334ace --- /dev/null +++ b/src/io/fs/readfile_test.go @@ -0,0 +1,43 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs_test + +import ( + . "io/fs" + "testing" + "testing/fstest" + "time" +) + +var testFsys = fstest.MapFS{ + "hello.txt": { + Data: []byte("hello, world"), + Mode: 0456, + ModTime: time.Now(), + Sys: &sysValue, + }, +} + +var sysValue int + +type readFileOnly struct{ ReadFileFS } + +func (readFileOnly) Open(name string) (File, error) { return nil, ErrNotExist } + +type openOnly struct{ FS } + +func TestReadFile(t *testing.T) { + // Test that ReadFile uses the method when present. + data, err := ReadFile(readFileOnly{testFsys}, "hello.txt") + if string(data) != "hello, world" || err != nil { + t.Fatalf(`ReadFile(readFileOnly, "hello.txt") = %q, %v, want %q, nil`, data, err, "hello, world") + } + + // Test that ReadFile uses Open when the method is not present. + data, err = ReadFile(openOnly{testFsys}, "hello.txt") + if string(data) != "hello, world" || err != nil { + t.Fatalf(`ReadFile(openOnly, "hello.txt") = %q, %v, want %q, nil`, data, err, "hello, world") + } +} diff --git a/src/testing/fstest/mapfs.go b/src/testing/fstest/mapfs.go index 84a943f409..e969ac2bd1 100644 --- a/src/testing/fstest/mapfs.go +++ b/src/testing/fstest/mapfs.go @@ -108,6 +108,18 @@ func (fsys MapFS) Open(name string) (fs.File, error) { return &mapDir{name, mapFileInfo{elem, file}, list, 0}, nil } +// fsOnly is a wrapper that hides all but the fs.FS methods, +// to avoid an infinite recursion when implementing special +// methods in terms of helpers that would use them. +// (In general, implementing these methods using the package fs helpers +// is redundant and unnecessary, but having the methods may make +// MapFS exercise more code paths when used in tests.) +type fsOnly struct{ fs.FS } + +func (fsys MapFS) ReadFile(name string) ([]byte, error) { + return fs.ReadFile(fsOnly{fsys}, name) +} + // A mapFileInfo implements fs.FileInfo and fs.DirEntry for a given map file. type mapFileInfo struct { name string diff --git a/src/testing/fstest/testfs.go b/src/testing/fstest/testfs.go index 2bb2120c19..66725ca2a4 100644 --- a/src/testing/fstest/testfs.go +++ b/src/testing/fstest/testfs.go @@ -310,6 +310,27 @@ func (t *fsTester) checkFile(file string) { // The return value doesn't matter. f.Close() + // Check that ReadFile works if present. + if fsys, ok := t.fsys.(fs.ReadFileFS); ok { + data2, err := fsys.ReadFile(file) + if err != nil { + t.errorf("%s: fsys.ReadFile: %v", file, err) + return + } + t.checkFileRead(file, "ReadAll vs fsys.ReadFile", data, data2) + + t.checkBadPath(file, "ReadFile", + func(name string) error { _, err := fsys.ReadFile(name); return err }) + } + + // Check that fs.ReadFile works with t.fsys. + data2, err := fs.ReadFile(t.fsys, file) + if err != nil { + t.errorf("%s: fs.ReadFile: %v", file, err) + return + } + t.checkFileRead(file, "ReadAll vs fs.ReadFile", data, data2) + // Use iotest.TestReader to check small reads, Seek, ReadAt. f, err = t.fsys.Open(file) if err != nil { @@ -329,8 +350,19 @@ func (t *fsTester) checkFileRead(file, desc string, data1, data2 []byte) { } } -// checkOpen checks that various invalid forms of file's name cannot be opened. +// checkBadPath checks that various invalid forms of file's name cannot be opened using t.fsys.Open. func (t *fsTester) checkOpen(file string) { + t.checkBadPath(file, "Open", func(file string) error { + f, err := t.fsys.Open(file) + if err == nil { + f.Close() + } + return err + }) +} + +// checkBadPath checks that various invalid forms of file's name cannot be opened using open. +func (t *fsTester) checkBadPath(file string, desc string, open func(string) error) { bad := []string{ "/" + file, file + "/.", @@ -356,9 +388,8 @@ func (t *fsTester) checkOpen(file string) { } for _, b := range bad { - if f, err := t.fsys.Open(b); err == nil { - f.Close() - t.errorf("%s: Open(%s) succeeded, want error", file, b) + if err := open(b); err == nil { + t.errorf("%s: %s(%s) succeeded, want error", file, desc, b) } } } -- cgit v1.3 From 10a1a1a37c007adef8425d273e6b276547982889 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 6 Jul 2020 11:26:45 -0400 Subject: io/fs: add Stat and StatFS Add Stat helper function, StatFS interface, and test. Add Stat method to fstest.MapFS. Add testing of Stat method to fstest.TestFS. For #41190. Change-Id: Icf8b6eb1c3fa6f93a9be8405ec5a9468fb1da97b Reviewed-on: https://go-review.googlesource.com/c/go/+/243913 Trust: Russ Cox Reviewed-by: Rob Pike --- src/io/fs/stat.go | 31 +++++++++++++++++++++++++++++++ src/io/fs/stat_test.go | 36 ++++++++++++++++++++++++++++++++++++ src/testing/fstest/mapfs.go | 4 ++++ src/testing/fstest/testfs.go | 25 ++++++++++++++++++++++++- 4 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 src/io/fs/stat.go create mode 100644 src/io/fs/stat_test.go (limited to 'src/testing/fstest/mapfs.go') diff --git a/src/io/fs/stat.go b/src/io/fs/stat.go new file mode 100644 index 0000000000..735a6e3281 --- /dev/null +++ b/src/io/fs/stat.go @@ -0,0 +1,31 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs + +// A StatFS is a file system with a Stat method. +type StatFS interface { + FS + + // Stat returns a FileInfo describing the file. + // If there is an error, it should be of type *PathError. + Stat(name string) (FileInfo, error) +} + +// Stat returns a FileInfo describing the named file from the file system. +// +// If fs implements StatFS, Stat calls fs.Stat. +// Otherwise, Stat opens the file to stat it. +func Stat(fsys FS, name string) (FileInfo, error) { + if fsys, ok := fsys.(StatFS); ok { + return fsys.Stat(name) + } + + file, err := fsys.Open(name) + if err != nil { + return nil, err + } + defer file.Close() + return file.Stat() +} diff --git a/src/io/fs/stat_test.go b/src/io/fs/stat_test.go new file mode 100644 index 0000000000..e312b6fbd9 --- /dev/null +++ b/src/io/fs/stat_test.go @@ -0,0 +1,36 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs_test + +import ( + "fmt" + . "io/fs" + "testing" +) + +type statOnly struct{ StatFS } + +func (statOnly) Open(name string) (File, error) { return nil, ErrNotExist } + +func TestStat(t *testing.T) { + check := func(desc string, info FileInfo, err error) { + t.Helper() + if err != nil || info == nil || info.Mode() != 0456 { + infoStr := "" + if info != nil { + infoStr = fmt.Sprintf("FileInfo(Mode: %#o)", info.Mode()) + } + t.Fatalf("Stat(%s) = %v, %v, want Mode:0456, nil", desc, infoStr, err) + } + } + + // Test that Stat uses the method when present. + info, err := Stat(statOnly{testFsys}, "hello.txt") + check("statOnly", info, err) + + // Test that Stat uses Open when the method is not present. + info, err = Stat(openOnly{testFsys}, "hello.txt") + check("openOnly", info, err) +} diff --git a/src/testing/fstest/mapfs.go b/src/testing/fstest/mapfs.go index e969ac2bd1..b01911e589 100644 --- a/src/testing/fstest/mapfs.go +++ b/src/testing/fstest/mapfs.go @@ -120,6 +120,10 @@ func (fsys MapFS) ReadFile(name string) ([]byte, error) { return fs.ReadFile(fsOnly{fsys}, name) } +func (fsys MapFS) Stat(name string) (fs.FileInfo, error) { + return fs.Stat(fsOnly{fsys}, name) +} + // A mapFileInfo implements fs.FileInfo and fs.DirEntry for a given map file. type mapFileInfo struct { name string diff --git a/src/testing/fstest/testfs.go b/src/testing/fstest/testfs.go index 66725ca2a4..290d2596cc 100644 --- a/src/testing/fstest/testfs.go +++ b/src/testing/fstest/testfs.go @@ -230,7 +230,30 @@ func (t *fsTester) checkStat(path string, entry fs.DirEntry) { fentry := formatEntry(entry) finfo := formatInfoEntry(info) if fentry != finfo { - t.errorf("%s: mismatch:\n\tentry = %v\n\tfile.Stat() = %v", path, fentry, finfo) + t.errorf("%s: mismatch:\n\tentry = %s\n\tfile.Stat() = %s", path, fentry, finfo) + } + + info2, err := fs.Stat(t.fsys, path) + if err != nil { + t.errorf("%s: fs.Stat: %v", path, err) + return + } + finfo = formatInfo(info) + finfo2 := formatInfo(info2) + if finfo2 != finfo { + t.errorf("%s: fs.Stat(...) = %s\n\twant %s", path, finfo2, finfo) + } + + if fsys, ok := t.fsys.(fs.StatFS); ok { + info2, err := fsys.Stat(path) + if err != nil { + t.errorf("%s: fsys.Stat: %v", path, err) + return + } + finfo2 := formatInfo(info2) + if finfo2 != finfo { + t.errorf("%s: fsys.Stat(...) = %s\n\twant %s", path, finfo2, finfo) + } } } -- cgit v1.3 From 7a131acfd142f0fc7612365078b9f00e371fc0e2 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 6 Jul 2020 11:26:56 -0400 Subject: io/fs: add ReadDir and ReadDirFS Add ReadDir helper function, ReadDirFS interface, and test. Add ReadDir method to fstest.MapFS. Add testing of ReadDir method to fstest.TestFS. For #41190. Change-Id: Ib860770ec7433ba77b29e626682b238f1b3bf54f Reviewed-on: https://go-review.googlesource.com/c/go/+/243914 Trust: Russ Cox Reviewed-by: Rob Pike --- src/io/fs/readdir.go | 47 ++++++++++++++++++++++++++++++++++++++++++++ src/io/fs/readdir_test.go | 35 +++++++++++++++++++++++++++++++++ src/testing/fstest/mapfs.go | 4 ++++ src/testing/fstest/testfs.go | 42 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 src/io/fs/readdir.go create mode 100644 src/io/fs/readdir_test.go (limited to 'src/testing/fstest/mapfs.go') diff --git a/src/io/fs/readdir.go b/src/io/fs/readdir.go new file mode 100644 index 0000000000..3a5aa6d86a --- /dev/null +++ b/src/io/fs/readdir.go @@ -0,0 +1,47 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs + +import ( + "errors" + "sort" +) + +// ReadDirFS is the interface implemented by a file system +// that provides an optimized implementation of ReadDir. +type ReadDirFS interface { + FS + + // ReadDir reads the named directory + // and returns a list of directory entries sorted by filename. + ReadDir(name string) ([]DirEntry, error) +} + +// ReadDir reads the named directory +// and returns a list of directory entries sorted by filename. +// +// If fs implements ReadDirFS, ReadDir calls fs.ReadDir. +// Otherwise ReadDir calls fs.Open and uses ReadDir and Close +// on the returned file. +func ReadDir(fsys FS, name string) ([]DirEntry, error) { + if fsys, ok := fsys.(ReadDirFS); ok { + return fsys.ReadDir(name) + } + + file, err := fsys.Open(name) + if err != nil { + return nil, err + } + defer file.Close() + + dir, ok := file.(ReadDirFile) + if !ok { + return nil, &PathError{Op: "readdir", Path: name, Err: errors.New("not implemented")} + } + + list, err := dir.ReadDir(-1) + sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() }) + return list, err +} diff --git a/src/io/fs/readdir_test.go b/src/io/fs/readdir_test.go new file mode 100644 index 0000000000..46a4bc2788 --- /dev/null +++ b/src/io/fs/readdir_test.go @@ -0,0 +1,35 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs_test + +import ( + . "io/fs" + "testing" +) + +type readDirOnly struct{ ReadDirFS } + +func (readDirOnly) Open(name string) (File, error) { return nil, ErrNotExist } + +func TestReadDir(t *testing.T) { + check := func(desc string, dirs []DirEntry, err error) { + t.Helper() + if err != nil || len(dirs) != 1 || dirs[0].Name() != "hello.txt" { + var names []string + for _, d := range dirs { + names = append(names, d.Name()) + } + t.Errorf("ReadDir(%s) = %v, %v, want %v, nil", desc, names, err, []string{"hello.txt"}) + } + } + + // Test that ReadDir uses the method when present. + dirs, err := ReadDir(readDirOnly{testFsys}, ".") + check("readDirOnly", dirs, err) + + // Test that ReadDir uses Open when the method is not present. + dirs, err = ReadDir(openOnly{testFsys}, ".") + check("openOnly", dirs, err) +} diff --git a/src/testing/fstest/mapfs.go b/src/testing/fstest/mapfs.go index b01911e589..1eaf8f0040 100644 --- a/src/testing/fstest/mapfs.go +++ b/src/testing/fstest/mapfs.go @@ -124,6 +124,10 @@ func (fsys MapFS) Stat(name string) (fs.FileInfo, error) { return fs.Stat(fsOnly{fsys}, name) } +func (fsys MapFS) ReadDir(name string) ([]fs.DirEntry, error) { + return fs.ReadDir(fsOnly{fsys}, name) +} + // A mapFileInfo implements fs.FileInfo and fs.DirEntry for a given map file. type mapFileInfo struct { name string diff --git a/src/testing/fstest/testfs.go b/src/testing/fstest/testfs.go index 290d2596cc..4ea6ed6095 100644 --- a/src/testing/fstest/testfs.go +++ b/src/testing/fstest/testfs.go @@ -196,6 +196,36 @@ func (t *fsTester) checkDir(dir string) { } } t.checkDirList(dir, "first Open+ReadDir(-1) vs third Open+ReadDir(1,2) loop", list, list2) + + // If fsys has ReadDir, check that it matches and is sorted. + if fsys, ok := t.fsys.(fs.ReadDirFS); ok { + list2, err := fsys.ReadDir(dir) + if err != nil { + t.errorf("%s: fsys.ReadDir: %v", dir, err) + return + } + t.checkDirList(dir, "first Open+ReadDir(-1) vs fsys.ReadDir", list, list2) + + for i := 0; i+1 < len(list2); i++ { + if list2[i].Name() >= list2[i+1].Name() { + t.errorf("%s: fsys.ReadDir: list not sorted: %s before %s", dir, list2[i].Name(), list2[i+1].Name()) + } + } + } + + // Check fs.ReadDir as well. + list2, err = fs.ReadDir(t.fsys, dir) + if err != nil { + t.errorf("%s: fs.ReadDir: %v", dir, err) + return + } + t.checkDirList(dir, "first Open+ReadDir(-1) vs fs.ReadDir", list, list2) + + for i := 0; i+1 < len(list2); i++ { + if list2[i].Name() >= list2[i+1].Name() { + t.errorf("%s: fs.ReadDir: list not sorted: %s before %s", dir, list2[i].Name(), list2[i+1].Name()) + } + } } // formatEntry formats an fs.DirEntry into a string for error messages and comparison. @@ -233,12 +263,22 @@ func (t *fsTester) checkStat(path string, entry fs.DirEntry) { t.errorf("%s: mismatch:\n\tentry = %s\n\tfile.Stat() = %s", path, fentry, finfo) } + einfo, err := entry.Info() + if err != nil { + t.errorf("%s: entry.Info: %v", path, err) + return + } + fentry = formatInfo(einfo) + finfo = formatInfo(info) + if fentry != finfo { + t.errorf("%s: mismatch:\n\tentry.Info() = %s\n\tfile.Stat() = %s\n", path, fentry, finfo) + } + info2, err := fs.Stat(t.fsys, path) if err != nil { t.errorf("%s: fs.Stat: %v", path, err) return } - finfo = formatInfo(info) finfo2 := formatInfo(info2) if finfo2 != finfo { t.errorf("%s: fs.Stat(...) = %s\n\twant %s", path, finfo2, finfo) -- cgit v1.3 From b64202bc29b9c1cf0118878d1c0acc9cdb2308f6 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 6 Jul 2020 11:27:12 -0400 Subject: io/fs: add Glob and GlobFS Add Glob helper function, GlobFS interface, and test. Add Glob method to fstest.MapFS. Add testing of Glob method to fstest.TestFS. For #41190. Change-Id: If89dd7f63e310ba5ca2651340267a9ff39fcc0c7 Reviewed-on: https://go-review.googlesource.com/c/go/+/243915 Trust: Russ Cox Reviewed-by: Rob Pike --- src/go/build/deps_test.go | 2 +- src/io/fs/glob.go | 116 +++++++++++++++++++++++++++++++++++++++++++ src/io/fs/glob_test.go | 82 ++++++++++++++++++++++++++++++ src/testing/fstest/mapfs.go | 4 ++ src/testing/fstest/testfs.go | 96 +++++++++++++++++++++++++++++++++++ 5 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 src/io/fs/glob.go create mode 100644 src/io/fs/glob_test.go (limited to 'src/testing/fstest/mapfs.go') diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index ccee539086..16a67791cf 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -123,7 +123,7 @@ var depsRules = ` < context < TIME; - TIME, io, sort + TIME, io, path, sort < io/fs; # MATH is RUNTIME plus the basic math packages. diff --git a/src/io/fs/glob.go b/src/io/fs/glob.go new file mode 100644 index 0000000000..77f6ebbaaf --- /dev/null +++ b/src/io/fs/glob.go @@ -0,0 +1,116 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs + +import ( + "path" + "runtime" +) + +// A GlobFS is a file system with a Glob method. +type GlobFS interface { + FS + + // Glob returns the names of all files matching pattern, + // providing an implementation of the top-level + // Glob function. + Glob(pattern string) ([]string, error) +} + +// Glob returns the names of all files matching pattern or nil +// if there is no matching file. The syntax of patterns is the same +// as in path.Match. The pattern may describe hierarchical names such as +// /usr/*/bin/ed (assuming the Separator is '/'). +// +// Glob ignores file system errors such as I/O errors reading directories. +// The only possible returned error is path.ErrBadPattern, reporting that +// the pattern is malformed. +// +// If fs implements GlobFS, Glob calls fs.Glob. +// Otherwise, Glob uses ReadDir to traverse the directory tree +// and look for matches for the pattern. +func Glob(fsys FS, pattern string) (matches []string, err error) { + if fsys, ok := fsys.(GlobFS); ok { + return fsys.Glob(pattern) + } + + if !hasMeta(pattern) { + if _, err = Stat(fsys, pattern); err != nil { + return nil, nil + } + return []string{pattern}, nil + } + + dir, file := path.Split(pattern) + dir = cleanGlobPath(dir) + + if !hasMeta(dir) { + return glob(fsys, dir, file, nil) + } + + // Prevent infinite recursion. See issue 15879. + if dir == pattern { + return nil, path.ErrBadPattern + } + + var m []string + m, err = Glob(fsys, dir) + if err != nil { + return + } + for _, d := range m { + matches, err = glob(fsys, d, file, matches) + if err != nil { + return + } + } + return +} + +// cleanGlobPath prepares path for glob matching. +func cleanGlobPath(path string) string { + switch path { + case "": + return "." + default: + return path[0 : len(path)-1] // chop off trailing separator + } +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches, returning the updated slice. +// If the directory cannot be opened, glob returns the existing matches. +// New matches are added in lexicographical order. +func glob(fs FS, dir, pattern string, matches []string) (m []string, e error) { + m = matches + infos, err := ReadDir(fs, dir) + if err != nil { + return // ignore I/O error + } + + for _, info := range infos { + n := info.Name() + matched, err := path.Match(pattern, n) + if err != nil { + return m, err + } + if matched { + m = append(m, path.Join(dir, n)) + } + } + return +} + +// hasMeta reports whether path contains any of the magic characters +// recognized by path.Match. +func hasMeta(path string) bool { + for i := 0; i < len(path); i++ { + c := path[i] + if c == '*' || c == '?' || c == '[' || runtime.GOOS == "windows" && c == '\\' { + return true + } + } + return false +} diff --git a/src/io/fs/glob_test.go b/src/io/fs/glob_test.go new file mode 100644 index 0000000000..0183a49b6c --- /dev/null +++ b/src/io/fs/glob_test.go @@ -0,0 +1,82 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs_test + +import ( + . "io/fs" + "os" + "testing" +) + +var globTests = []struct { + fs FS + pattern, result string +}{ + {os.DirFS("."), "glob.go", "glob.go"}, + {os.DirFS("."), "gl?b.go", "glob.go"}, + {os.DirFS("."), "*", "glob.go"}, + {os.DirFS(".."), "*/glob.go", "fs/glob.go"}, +} + +func TestGlob(t *testing.T) { + for _, tt := range globTests { + matches, err := Glob(tt.fs, tt.pattern) + if err != nil { + t.Errorf("Glob error for %q: %s", tt.pattern, err) + continue + } + if !contains(matches, tt.result) { + t.Errorf("Glob(%#q) = %#v want %v", tt.pattern, matches, tt.result) + } + } + for _, pattern := range []string{"no_match", "../*/no_match"} { + matches, err := Glob(os.DirFS("."), pattern) + if err != nil { + t.Errorf("Glob error for %q: %s", pattern, err) + continue + } + if len(matches) != 0 { + t.Errorf("Glob(%#q) = %#v want []", pattern, matches) + } + } +} + +func TestGlobError(t *testing.T) { + _, err := Glob(os.DirFS("."), "[]") + if err == nil { + t.Error("expected error for bad pattern; got none") + } +} + +// contains reports whether vector contains the string s. +func contains(vector []string, s string) bool { + for _, elem := range vector { + if elem == s { + return true + } + } + return false +} + +type globOnly struct{ GlobFS } + +func (globOnly) Open(name string) (File, error) { return nil, ErrNotExist } + +func TestGlobMethod(t *testing.T) { + check := func(desc string, names []string, err error) { + t.Helper() + if err != nil || len(names) != 1 || names[0] != "hello.txt" { + t.Errorf("Glob(%s) = %v, %v, want %v, nil", desc, names, err, []string{"hello.txt"}) + } + } + + // Test that ReadDir uses the method when present. + names, err := Glob(globOnly{testFsys}, "*.txt") + check("readDirOnly", names, err) + + // Test that ReadDir uses Open when the method is not present. + names, err = Glob(openOnly{testFsys}, "*.txt") + check("openOnly", names, err) +} diff --git a/src/testing/fstest/mapfs.go b/src/testing/fstest/mapfs.go index 1eaf8f0040..10e56f5b3c 100644 --- a/src/testing/fstest/mapfs.go +++ b/src/testing/fstest/mapfs.go @@ -128,6 +128,10 @@ func (fsys MapFS) ReadDir(name string) ([]fs.DirEntry, error) { return fs.ReadDir(fsOnly{fsys}, name) } +func (fsys MapFS) Glob(pattern string) ([]string, error) { + return fs.Glob(fsOnly{fsys}, pattern) +} + // A mapFileInfo implements fs.FileInfo and fs.DirEntry for a given map file. type mapFileInfo struct { name string diff --git a/src/testing/fstest/testfs.go b/src/testing/fstest/testfs.go index 4ea6ed6095..21cd00e5b6 100644 --- a/src/testing/fstest/testfs.go +++ b/src/testing/fstest/testfs.go @@ -11,6 +11,8 @@ import ( "io" "io/fs" "io/ioutil" + "path" + "reflect" "sort" "strings" "testing/iotest" @@ -226,6 +228,8 @@ func (t *fsTester) checkDir(dir string) { t.errorf("%s: fs.ReadDir: list not sorted: %s before %s", dir, list2[i].Name(), list2[i+1].Name()) } } + + t.checkGlob(dir, list) } // formatEntry formats an fs.DirEntry into a string for error messages and comparison. @@ -243,6 +247,98 @@ func formatInfo(info fs.FileInfo) string { return fmt.Sprintf("%s IsDir=%v Mode=%v Size=%d ModTime=%v", info.Name(), info.IsDir(), info.Mode(), info.Size(), info.ModTime()) } +// checkGlob checks that various glob patterns work if the file system implements GlobFS. +func (t *fsTester) checkGlob(dir string, list []fs.DirEntry) { + if _, ok := t.fsys.(fs.GlobFS); !ok { + return + } + + // Make a complex glob pattern prefix that only matches dir. + var glob string + if dir != "." { + elem := strings.Split(dir, "/") + for i, e := range elem { + var pattern []rune + for j, r := range e { + if r == '*' || r == '?' || r == '\\' || r == '[' { + pattern = append(pattern, '\\', r) + continue + } + switch (i + j) % 5 { + case 0: + pattern = append(pattern, r) + case 1: + pattern = append(pattern, '[', r, ']') + case 2: + pattern = append(pattern, '[', r, '-', r, ']') + case 3: + pattern = append(pattern, '[', '\\', r, ']') + case 4: + pattern = append(pattern, '[', '\\', r, '-', '\\', r, ']') + } + } + elem[i] = string(pattern) + } + glob = strings.Join(elem, "/") + "/" + } + + // Try to find a letter that appears in only some of the final names. + c := rune('a') + for ; c <= 'z'; c++ { + have, haveNot := false, false + for _, d := range list { + if strings.ContainsRune(d.Name(), c) { + have = true + } else { + haveNot = true + } + } + if have && haveNot { + break + } + } + if c > 'z' { + c = 'a' + } + glob += "*" + string(c) + "*" + + var want []string + for _, d := range list { + if strings.ContainsRune(d.Name(), c) { + want = append(want, path.Join(dir, d.Name())) + } + } + + names, err := t.fsys.(fs.GlobFS).Glob(glob) + if err != nil { + t.errorf("%s: Glob(%#q): %v", dir, glob, err) + return + } + if reflect.DeepEqual(want, names) { + return + } + + if !sort.StringsAreSorted(names) { + t.errorf("%s: Glob(%#q): unsorted output:\n%s", dir, glob, strings.Join(names, "\n")) + sort.Strings(names) + } + + var problems []string + for len(want) > 0 || len(names) > 0 { + switch { + case len(want) > 0 && len(names) > 0 && want[0] == names[0]: + want, names = want[1:], names[1:] + case len(want) > 0 && (len(names) == 0 || want[0] < names[0]): + problems = append(problems, "missing: "+want[0]) + want = want[1:] + default: + problems = append(problems, "extra: "+names[0]) + names = names[1:] + } + } + t.errorf("%s: Glob(%#q): wrong output:\n%s", dir, glob, strings.Join(problems, "\n")) +} + // checkStat checks that a direct stat of path matches entry, // which was found in the parent's directory listing. func (t *fsTester) checkStat(path string, entry fs.DirEntry) { -- cgit v1.3 From 478bde3a4388997924a02ee9296864866d8ba3ba Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 2 Dec 2020 12:49:20 -0500 Subject: io/fs: add Sub Sub provides a convenient way to refer to a subdirectory automatically in future operations, like Unix's chdir(2). The CL also includes updates to fstest to check Sub implementations. As part of updating fstest, I changed the meaning of TestFS's expected list to introduce a special case: if you list no expected files, that means the FS must be empty. In general it's OK not to list all the expected files, but if you list none, that's almost certainly a mistake - if your FS were broken and empty, you wouldn't find out. Making no expected files mean "must be empty" makes the mistake less likely - if your file system ever worked, then your test will keep it working. That change found a testing bug: embedtest was making exactly that mistake. Fixes #42322. Change-Id: I63fd4aa866b30061a0e51ca9a1927e576d6ec41e Reviewed-on: https://go-review.googlesource.com/c/go/+/274856 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Ian Lance Taylor --- doc/go1.16.html | 2 +- src/embed/internal/embedtest/embed_test.go | 2 +- src/io/fs/readdir_test.go | 12 ++- src/io/fs/readfile_test.go | 16 ++++ src/io/fs/sub.go | 127 +++++++++++++++++++++++++++++ src/io/fs/sub_test.go | 57 +++++++++++++ src/os/file.go | 7 ++ src/testing/fstest/mapfs.go | 10 +++ src/testing/fstest/testfs.go | 42 ++++++++++ 9 files changed, 271 insertions(+), 4 deletions(-) create mode 100644 src/io/fs/sub.go create mode 100644 src/io/fs/sub_test.go (limited to 'src/testing/fstest/mapfs.go') diff --git a/doc/go1.16.html b/doc/go1.16.html index 4d4b459009..62d9b97db8 100644 --- a/doc/go1.16.html +++ b/doc/go1.16.html @@ -384,7 +384,7 @@ Do not send CLs removing the interior tags from such phrases. the new embed.FS type implements fs.FS, as does zip.Reader. - The new os.Dir function + The new os.DirFS function provides an implementation of fs.FS backed by a tree of operating system files.

diff --git a/src/embed/internal/embedtest/embed_test.go b/src/embed/internal/embedtest/embed_test.go index b1707a4c04..c6a7bea7a3 100644 --- a/src/embed/internal/embedtest/embed_test.go +++ b/src/embed/internal/embedtest/embed_test.go @@ -65,7 +65,7 @@ func TestGlobal(t *testing.T) { testFiles(t, global, "testdata/hello.txt", "hello, world\n") testFiles(t, global, "testdata/glass.txt", "I can eat glass and it doesn't hurt me.\n") - if err := fstest.TestFS(global); err != nil { + if err := fstest.TestFS(global, "concurrency.txt", "testdata/hello.txt"); err != nil { t.Fatal(err) } diff --git a/src/io/fs/readdir_test.go b/src/io/fs/readdir_test.go index 46a4bc2788..405bfa67ca 100644 --- a/src/io/fs/readdir_test.go +++ b/src/io/fs/readdir_test.go @@ -16,12 +16,12 @@ func (readDirOnly) Open(name string) (File, error) { return nil, ErrNotExist } func TestReadDir(t *testing.T) { check := func(desc string, dirs []DirEntry, err error) { t.Helper() - if err != nil || len(dirs) != 1 || dirs[0].Name() != "hello.txt" { + if err != nil || len(dirs) != 2 || dirs[0].Name() != "hello.txt" || dirs[1].Name() != "sub" { var names []string for _, d := range dirs { names = append(names, d.Name()) } - t.Errorf("ReadDir(%s) = %v, %v, want %v, nil", desc, names, err, []string{"hello.txt"}) + t.Errorf("ReadDir(%s) = %v, %v, want %v, nil", desc, names, err, []string{"hello.txt", "sub"}) } } @@ -32,4 +32,12 @@ func TestReadDir(t *testing.T) { // Test that ReadDir uses Open when the method is not present. dirs, err = ReadDir(openOnly{testFsys}, ".") check("openOnly", dirs, err) + + // Test that ReadDir on Sub of . works (sub_test checks non-trivial subs). + sub, err := Sub(testFsys, ".") + if err != nil { + t.Fatal(err) + } + dirs, err = ReadDir(sub, ".") + check("sub(.)", dirs, err) } diff --git a/src/io/fs/readfile_test.go b/src/io/fs/readfile_test.go index 0afa334ace..07219c1445 100644 --- a/src/io/fs/readfile_test.go +++ b/src/io/fs/readfile_test.go @@ -18,6 +18,12 @@ var testFsys = fstest.MapFS{ ModTime: time.Now(), Sys: &sysValue, }, + "sub/goodbye.txt": { + Data: []byte("goodbye, world"), + Mode: 0456, + ModTime: time.Now(), + Sys: &sysValue, + }, } var sysValue int @@ -40,4 +46,14 @@ func TestReadFile(t *testing.T) { if string(data) != "hello, world" || err != nil { t.Fatalf(`ReadFile(openOnly, "hello.txt") = %q, %v, want %q, nil`, data, err, "hello, world") } + + // Test that ReadFile on Sub of . works (sub_test checks non-trivial subs). + sub, err := Sub(testFsys, ".") + if err != nil { + t.Fatal(err) + } + data, err = ReadFile(sub, "hello.txt") + if string(data) != "hello, world" || err != nil { + t.Fatalf(`ReadFile(sub(.), "hello.txt") = %q, %v, want %q, nil`, data, err, "hello, world") + } } diff --git a/src/io/fs/sub.go b/src/io/fs/sub.go new file mode 100644 index 0000000000..381f409504 --- /dev/null +++ b/src/io/fs/sub.go @@ -0,0 +1,127 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs + +import ( + "errors" + "path" +) + +// A SubFS is a file system with a Sub method. +type SubFS interface { + FS + + // Sub returns an FS corresponding to the subtree rooted at dir. + Sub(dir string) (FS, error) +} + +// Sub returns an FS corresponding to the subtree rooted at fsys's dir. +// +// If fs implements SubFS, Sub calls returns fsys.Sub(dir). +// Otherwise, if dir is ".", Sub returns fsys unchanged. +// Otherwise, Sub returns a new FS implementation sub that, +// in effect, implements sub.Open(dir) as fsys.Open(path.Join(dir, name)). +// The implementation also translates calls to ReadDir, ReadFile, and Glob appropriately. +// +// Note that Sub(os.DirFS("/"), "prefix") is equivalent to os.DirFS("/prefix") +// and that neither of them guarantees to avoid operating system +// accesses outside "/prefix", because the implementation of os.DirFS +// does not check for symbolic links inside "/prefix" that point to +// other directories. That is, os.DirFS is not a general substitute for a +// chroot-style security mechanism, and Sub does not change that fact. +func Sub(fsys FS, dir string) (FS, error) { + if !ValidPath(dir) { + return nil, &PathError{Op: "sub", Path: dir, Err: errors.New("invalid name")} + } + if dir == "." { + return fsys, nil + } + if fsys, ok := fsys.(SubFS); ok { + return fsys.Sub(dir) + } + return &subFS{fsys, dir}, nil +} + +type subFS struct { + fsys FS + dir string +} + +// fullName maps name to the fully-qualified name dir/name. +func (f *subFS) fullName(op string, name string) (string, error) { + if !ValidPath(name) { + return "", &PathError{Op: op, Path: name, Err: errors.New("invalid name")} + } + return path.Join(f.dir, name), nil +} + +// shorten maps name, which should start with f.dir, back to the suffix after f.dir. +func (f *subFS) shorten(name string) (rel string, ok bool) { + if name == f.dir { + return ".", true + } + if len(name) >= len(f.dir)+2 && name[len(f.dir)] == '/' && name[:len(f.dir)] == f.dir { + return name[len(f.dir)+1:], true + } + return "", false +} + +// fixErr shortens any reported names in PathErrors by stripping dir. +func (f *subFS) fixErr(err error) error { + if e, ok := err.(*PathError); ok { + if short, ok := f.shorten(e.Path); ok { + e.Path = short + } + } + return err +} + +func (f *subFS) Open(name string) (File, error) { + full, err := f.fullName("open", name) + if err != nil { + return nil, err + } + file, err := f.fsys.Open(full) + return file, f.fixErr(err) +} + +func (f *subFS) ReadDir(name string) ([]DirEntry, error) { + full, err := f.fullName("open", name) + if err != nil { + return nil, err + } + dir, err := ReadDir(f.fsys, full) + return dir, f.fixErr(err) +} + +func (f *subFS) ReadFile(name string) ([]byte, error) { + full, err := f.fullName("open", name) + if err != nil { + return nil, err + } + data, err := ReadFile(f.fsys, full) + return data, f.fixErr(err) +} + +func (f *subFS) Glob(pattern string) ([]string, error) { + // Check pattern is well-formed. + if _, err := path.Match(pattern, ""); err != nil { + return nil, err + } + if pattern == "." { + return []string{"."}, nil + } + + full := f.dir + "/" + pattern + list, err := Glob(f.fsys, full) + for i, name := range list { + name, ok := f.shorten(name) + if !ok { + return nil, errors.New("invalid result from inner fsys Glob: " + name + " not in " + f.dir) // can't use fmt in this package + } + list[i] = name + } + return list, f.fixErr(err) +} diff --git a/src/io/fs/sub_test.go b/src/io/fs/sub_test.go new file mode 100644 index 0000000000..451b0efb02 --- /dev/null +++ b/src/io/fs/sub_test.go @@ -0,0 +1,57 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs_test + +import ( + . "io/fs" + "testing" +) + +type subOnly struct{ SubFS } + +func (subOnly) Open(name string) (File, error) { return nil, ErrNotExist } + +func TestSub(t *testing.T) { + check := func(desc string, sub FS, err error) { + t.Helper() + if err != nil { + t.Errorf("Sub(sub): %v", err) + return + } + data, err := ReadFile(sub, "goodbye.txt") + if string(data) != "goodbye, world" || err != nil { + t.Errorf(`ReadFile(%s, "goodbye.txt" = %q, %v, want %q, nil`, desc, string(data), err, "goodbye, world") + } + + dirs, err := ReadDir(sub, ".") + if err != nil || len(dirs) != 1 || dirs[0].Name() != "goodbye.txt" { + var names []string + for _, d := range dirs { + names = append(names, d.Name()) + } + t.Errorf(`ReadDir(%s, ".") = %v, %v, want %v, nil`, desc, names, err, []string{"goodbye.txt"}) + } + } + + // Test that Sub uses the method when present. + sub, err := Sub(subOnly{testFsys}, "sub") + check("subOnly", sub, err) + + // Test that Sub uses Open when the method is not present. + sub, err = Sub(openOnly{testFsys}, "sub") + check("openOnly", sub, err) + + _, err = sub.Open("nonexist") + if err == nil { + t.Fatal("Open(nonexist): succeeded") + } + pe, ok := err.(*PathError) + if !ok { + t.Fatalf("Open(nonexist): error is %T, want *PathError", err) + } + if pe.Path != "nonexist" { + t.Fatalf("Open(nonexist): err.Path = %q, want %q", pe.Path, "nonexist") + } +} diff --git a/src/os/file.go b/src/os/file.go index 304b055dbe..416bc0efa6 100644 --- a/src/os/file.go +++ b/src/os/file.go @@ -609,6 +609,13 @@ func isWindowsNulName(name string) bool { } // DirFS returns a file system (an fs.FS) for the tree of files rooted at the directory dir. +// +// Note that DirFS("/prefix") only guarantees that the Open calls it makes to the +// operating system will begin with "/prefix": DirFS("/prefix").Open("file") is the +// same as os.Open("/prefix/file"). So if /prefix/file is a symbolic link pointing outside +// the /prefix tree, then using DirFS does not stop the access any more than using +// os.Open does. DirFS is therefore not a general substitute for a chroot-style security +// mechanism when the directory tree contains arbitrary content. func DirFS(dir string) fs.FS { return dirFS(dir) } diff --git a/src/testing/fstest/mapfs.go b/src/testing/fstest/mapfs.go index 10e56f5b3c..a5d4a23fac 100644 --- a/src/testing/fstest/mapfs.go +++ b/src/testing/fstest/mapfs.go @@ -132,6 +132,16 @@ func (fsys MapFS) Glob(pattern string) ([]string, error) { return fs.Glob(fsOnly{fsys}, pattern) } +type noSub struct { + MapFS +} + +func (noSub) Sub() {} // not the fs.SubFS signature + +func (fsys MapFS) Sub(dir string) (fs.FS, error) { + return fs.Sub(noSub{fsys}, dir) +} + // A mapFileInfo implements fs.FileInfo and fs.DirEntry for a given map file. type mapFileInfo struct { name string diff --git a/src/testing/fstest/testfs.go b/src/testing/fstest/testfs.go index 4912a271b2..2602bdf0cc 100644 --- a/src/testing/fstest/testfs.go +++ b/src/testing/fstest/testfs.go @@ -22,6 +22,8 @@ import ( // It walks the entire tree of files in fsys, // opening and checking that each file behaves correctly. // It also checks that the file system contains at least the expected files. +// As a special case, if no expected files are listed, fsys must be empty. +// Otherwise, fsys must only contain at least the listed files: it can also contain others. // // If TestFS finds any misbehaviors, it returns an error reporting all of them. // The error text spans multiple lines, one per detected misbehavior. @@ -33,6 +35,32 @@ import ( // } // func TestFS(fsys fs.FS, expected ...string) error { + if err := testFS(fsys, expected...); err != nil { + return err + } + for _, name := range expected { + if i := strings.Index(name, "/"); i >= 0 { + dir, dirSlash := name[:i], name[:i+1] + var subExpected []string + for _, name := range expected { + if strings.HasPrefix(name, dirSlash) { + subExpected = append(subExpected, name[len(dirSlash):]) + } + } + sub, err := fs.Sub(fsys, dir) + if err != nil { + return err + } + if err := testFS(sub, subExpected...); err != nil { + return fmt.Errorf("testing fs.Sub(fsys, %s): %v", dir, err) + } + break // one sub-test is enough + } + } + return nil +} + +func testFS(fsys fs.FS, expected ...string) error { t := fsTester{fsys: fsys} t.checkDir(".") t.checkOpen(".") @@ -43,6 +71,20 @@ func TestFS(fsys fs.FS, expected ...string) error { for _, file := range t.files { found[file] = true } + delete(found, ".") + if len(expected) == 0 && len(found) > 0 { + var list []string + for k := range found { + if k != "." { + list = append(list, k) + } + } + sort.Strings(list) + if len(list) > 15 { + list = append(list[:10], "...") + } + t.errorf("expected empty file system but found files:\n%s", strings.Join(list, "\n")) + } for _, name := range expected { if !found[name] { t.errorf("expected but not found: %s", name) -- cgit v1.3