From 25d28ec55aded46e0be9c2298f24287d296a9e47 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 8 Jul 2020 17:13:16 -0400 Subject: cmd/go: add //go:embed support The final piece of //go:embed support: have the go command stitch together parsing in go/build, low-level data initialization in cmd/compile, and the new data structures in package embed, to make the //go:embed feature actually function. And test, now that all the pieces are available to work together. For #41191. (Issue not fixed: still need to add a tool for use by Bazel.) Change-Id: Ib1d198345c3b4d557d340f292eda13b984b65d65 Reviewed-on: https://go-review.googlesource.com/c/go/+/243945 Trust: Russ Cox Trust: Jay Conrod Trust: Johan Brandhorst Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Jay Conrod Reviewed-by: Johan Brandhorst --- src/embed/internal/embedtest/testdata/ascii.txt | 25 ++++++++++++++++++++++ src/embed/internal/embedtest/testdata/glass.txt | 1 + src/embed/internal/embedtest/testdata/hello.txt | 1 + src/embed/internal/embedtest/testdata/i/i18n.txt | 1 + .../internal/embedtest/testdata/i/j/k/k8s.txt | 1 + src/embed/internal/embedtest/testdata/ken.txt | 1 + 6 files changed, 30 insertions(+) create mode 100644 src/embed/internal/embedtest/testdata/ascii.txt create mode 100644 src/embed/internal/embedtest/testdata/glass.txt create mode 100644 src/embed/internal/embedtest/testdata/hello.txt create mode 100644 src/embed/internal/embedtest/testdata/i/i18n.txt create mode 100644 src/embed/internal/embedtest/testdata/i/j/k/k8s.txt create mode 100644 src/embed/internal/embedtest/testdata/ken.txt (limited to 'src/embed/internal/embedtest/testdata') diff --git a/src/embed/internal/embedtest/testdata/ascii.txt b/src/embed/internal/embedtest/testdata/ascii.txt new file mode 100644 index 0000000000..0cfebf6e9c --- /dev/null +++ b/src/embed/internal/embedtest/testdata/ascii.txt @@ -0,0 +1,25 @@ + !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmn +!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmno +"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnop +#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopq +$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqr +%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrs +&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrst +'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstu +()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuv +)*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw +*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwx ++,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxy +,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz +-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{ +./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{| +/0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} +0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} +123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} ! +23456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !" +3456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"# +456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"#$ +56789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"#$% +6789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"#$%& +789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"#$%&' +89:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"#$%&'( diff --git a/src/embed/internal/embedtest/testdata/glass.txt b/src/embed/internal/embedtest/testdata/glass.txt new file mode 100644 index 0000000000..8350baf437 --- /dev/null +++ b/src/embed/internal/embedtest/testdata/glass.txt @@ -0,0 +1 @@ +I can eat glass and it doesn't hurt me. diff --git a/src/embed/internal/embedtest/testdata/hello.txt b/src/embed/internal/embedtest/testdata/hello.txt new file mode 100644 index 0000000000..4b5fa63702 --- /dev/null +++ b/src/embed/internal/embedtest/testdata/hello.txt @@ -0,0 +1 @@ +hello, world diff --git a/src/embed/internal/embedtest/testdata/i/i18n.txt b/src/embed/internal/embedtest/testdata/i/i18n.txt new file mode 100644 index 0000000000..5ee27c63b6 --- /dev/null +++ b/src/embed/internal/embedtest/testdata/i/i18n.txt @@ -0,0 +1 @@ +internationalization diff --git a/src/embed/internal/embedtest/testdata/i/j/k/k8s.txt b/src/embed/internal/embedtest/testdata/i/j/k/k8s.txt new file mode 100644 index 0000000000..807e21be4c --- /dev/null +++ b/src/embed/internal/embedtest/testdata/i/j/k/k8s.txt @@ -0,0 +1 @@ +kubernetes diff --git a/src/embed/internal/embedtest/testdata/ken.txt b/src/embed/internal/embedtest/testdata/ken.txt new file mode 100644 index 0000000000..bb2598132e --- /dev/null +++ b/src/embed/internal/embedtest/testdata/ken.txt @@ -0,0 +1 @@ +If a program is too slow, it must have a loop. -- cgit v1.3 From 37588ffcb221c12c12882b591a16243ae2799fd1 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 2 Dec 2020 15:09:12 -0500 Subject: cmd/go, embed: exclude .* and _* from embedded directory trees Discussion on #42328 led to a decision to exclude files matching .* and _* from embedded directory results when embedding an entire directory tree. This CL implements that new behavior. Fixes #42328. Change-Id: I6188994e96348b3449c7d9d3d0d181cfbf2d4db1 Reviewed-on: https://go-review.googlesource.com/c/go/+/275092 Trust: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Ian Lance Taylor --- src/cmd/go/internal/load/pkg.go | 17 +++++++++++++---- src/embed/embed.go | 5 ++++- src/embed/internal/embedtest/embed_test.go | 21 +++++++++++++++++++++ .../embedtest/testdata/.hidden/.more/tip.txt | 1 + .../embedtest/testdata/.hidden/_more/tip.txt | 1 + .../internal/embedtest/testdata/.hidden/fortune.txt | 2 ++ .../embedtest/testdata/.hidden/more/tip.txt | 1 + .../internal/embedtest/testdata/_hidden/fortune.txt | 2 ++ 8 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 src/embed/internal/embedtest/testdata/.hidden/.more/tip.txt create mode 100644 src/embed/internal/embedtest/testdata/.hidden/_more/tip.txt create mode 100644 src/embed/internal/embedtest/testdata/.hidden/fortune.txt create mode 100644 src/embed/internal/embedtest/testdata/.hidden/more/tip.txt create mode 100644 src/embed/internal/embedtest/testdata/_hidden/fortune.txt (limited to 'src/embed/internal/embedtest/testdata') diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 30ca33b663..cbc683da2b 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -1998,6 +1998,16 @@ func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[stri return err } rel := filepath.ToSlash(path[len(p.Dir)+1:]) + name := info.Name() + if path != file && (isBadEmbedName(name) || name[0] == '.' || name[0] == '_') { + // Ignore bad names, assuming they won't go into modules. + // Also avoid hidden files that user may not know about. + // See golang.org/issue/42328. + if info.IsDir() { + return fs.SkipDir + } + return nil + } if info.IsDir() { if _, err := fsys.Stat(filepath.Join(path, "go.mod")); err == nil { return filepath.SkipDir @@ -2007,10 +2017,6 @@ func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[stri if !info.Mode().IsRegular() { return nil } - if isBadEmbedName(info.Name()) { - // Ignore bad names, assuming they won't go into modules. - return nil - } count++ if have[rel] != pid { have[rel] = pid @@ -2050,6 +2056,9 @@ func validEmbedPattern(pattern string) bool { // as existing for embedding. func isBadEmbedName(name string) bool { switch name { + // Empty string should be impossible but make it bad. + case "": + return true // Version control directories won't be present in module. case ".bzr", ".hg", ".git", ".svn": return true diff --git a/src/embed/embed.go b/src/embed/embed.go index b22975cc3a..29e0adf1a6 100644 --- a/src/embed/embed.go +++ b/src/embed/embed.go @@ -59,12 +59,15 @@ // as Go double-quoted or back-quoted string literals. // // If a pattern names a directory, all files in the subtree rooted at that directory are -// embedded (recursively), so the variable in the above example is equivalent to: +// embedded (recursively), except that files with names beginning with ‘.’ or ‘_’ +// are excluded. So the variable in the above example is almost equivalent to: // // // content is our static web server content. // //go:embed image template html/index.html // var content embed.FS // +// The difference is that ‘image/*’ embeds ‘image/.tempfile’ while ‘image’ does not. +// // The //go:embed directive can be used with both exported and unexported variables, // depending on whether the package wants to make the data available to other packages. // Similarly, it can be used with both global and function-local variables, diff --git a/src/embed/internal/embedtest/embed_test.go b/src/embed/internal/embedtest/embed_test.go index c82ca9fed2..b1707a4c04 100644 --- a/src/embed/internal/embedtest/embed_test.go +++ b/src/embed/internal/embedtest/embed_test.go @@ -101,3 +101,24 @@ func TestDir(t *testing.T) { testDir(t, all, "testdata/i/j", "k/") testDir(t, all, "testdata/i/j/k", "k8s.txt") } + +func TestHidden(t *testing.T) { + //go:embed testdata + var dir embed.FS + + //go:embed testdata/* + var star embed.FS + + t.Logf("//go:embed testdata") + + testDir(t, dir, "testdata", + "ascii.txt", "glass.txt", "hello.txt", "i/", "ken.txt") + + t.Logf("//go:embed testdata/*") + + testDir(t, star, "testdata", + ".hidden/", "_hidden/", "ascii.txt", "glass.txt", "hello.txt", "i/", "ken.txt") + + testDir(t, star, "testdata/.hidden", + "fortune.txt", "more/") // but not .more or _more +} diff --git a/src/embed/internal/embedtest/testdata/.hidden/.more/tip.txt b/src/embed/internal/embedtest/testdata/.hidden/.more/tip.txt new file mode 100644 index 0000000000..71b9c6955d --- /dev/null +++ b/src/embed/internal/embedtest/testdata/.hidden/.more/tip.txt @@ -0,0 +1 @@ +#define struct union /* Great space saver */ diff --git a/src/embed/internal/embedtest/testdata/.hidden/_more/tip.txt b/src/embed/internal/embedtest/testdata/.hidden/_more/tip.txt new file mode 100644 index 0000000000..71b9c6955d --- /dev/null +++ b/src/embed/internal/embedtest/testdata/.hidden/_more/tip.txt @@ -0,0 +1 @@ +#define struct union /* Great space saver */ diff --git a/src/embed/internal/embedtest/testdata/.hidden/fortune.txt b/src/embed/internal/embedtest/testdata/.hidden/fortune.txt new file mode 100644 index 0000000000..31f2013f94 --- /dev/null +++ b/src/embed/internal/embedtest/testdata/.hidden/fortune.txt @@ -0,0 +1,2 @@ +WARNING: terminal is not fully functional + - (press RETURN) diff --git a/src/embed/internal/embedtest/testdata/.hidden/more/tip.txt b/src/embed/internal/embedtest/testdata/.hidden/more/tip.txt new file mode 100644 index 0000000000..71b9c6955d --- /dev/null +++ b/src/embed/internal/embedtest/testdata/.hidden/more/tip.txt @@ -0,0 +1 @@ +#define struct union /* Great space saver */ diff --git a/src/embed/internal/embedtest/testdata/_hidden/fortune.txt b/src/embed/internal/embedtest/testdata/_hidden/fortune.txt new file mode 100644 index 0000000000..31f2013f94 --- /dev/null +++ b/src/embed/internal/embedtest/testdata/_hidden/fortune.txt @@ -0,0 +1,2 @@ +WARNING: terminal is not fully functional + - (press RETURN) -- cgit v1.3