cmd/go: add //go:embed support

The final piece of //go:embed support: have the go command stitch
together parsing in go/build, low-level data initialization in cmd/compile,
and the new data structures in package embed, to make the //go:embed
feature actually function.

And test, now that all the pieces are available to work together.

For #41191.
(Issue not fixed: still need to add a tool for use by Bazel.)

Change-Id: Ib1d198345c3b4d557d340f292eda13b984b65d65
Reviewed-on: https://go-review.googlesource.com/c/go/+/243945
Trust: Russ Cox <rsc@golang.org>
Trust: Jay Conrod <jayconrod@google.com>
Trust: Johan Brandhorst <johan.brandhorst@gmail.com>
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>
Reviewed-by: Johan Brandhorst <johan.brandhorst@gmail.com>

9 files changed, 240 insertions, 0 deletions

diff --git a/src/embed/internal/embedtest/concurrency.txt b/src/embed/internal/embedtest/concurrency.txt
new file mode 100644
index 0000000000..0814741261
--- /dev/null
+++ b/src/embed/internal/embedtest/concurrency.txt
@@ -0,0 +1 @@
+Concurrency is not parallelism.
diff --git a/src/embed/internal/embedtest/embed_test.go b/src/embed/internal/embedtest/embed_test.go
new file mode 100644
index 0000000000..c82ca9fed2
--- /dev/null
+++ b/src/embed/internal/embedtest/embed_test.go
@@ -0,0 +1,103 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package embedtest
+
+import (
+	"embed"
+	"reflect"
+	"testing"
+	"testing/fstest"
+)
+
+//go:embed testdata/h*.txt
+//go:embed c*.txt testdata/g*.txt
+var global embed.FS
+
+//go:embed c*txt
+var concurrency string
+
+//go:embed testdata/g*.txt
+var glass []byte
+
+func testFiles(t *testing.T, f embed.FS, name, data string) {
+	t.Helper()
+	d, err := f.ReadFile(name)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if string(d) != data {
+		t.Errorf("read %v = %q, want %q", name, d, data)
+	}
+}
+
+func testString(t *testing.T, s, name, data string) {
+	t.Helper()
+	if s != data {
+		t.Errorf("%v = %q, want %q", name, s, data)
+	}
+}
+
+func testDir(t *testing.T, f embed.FS, name string, expect ...string) {
+	t.Helper()
+	dirs, err := f.ReadDir(name)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	var names []string
+	for _, d := range dirs {
+		name := d.Name()
+		if d.IsDir() {
+			name += "/"
+		}
+		names = append(names, name)
+	}
+	if !reflect.DeepEqual(names, expect) {
+		t.Errorf("readdir %v = %v, want %v", name, names, expect)
+	}
+}
+
+func TestGlobal(t *testing.T) {
+	testFiles(t, global, "concurrency.txt", "Concurrency is not parallelism.\n")
+	testFiles(t, global, "testdata/hello.txt", "hello, world\n")
+	testFiles(t, global, "testdata/glass.txt", "I can eat glass and it doesn't hurt me.\n")
+
+	if err := fstest.TestFS(global); err != nil {
+		t.Fatal(err)
+	}
+
+	testString(t, concurrency, "concurrency", "Concurrency is not parallelism.\n")
+	testString(t, string(glass), "glass", "I can eat glass and it doesn't hurt me.\n")
+}
+
+func TestLocal(t *testing.T) {
+	//go:embed testdata/k*.txt
+	var local embed.FS
+	testFiles(t, local, "testdata/ken.txt", "If a program is too slow, it must have a loop.\n")
+
+	//go:embed testdata/k*.txt
+	var s string
+	testString(t, s, "local variable s", "If a program is too slow, it must have a loop.\n")
+
+	//go:embed testdata/h*.txt
+	var b []byte
+	testString(t, string(b), "local variable b", "hello, world\n")
+}
+
+func TestDir(t *testing.T) {
+	//go:embed testdata
+	var all embed.FS
+
+	testFiles(t, all, "testdata/hello.txt", "hello, world\n")
+	testFiles(t, all, "testdata/i/i18n.txt", "internationalization\n")
+	testFiles(t, all, "testdata/i/j/k/k8s.txt", "kubernetes\n")
+	testFiles(t, all, "testdata/ken.txt", "If a program is too slow, it must have a loop.\n")
+
+	testDir(t, all, ".", "testdata/")
+	testDir(t, all, "testdata/i", "i18n.txt", "j/")
+	testDir(t, all, "testdata/i/j", "k/")
+	testDir(t, all, "testdata/i/j/k", "k8s.txt")
+}
diff --git a/src/embed/internal/embedtest/embedx_test.go b/src/embed/internal/embedtest/embedx_test.go
new file mode 100644
index 0000000000..53d45488f1
--- /dev/null
+++ b/src/embed/internal/embedtest/embedx_test.go
@@ -0,0 +1,106 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package embedtest_test
+
+import (
+	"embed"
+	"io/ioutil"
+	"testing"
+)
+
+var (
+	global2      = global
+	concurrency2 = concurrency
+	glass2       = glass
+	sbig2        = sbig
+	bbig2        = bbig
+)
+
+//go:embed testdata/*.txt
+var global embed.FS
+
+//go:embed c*txt
+var concurrency string
+
+//go:embed testdata/g*.txt
+var glass []byte
+
+//go:embed testdata/ascii.txt
+var sbig string
+
+//go:embed testdata/ascii.txt
+var bbig []byte
+
+func testFiles(t *testing.T, f embed.FS, name, data string) {
+	t.Helper()
+	d, err := f.ReadFile(name)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if string(d) != data {
+		t.Errorf("read %v = %q, want %q", name, d, data)
+	}
+}
+
+func testString(t *testing.T, s, name, data string) {
+	t.Helper()
+	if s != data {
+		t.Errorf("%v = %q, want %q", name, s, data)
+	}
+}
+
+func TestXGlobal(t *testing.T) {
+	testFiles(t, global, "testdata/hello.txt", "hello, world\n")
+	testString(t, concurrency, "concurrency", "Concurrency is not parallelism.\n")
+	testString(t, string(glass), "glass", "I can eat glass and it doesn't hurt me.\n")
+	testString(t, concurrency2, "concurrency2", "Concurrency is not parallelism.\n")
+	testString(t, string(glass2), "glass2", "I can eat glass and it doesn't hurt me.\n")
+
+	big, err := ioutil.ReadFile("testdata/ascii.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	testString(t, sbig, "sbig", string(big))
+	testString(t, sbig2, "sbig2", string(big))
+	testString(t, string(bbig), "bbig", string(big))
+	testString(t, string(bbig2), "bbig", string(big))
+
+	if t.Failed() {
+		return
+	}
+
+	// Could check &glass[0] == &glass2[0] but also want to make sure write does not fault
+	// (data must not be in read-only memory).
+	old := glass[0]
+	glass[0]++
+	if glass2[0] != glass[0] {
+		t.Fatalf("glass and glass2 do not share storage")
+	}
+	glass[0] = old
+
+	// Could check &bbig[0] == &bbig2[0] but also want to make sure write does not fault
+	// (data must not be in read-only memory).
+	old = bbig[0]
+	bbig[0]++
+	if bbig2[0] != bbig[0] {
+		t.Fatalf("bbig and bbig2 do not share storage")
+	}
+	bbig[0] = old
+}
+
+func TestXLocal(t *testing.T) {
+	//go:embed testdata/*o.txt
+	var local embed.FS
+	testFiles(t, local, "testdata/hello.txt", "hello, world\n")
+
+	//go:embed testdata/k*.txt
+	var s string
+	testString(t, s, "local variable s", "If a program is too slow, it must have a loop.\n")
+
+	//go:embed testdata/h*.txt
+	var b []byte
+	testString(t, string(b), "local variable b", "hello, world\n")
+}
diff --git a/src/embed/internal/embedtest/testdata/ascii.txt b/src/embed/internal/embedtest/testdata/ascii.txt
new file mode 100644
index 0000000000..0cfebf6e9c
--- /dev/null
+++ b/src/embed/internal/embedtest/testdata/ascii.txt
@@ -0,0 +1,25 @@
+ !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmn
+!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmno
+"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnop
+#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopq
+$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqr
+%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrs
+&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrst
+'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstu
+()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuv
+)*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw
+*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwx
++,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxy
+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz
+-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{
+./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|
+/0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}
+0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} 
+123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !
+23456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"
+3456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"#
+456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"#$
+56789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"#$%
+6789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"#$%&
+789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"#$%&'
+89:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|} !"#$%&'(
diff --git a/src/embed/internal/embedtest/testdata/glass.txt b/src/embed/internal/embedtest/testdata/glass.txt
new file mode 100644
index 0000000000..8350baf437
--- /dev/null
+++ b/src/embed/internal/embedtest/testdata/glass.txt
@@ -0,0 +1 @@
+I can eat glass and it doesn't hurt me.
diff --git a/src/embed/internal/embedtest/testdata/hello.txt b/src/embed/internal/embedtest/testdata/hello.txt
new file mode 100644
index 0000000000..4b5fa63702
--- /dev/null
+++ b/src/embed/internal/embedtest/testdata/hello.txt
@@ -0,0 +1 @@
+hello, world
diff --git a/src/embed/internal/embedtest/testdata/i/i18n.txt b/src/embed/internal/embedtest/testdata/i/i18n.txt
new file mode 100644
index 0000000000..5ee27c63b6
--- /dev/null
+++ b/src/embed/internal/embedtest/testdata/i/i18n.txt
@@ -0,0 +1 @@
+internationalization
diff --git a/src/embed/internal/embedtest/testdata/i/j/k/k8s.txt b/src/embed/internal/embedtest/testdata/i/j/k/k8s.txt
new file mode 100644
index 0000000000..807e21be4c
--- /dev/null
+++ b/src/embed/internal/embedtest/testdata/i/j/k/k8s.txt
@@ -0,0 +1 @@
+kubernetes
diff --git a/src/embed/internal/embedtest/testdata/ken.txt b/src/embed/internal/embedtest/testdata/ken.txt
new file mode 100644
index 0000000000..bb2598132e
--- /dev/null
+++ b/src/embed/internal/embedtest/testdata/ken.txt
@@ -0,0 +1 @@
+If a program is too slow, it must have a loop.