text/template/parse: simplify I/O in lexing

The concurrent model for delivering tokens was fine for pedagogy,
but has caused a few problems as the package has evolved (that is,
got more complicated). It's easy to eliminate it, simplifying or
removing some of the hacks used to work around these prolems.

The old lexer would deliver tokens over a channel to the parsing
goroutine, and continue running until EOF. In this rewrite, we
instead run the machine until a token is ready, and shut it down
until the next token is needed. The mechanism is just to return nil
as the state function, which requires a bit more threading of return
values through the state functions but is not difficult. The change
is modest.

A couple of error messages change, but otherwise the change has no
external effect. This is just an internal cleanup, long overdue.

benchmark                      old ns/op     new ns/op     delta
BenchmarkParseLarge-20         12222729      6769966       -44.61%
BenchmarkVariableString-20     73.5          73.4          -0.16%
BenchmarkListString-20         1827          1841          +0.77%

benchmark                      old allocs     new allocs     delta
BenchmarkVariableString-20     3              3              +0.00%
BenchmarkListString-20         31             31             +0.00%

benchmark                      old bytes     new bytes     delta
BenchmarkVariableString-20     72            72            +0.00%
BenchmarkListString-20         1473          1473          +0.00%

Fixes #53261

Change-Id: I4133bed2f8df16d398b707fb9509230325765c57
Reviewed-on: https://go-review.googlesource.com/c/go/+/421883
Reviewed-by: Austin Clements <austin@google.com>
Reviewed-by: Russ Cox <rsc@golang.org>

1 files changed, 9 insertions, 19 deletions

diff --git a/src/text/template/parse/lex_test.go b/src/text/template/parse/lex_test.go
index c5f429667c..947889a80b 100644
--- a/src/text/template/parse/lex_test.go
+++ b/src/text/template/parse/lex_test.go
@@ -359,8 +359,7 @@ var lexTests = []lexTest{
 	{"extra right paren", "{{3)}}", []item{
 		tLeft,
 		mkItem(itemNumber, "3"),
-		tRpar,
-		mkItem(itemError, `unexpected right paren U+0029 ')'`),
+		mkItem(itemError, "unexpected right paren"),
 	}},
 
 	// Fixed bugs
@@ -394,7 +393,12 @@ var lexTests = []lexTest{
 
 // collect gathers the emitted items into a slice.
 func collect(t *lexTest, left, right string) (items []item) {
-	l := lex(t.name, t.input, left, right, true, true, true)
+	l := lex(t.name, t.input, left, right)
+	l.options = lexOptions{
+		emitComment: true,
+		breakOK:     true,
+		continueOK:  true,
+	}
 	for {
 		item := l.nextItem()
 		items = append(items, item)
@@ -431,7 +435,9 @@ func TestLex(t *testing.T) {
 		items := collect(&test, "", "")
 		if !equal(items, test.items, false) {
 			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
+			return // TODO
 		}
+		t.Log(test.name, "OK")
 	}
 }
 
@@ -546,22 +552,6 @@ func TestPos(t *testing.T) {
 	}
 }
 
-// Test that an error shuts down the lexing goroutine.
-func TestShutdown(t *testing.T) {
-	// We need to duplicate template.Parse here to hold on to the lexer.
-	const text = "erroneous{{define}}{{else}}1234"
-	lexer := lex("foo", text, "{{", "}}", false, true, true)
-	_, err := New("root").parseLexer(lexer)
-	if err == nil {
-		t.Fatalf("expected error")
-	}
-	// The error should have drained the input. Therefore, the lexer should be shut down.
-	token, ok := <-lexer.items
-	if ok {
-		t.Fatalf("input was not drained; got %v", token)
-	}
-}
-
 // parseLexer is a local version of parse that lets us pass in the lexer instead of building it.
 // We expect an error, so the tree set and funcs list are explicitly nil.
 func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {