text/template: accept new number syntax

This CL updates text/template's scanner to accept the
new number syntaxes:

 - Hexadecimal floating-point values.
 - Digit-separating underscores.
 - Leading 0b and 0o prefixes.

See golang.org/design/19308-number-literals for background.

For #12711.
For #19308.
For #28493.
For #29008.

Change-Id: I68c16ea35c3f506701063781388de72bafee6b8d
Reviewed-on: https://go-review.googlesource.com/c/160248
Reviewed-by: Rob Pike <r@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

4 files changed, 40 insertions, 7 deletions

diff --git a/src/text/template/parse/lex.go b/src/text/template/parse/lex.go
index 94a676c579..92b97f423f 100644
--- a/src/text/template/parse/lex.go
+++ b/src/text/template/parse/lex.go
@@ -565,17 +565,28 @@ func (l *lexer) scanNumber() bool {
 	// Optional leading sign.
 	l.accept("+-")
 	// Is it hex?
-	digits := "0123456789"
-	if l.accept("0") && l.accept("xX") {
-		digits = "0123456789abcdefABCDEF"
+	digits := "0123456789_"
+	if l.accept("0") {
+		// Note: Leading 0 does not mean octal in floats.
+		if l.accept("xX") {
+			digits = "0123456789abcdefABCDEF_"
+		} else if l.accept("oO") {
+			digits = "01234567_"
+		} else if l.accept("bB") {
+			digits = "01_"
+		}
 	}
 	l.acceptRun(digits)
 	if l.accept(".") {
 		l.acceptRun(digits)
 	}
-	if l.accept("eE") {
+	if len(digits) == 10+1 && l.accept("eE") {
+		l.accept("+-")
+		l.acceptRun("0123456789_")
+	}
+	if len(digits) == 16+6+1 && l.accept("pP") {
 		l.accept("+-")
-		l.acceptRun("0123456789")
+		l.acceptRun("0123456789_")
 	}
 	// Is it imaginary?
 	l.accept("i")
diff --git a/src/text/template/parse/lex_test.go b/src/text/template/parse/lex_test.go
index 6e7ece9db3..563c4fc1cb 100644
--- a/src/text/template/parse/lex_test.go
+++ b/src/text/template/parse/lex_test.go
@@ -120,7 +120,7 @@ var lexTests = []lexTest{
 	{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
 	{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
 	{"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
-	{"numbers", "{{1 02 0x14 -7.2i 1e3 +1.2e-4 4.2i 1+2i}}", []item{
+	{"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{
 		tLeft,
 		mkItem(itemNumber, "1"),
 		tSpace,
@@ -128,15 +128,25 @@ var lexTests = []lexTest{
 		tSpace,
 		mkItem(itemNumber, "0x14"),
 		tSpace,
+		mkItem(itemNumber, "0X14"),
+		tSpace,
 		mkItem(itemNumber, "-7.2i"),
 		tSpace,
 		mkItem(itemNumber, "1e3"),
 		tSpace,
+		mkItem(itemNumber, "1E3"),
+		tSpace,
 		mkItem(itemNumber, "+1.2e-4"),
 		tSpace,
 		mkItem(itemNumber, "4.2i"),
 		tSpace,
 		mkItem(itemComplex, "1+2i"),
+		tSpace,
+		mkItem(itemNumber, "1_2"),
+		tSpace,
+		mkItem(itemNumber, "0x1.e_fp4"),
+		tSpace,
+		mkItem(itemNumber, "0X1.E_FP4"),
 		tRight,
 		tEOF,
 	}},
diff --git a/src/text/template/parse/node.go b/src/text/template/parse/node.go
index dca83dacce..1174a4b970 100644
--- a/src/text/template/parse/node.go
+++ b/src/text/template/parse/node.go
@@ -596,7 +596,7 @@ func (t *Tree) newNumber(pos Pos, text string, typ itemType) (*NumberNode, error
 		if err == nil {
 			// If we parsed it as a float but it looks like an integer,
 			// it's a huge number too large to fit in an int. Reject it.
-			if !strings.ContainsAny(text, ".eE") {
+			if !strings.ContainsAny(text, ".eEpP") {
 				return nil, fmt.Errorf("integer overflow: %q", text)
 			}
 			n.IsFloat = true
diff --git a/src/text/template/parse/parse_test.go b/src/text/template/parse/parse_test.go
index 15cc65670a..5cb41d0bf5 100644
--- a/src/text/template/parse/parse_test.go
+++ b/src/text/template/parse/parse_test.go
@@ -30,8 +30,15 @@ var numberTests = []numberTest{
 	{"0", true, true, true, false, 0, 0, 0, 0},
 	{"-0", true, true, true, false, 0, 0, 0, 0}, // check that -0 is a uint.
 	{"73", true, true, true, false, 73, 73, 73, 0},
+	{"7_3", true, true, true, false, 73, 73, 73, 0},
+	{"0b10_010_01", true, true, true, false, 73, 73, 73, 0},
+	{"0B10_010_01", true, true, true, false, 73, 73, 73, 0},
 	{"073", true, true, true, false, 073, 073, 073, 0},
+	{"0o73", true, true, true, false, 073, 073, 073, 0},
+	{"0O73", true, true, true, false, 073, 073, 073, 0},
 	{"0x73", true, true, true, false, 0x73, 0x73, 0x73, 0},
+	{"0X73", true, true, true, false, 0x73, 0x73, 0x73, 0},
+	{"0x7_3", true, true, true, false, 0x73, 0x73, 0x73, 0},
 	{"-73", true, false, true, false, -73, 0, -73, 0},
 	{"+73", true, false, true, false, 73, 0, 73, 0},
 	{"100", true, true, true, false, 100, 100, 100, 0},
@@ -39,7 +46,12 @@ var numberTests = []numberTest{
 	{"-1e9", true, false, true, false, -1e9, 0, -1e9, 0},
 	{"-1.2", false, false, true, false, 0, 0, -1.2, 0},
 	{"1e19", false, true, true, false, 0, 1e19, 1e19, 0},
+	{"1e1_9", false, true, true, false, 0, 1e19, 1e19, 0},
+	{"1E19", false, true, true, false, 0, 1e19, 1e19, 0},
 	{"-1e19", false, false, true, false, 0, 0, -1e19, 0},
+	{"0x_1p4", true, true, true, false, 16, 16, 16, 0},
+	{"0X_1P4", true, true, true, false, 16, 16, 16, 0},
+	{"0x_1p-4", false, false, true, false, 0, 0, 1 / 16., 0},
 	{"4i", false, false, false, true, 0, 0, 0, 4i},
 	{"-1.2+4.2i", false, false, false, true, 0, 0, 0, -1.2 + 4.2i},
 	{"073i", false, false, false, true, 0, 0, 0, 73i}, // not octal!