cmd/compile: handle pragmas immediately with -newparser=1

Instead of saving all pragmas and processing them after parsing is
finished, process them immediately during scanning like the current
lexer does.

This is a bit unfortunate because it means we can't use
syntax.ParseFile to concurrently parse files yet, but it fixes how we
report syntax errors in the presence of //line pragmas.

While here, add a bunch more gcCompat entries to syntax/parser.go to
get "go build -toolexec='toolstash -cmp' std cmd" passing. There are
still a few remaining cases only triggered building unit tests, but
this seems like a nice checkpoint.

Change-Id: Iaf3bbcf2849857a460496f31eea228e0c585ce13
Reviewed-on: https://go-review.googlesource.com/28226
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

8 files changed, 86 insertions, 30 deletions

diff --git a/src/cmd/compile/internal/syntax/dumper_test.go b/src/cmd/compile/internal/syntax/dumper_test.go
index fd38e7ca78..1ee1d982d0 100644
--- a/src/cmd/compile/internal/syntax/dumper_test.go
+++ b/src/cmd/compile/internal/syntax/dumper_test.go
@@ -14,7 +14,7 @@ func TestDump(t *testing.T) {
 		t.Skip("skipping test in short mode")
 	}
 
-	ast, err := ReadFile(*src, nil, 0)
+	ast, err := ReadFile(*src, nil, nil, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/src/cmd/compile/internal/syntax/nodes.go b/src/cmd/compile/internal/syntax/nodes.go
index e56b1235fe..280a2e8f69 100644
--- a/src/cmd/compile/internal/syntax/nodes.go
+++ b/src/cmd/compile/internal/syntax/nodes.go
@@ -10,6 +10,7 @@ package syntax
 type Node interface {
 	Line() uint32
 	aNode()
+	init(p *parser)
 }
 
 type node struct {
@@ -35,16 +36,10 @@ func (n *node) init(p *parser) {
 type File struct {
 	PkgName  *Name
 	DeclList []Decl
-	Pragmas  []Pragma
 	Lines    int
 	node
 }
 
-type Pragma struct {
-	Line int
-	Text string
-}
-
 // ----------------------------------------------------------------------------
 // Declarations
 
@@ -90,6 +85,7 @@ type (
 		Name    *Name
 		Type    *FuncType
 		Body    []Stmt // nil means no body (forward declaration)
+		Pragma  Pragma // TODO(mdempsky): Cleaner solution.
 		EndLine uint32 // TODO(mdempsky): Cleaner solution.
 		decl
 	}
@@ -130,7 +126,8 @@ type (
 	CompositeLit struct {
 		Type     Expr // nil means no literal type
 		ElemList []Expr
-		NKeys    int // number of elements with keys
+		NKeys    int    // number of elements with keys
+		EndLine  uint32 // TODO(mdempsky): Cleaner solution.
 		expr
 	}
 
diff --git a/src/cmd/compile/internal/syntax/parser.go b/src/cmd/compile/internal/syntax/parser.go
index 9544001a2e..ae9c14c811 100644
--- a/src/cmd/compile/internal/syntax/parser.go
+++ b/src/cmd/compile/internal/syntax/parser.go
@@ -28,7 +28,7 @@ type parser struct {
 	nerrors int // error count
 }
 
-func (p *parser) init(src io.Reader, errh ErrorHandler) {
+func (p *parser) init(src io.Reader, errh ErrorHandler, pragh PragmaHandler) {
 	p.scanner.init(src, func(pos, line int, msg string) {
 		p.nerrors++
 		if !debug && errh != nil {
@@ -36,7 +36,7 @@ func (p *parser) init(src io.Reader, errh ErrorHandler) {
 			return
 		}
 		panic(fmt.Sprintf("%d: %s\n", line, msg))
-	})
+	}, pragh)
 
 	p.fnest = 0
 	p.xnest = 0
@@ -245,6 +245,10 @@ func (p *parser) file() *File {
 			continue
 		}
 
+		// Reset p.pragma BEFORE advancing to the next token (consuming ';')
+		// since comments before may set pragmas for the next function decl.
+		p.pragma = 0
+
 		if p.tok != _EOF && !p.got(_Semi) {
 			p.syntax_error("after top level declaration")
 			p.advance(_Const, _Type, _Var, _Func)
@@ -253,7 +257,6 @@ func (p *parser) file() *File {
 	// p.tok == _EOF
 
 	f.Lines = p.source.line
-	f.Pragmas = p.pragmas
 
 	return f
 }
@@ -372,6 +375,9 @@ func (p *parser) varDecl(group *Group) Decl {
 		}
 	}
 	d.Group = group
+	if gcCompat {
+		d.init(p)
+	}
 
 	return d
 }
@@ -426,8 +432,12 @@ func (p *parser) funcDecl() *FuncDecl {
 
 	f.Name = p.name()
 	f.Type = p.funcType()
+	if gcCompat {
+		f.node = f.Type.node
+	}
 	f.Body = p.funcBody()
 
+	f.Pragma = p.pragma
 	f.EndLine = uint32(p.line)
 
 	// TODO(gri) deal with function properties
@@ -465,6 +475,9 @@ func (p *parser) binaryExpr(prec int) Expr {
 		tprec := p.prec
 		p.next()
 		t.Y = p.binaryExpr(tprec)
+		if gcCompat {
+			t.init(p)
+		}
 		x = t
 	}
 	return x
@@ -485,6 +498,9 @@ func (p *parser) unaryExpr() Expr {
 			x.Op = p.op
 			p.next()
 			x.X = p.unaryExpr()
+			if gcCompat {
+				x.init(p)
+			}
 			return x
 
 		case And:
@@ -730,6 +746,9 @@ loop:
 				p.syntax_error("expecting name or (")
 				p.advance(_Semi, _Rparen)
 			}
+			if gcCompat {
+				x.init(p)
+			}
 
 		case _Lbrack:
 			p.next()
@@ -851,6 +870,9 @@ func (p *parser) complitexpr() *CompositeLit {
 			l.init(p)
 			l.Key = e
 			l.Value = p.bare_complitexpr()
+			if gcCompat {
+				l.init(p)
+			}
 			e = l
 			x.NKeys++
 		}
@@ -860,6 +882,7 @@ func (p *parser) complitexpr() *CompositeLit {
 		}
 	}
 
+	x.EndLine = uint32(p.line)
 	p.xnest--
 	p.want(_Rbrace)
 
@@ -996,6 +1019,9 @@ func (p *parser) funcType() *FuncType {
 	typ.init(p)
 	typ.ParamList = p.paramList()
 	typ.ResultList = p.funcResult()
+	if gcCompat {
+		typ.init(p)
+	}
 	return typ
 }
 
@@ -1134,6 +1160,10 @@ func (p *parser) addField(styp *StructType, name *Name, typ Expr, tag *BasicLit)
 	f.Type = typ
 	styp.FieldList = append(styp.FieldList, f)
 
+	if gcCompat && name != nil {
+		f.node = name.node
+	}
+
 	if debug && tag != nil && len(styp.FieldList) != len(styp.TagList) {
 		panic("inconsistent struct field list")
 	}
@@ -1443,6 +1473,9 @@ func (p *parser) simpleStmt(lhs Expr, rangeOk bool) SimpleStmt {
 			s.init(p)
 			s.Chan = lhs
 			s.Value = p.expr()
+			if gcCompat {
+				s.init(p)
+			}
 			return s
 
 		default:
@@ -1509,6 +1542,9 @@ func (p *parser) rangeClause(lhs Expr, def bool) *RangeClause {
 	r.Lhs = lhs
 	r.Def = def
 	r.X = p.expr()
+	if gcCompat {
+		r.init(p)
+	}
 	return r
 }
 
@@ -1583,6 +1619,9 @@ func (p *parser) forStmt() Stmt {
 
 	p.want(_For)
 	s.Init, s.Cond, s.Post = p.header(true)
+	if gcCompat {
+		s.init(p)
+	}
 	s.Body = p.stmtBody("for clause")
 
 	return s
@@ -1672,6 +1711,10 @@ func (p *parser) ifStmt() *IfStmt {
 		p.error("missing condition in if statement")
 	}
 
+	if gcCompat {
+		s.init(p)
+	}
+
 	s.Then = p.stmtBody("if clause")
 
 	if p.got(_Else) {
@@ -1914,6 +1957,9 @@ func (p *parser) stmt() Stmt {
 		if p.tok != _Semi && p.tok != _Rbrace {
 			s.Results = p.exprList()
 		}
+		if gcCompat {
+			s.init(p)
+		}
 		return s
 
 	case _Semi:
diff --git a/src/cmd/compile/internal/syntax/parser_test.go b/src/cmd/compile/internal/syntax/parser_test.go
index 12fc019414..8e6b77d0c6 100644
--- a/src/cmd/compile/internal/syntax/parser_test.go
+++ b/src/cmd/compile/internal/syntax/parser_test.go
@@ -22,7 +22,7 @@ var src = flag.String("src", "parser.go", "source file to parse")
 var verify = flag.Bool("verify", false, "verify idempotent printing")
 
 func TestParse(t *testing.T) {
-	_, err := ReadFile(*src, nil, 0)
+	_, err := ReadFile(*src, nil, nil, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -52,7 +52,7 @@ func TestStdLib(t *testing.T) {
 				if debug {
 					fmt.Printf("parsing %s\n", filename)
 				}
-				ast, err := ReadFile(filename, nil, 0)
+				ast, err := ReadFile(filename, nil, nil, 0)
 				if err != nil {
 					t.Fatal(err)
 				}
@@ -133,7 +133,7 @@ func verifyPrint(filename string, ast1 *File) {
 		panic(err)
 	}
 
-	ast2, err := ReadBytes(buf1.Bytes(), nil, 0)
+	ast2, err := ReadBytes(buf1.Bytes(), nil, nil, 0)
 	if err != nil {
 		panic(err)
 	}
diff --git a/src/cmd/compile/internal/syntax/printer_test.go b/src/cmd/compile/internal/syntax/printer_test.go
index e8c2201e60..a2d43068dd 100644
--- a/src/cmd/compile/internal/syntax/printer_test.go
+++ b/src/cmd/compile/internal/syntax/printer_test.go
@@ -15,7 +15,7 @@ func TestPrint(t *testing.T) {
 		t.Skip("skipping test in short mode")
 	}
 
-	ast, err := ReadFile(*src, nil, 0)
+	ast, err := ReadFile(*src, nil, nil, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/src/cmd/compile/internal/syntax/scanner.go b/src/cmd/compile/internal/syntax/scanner.go
index d02bb6d11b..e78950ad1a 100644
--- a/src/cmd/compile/internal/syntax/scanner.go
+++ b/src/cmd/compile/internal/syntax/scanner.go
@@ -15,6 +15,7 @@ import (
 type scanner struct {
 	source
 	nlsemi bool // if set '\n' and EOF translate to ';'
+	pragma Pragma
 
 	// current token, valid after calling next()
 	pos, line int
@@ -24,12 +25,13 @@ type scanner struct {
 	op        Operator // valid if tok is _Operator, _AssignOp, or _IncOp
 	prec      int      // valid if tok is _Operator, _AssignOp, or _IncOp
 
-	pragmas []Pragma
+	pragh PragmaHandler
 }
 
-func (s *scanner) init(src io.Reader, errh ErrorHandler) {
+func (s *scanner) init(src io.Reader, errh ErrorHandler, pragh PragmaHandler) {
 	s.source.init(src, errh)
 	s.nlsemi = false
+	s.pragh = pragh
 }
 
 func (s *scanner) next() {
@@ -540,6 +542,10 @@ func (s *scanner) lineComment() {
 	// recognize pragmas
 	var prefix string
 	r := s.getr()
+	if s.pragh == nil {
+		goto skip
+	}
+
 	switch r {
 	case 'g':
 		prefix = "go:"
@@ -565,10 +571,7 @@ func (s *scanner) lineComment() {
 		}
 		r = s.getr()
 	}
-	s.pragmas = append(s.pragmas, Pragma{
-		Line: s.line,
-		Text: strings.TrimSuffix(string(s.stopLit()), "\r"),
-	})
+	s.pragma |= s.pragh(0, s.line, strings.TrimSuffix(string(s.stopLit()), "\r"))
 	return
 
 skip:
diff --git a/src/cmd/compile/internal/syntax/scanner_test.go b/src/cmd/compile/internal/syntax/scanner_test.go
index 70b986ed4e..38a7e0da4c 100644
--- a/src/cmd/compile/internal/syntax/scanner_test.go
+++ b/src/cmd/compile/internal/syntax/scanner_test.go
@@ -22,7 +22,7 @@ func TestScanner(t *testing.T) {
 	defer src.Close()
 
 	var s scanner
-	s.init(src, nil)
+	s.init(src, nil, nil)
 	for {
 		s.next()
 		if s.tok == _EOF {
@@ -51,7 +51,7 @@ func TestTokens(t *testing.T) {
 
 	// scan source
 	var got scanner
-	got.init(&bytesReader{buf}, nil)
+	got.init(&bytesReader{buf}, nil, nil)
 	got.next()
 	for i, want := range sampleTokens {
 		nlsemi := false
@@ -338,7 +338,7 @@ func TestScanErrors(t *testing.T) {
 			} else if nerrors > 1 {
 				t.Errorf("%q: got unexpected %q at pos = %d, line = %d", test.src, msg, pos, line)
 			}
-		})
+		}, nil)
 
 		for {
 			s.next()
diff --git a/src/cmd/compile/internal/syntax/syntax.go b/src/cmd/compile/internal/syntax/syntax.go
index 85dddaa47f..6c0abd118d 100644
--- a/src/cmd/compile/internal/syntax/syntax.go
+++ b/src/cmd/compile/internal/syntax/syntax.go
@@ -12,17 +12,27 @@ import (
 
 type Mode uint
 
+// A Pragma value is a set of flags that augment a function
+// declaration. Callers may assign meaning to the flags as
+// appropriate.
+type Pragma uint16
+
 type ErrorHandler func(pos, line int, msg string)
 
+// A PragmaHandler is used to process //line and //go: directives as
+// they're scanned. The returned Pragma value will be unioned into the
+// next FuncDecl node.
+type PragmaHandler func(pos, line int, text string) Pragma
+
 // TODO(gri) These need a lot more work.
 
-func ReadFile(filename string, errh ErrorHandler, mode Mode) (*File, error) {
+func ReadFile(filename string, errh ErrorHandler, pragh PragmaHandler, mode Mode) (*File, error) {
 	src, err := os.Open(filename)
 	if err != nil {
 		return nil, err
 	}
 	defer src.Close()
-	return Read(src, errh, mode)
+	return Read(src, errh, pragh, mode)
 }
 
 type bytesReader struct {
@@ -38,13 +48,13 @@ func (r *bytesReader) Read(p []byte) (int, error) {
 	return 0, io.EOF
 }
 
-func ReadBytes(src []byte, errh ErrorHandler, mode Mode) (*File, error) {
-	return Read(&bytesReader{src}, errh, mode)
+func ReadBytes(src []byte, errh ErrorHandler, pragh PragmaHandler, mode Mode) (*File, error) {
+	return Read(&bytesReader{src}, errh, pragh, mode)
 }
 
-func Read(src io.Reader, errh ErrorHandler, mode Mode) (*File, error) {
+func Read(src io.Reader, errh ErrorHandler, pragh PragmaHandler, mode Mode) (*File, error) {
 	var p parser
-	p.init(src, errh)
+	p.init(src, errh, pragh)
 
 	p.next()
 	ast := p.file()