bufio: fix handling of empty tokens at end of line/file

Fixes #8672.

LGTM=rsc
R=golang-codereviews, rsc
CC=golang-codereviews
https://golang.org/cl/145390043

1 files changed, 36 insertions, 0 deletions

diff --git a/src/bufio/scan_test.go b/src/bufio/scan_test.go
index 3ddb25acf9..1454a8113c 100644
--- a/src/bufio/scan_test.go
+++ b/src/bufio/scan_test.go
@@ -419,3 +419,39 @@ func TestScanWordsExcessiveWhiteSpace(t *testing.T) {
 		t.Fatalf("unexpected token: %v", token)
 	}
 }
+
+// Test that empty tokens, including at end of line or end of file, are found by the scanner.
+// Issue 8672: Could miss final empty token.
+
+func commaSplit(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	for i := 0; i < len(data); i++ {
+		if data[i] == ',' {
+			return i + 1, data[:i], nil
+		}
+	}
+	if !atEOF {
+		return 0, nil, nil
+	}
+	return 0, data, nil
+}
+
+func TestEmptyTokens(t *testing.T) {
+	s := NewScanner(strings.NewReader("1,2,3,"))
+	values := []string{"1", "2", "3", ""}
+	s.Split(commaSplit)
+	var i int
+	for i = 0; i < len(values); i++ {
+		if !s.Scan() {
+			break
+		}
+		if s.Text() != values[i] {
+			t.Errorf("%d: expected %q got %q", i, values[i], s.Text())
+		}
+	}
+	if i != len(values) {
+		t.Errorf("got %d fields, expected %d", i, len(values))
+	}
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+}