bufio: fix scanning with a final empty token.

The Scan function's interface to the split function was not sufficient
to handle an empty final token in a pure function; state was required.
This was ugly.

We introduce a special error value that a split function can return
that signals that this token is OK, but is the last one and scanning
should stop immediately _after_ this token.

The same effect could be achieved using the same trick (a special
error value) and checking for that error after Scan finishes, but it's
a little clumsy. Providing a published sentinel value in bufio is
cleaner and means everyone can use the same trick. The result
is an error-free scan.

Rewrite the test (that was only barely working) to use the value
and be more robust.

Also write a new example showing how to do it.

Fixes #11836

Change-Id: Iaae77d0f95b4a2efa0175ced94d93c66353079e8
Reviewed-on: https://go-review.googlesource.com/14924
Reviewed-by: Ian Lance Taylor <iant@golang.org>

1 files changed, 15 insertions, 11 deletions

diff --git a/src/bufio/scan_test.go b/src/bufio/scan_test.go
index ac65de9c44..07b1a56dc0 100644
--- a/src/bufio/scan_test.go
+++ b/src/bufio/scan_test.go
@@ -429,33 +429,37 @@ func commaSplit(data []byte, atEOF bool) (advance int, token []byte, err error)
 			return i + 1, data[:i], nil
 		}
 	}
-	if !atEOF {
-		return 0, nil, nil
-	}
-	return 0, data, nil
+	return 0, data, ErrFinalToken
 }
 
-func TestEmptyTokens(t *testing.T) {
-	s := NewScanner(strings.NewReader("1,2,3,"))
-	values := []string{"1", "2", "3", ""}
+func testEmptyTokens(t *testing.T, text string, values []string) {
+	s := NewScanner(strings.NewReader(text))
 	s.Split(commaSplit)
 	var i int
-	for i = 0; i < len(values); i++ {
-		if !s.Scan() {
-			break
+	for i = 0; s.Scan(); i++ {
+		if i >= len(values) {
+			t.Fatalf("got %d fields, expected %d", i+1, len(values))
 		}
 		if s.Text() != values[i] {
 			t.Errorf("%d: expected %q got %q", i, values[i], s.Text())
 		}
 	}
 	if i != len(values) {
-		t.Errorf("got %d fields, expected %d", i, len(values))
+		t.Fatalf("got %d fields, expected %d", i, len(values))
 	}
 	if err := s.Err(); err != nil {
 		t.Fatal(err)
 	}
 }
 
+func TestEmptyTokens(t *testing.T) {
+	testEmptyTokens(t, "1,2,3,", []string{"1", "2", "3", ""})
+}
+
+func TestWithNoEmptyTokens(t *testing.T) {
+	testEmptyTokens(t, "1,2,3", []string{"1", "2", "3"})
+}
+
 func loopAtEOFSplit(data []byte, atEOF bool) (advance int, token []byte, err error) {
 	if len(data) > 0 {
 		return 1, data[:1], nil