unicode: use new Scanner interface in table creation

Update norm and local/collate as well.

R=mpvl
CC=golang-dev
https://golang.org/cl/7395045

4 files changed, 38 insertions, 61 deletions

diff --git a/src/pkg/exp/locale/collate/maketables.go b/src/pkg/exp/locale/collate/maketables.go
index 19f176e9ef..86feaf2886 100644
--- a/src/pkg/exp/locale/collate/maketables.go
+++ b/src/pkg/exp/locale/collate/maketables.go
@@ -236,18 +236,10 @@ func parseUCA(builder *build.Builder) {
 	}
 	failOnError(err)
 	defer r.Close()
-	input := bufio.NewReader(r)
+	scanner := bufio.NewScanner(r)
 	colelem := regexp.MustCompile(`\[([.*])([0-9A-F.]+)\]`)
-	for i := 1; err == nil; i++ {
-		l, prefix, e := input.ReadLine()
-		err = e
-		line := string(l)
-		if prefix {
-			log.Fatalf("%d: buffer overflow", i)
-		}
-		if err != nil && err != io.EOF {
-			log.Fatalf("%d: %v", i, err)
-		}
+	for i := 1; scanner.Scan(); i++ {
+		line := scanner.Text()
 		if len(line) == 0 || line[0] == '#' {
 			continue
 		}
@@ -300,6 +292,9 @@ func parseUCA(builder *build.Builder) {
 			failOnError(builder.Add(lhs, rhs, vars))
 		}
 	}
+	if scanner.Err() != nil {
+		log.Fatal(scanner.Err())
+	}
 }
 
 func convHex(line int, s string) int {
diff --git a/src/pkg/exp/locale/collate/regtest.go b/src/pkg/exp/locale/collate/regtest.go
index fd4fc6eb34..55c81f5738 100644
--- a/src/pkg/exp/locale/collate/regtest.go
+++ b/src/pkg/exp/locale/collate/regtest.go
@@ -166,16 +166,10 @@ func loadTestData() []Test {
 		ff, err := f.Open()
 		Error(err)
 		defer ff.Close()
-		input := bufio.NewReader(ff)
+		scanner := bufio.NewScanner(ff)
 		test := Test{name: path.Base(f.Name)}
-		for {
-			line, err := input.ReadString('\n')
-			if err != nil {
-				if err == io.EOF {
-					break
-				}
-				log.Fatal(err)
-			}
+		for scanner.Scan() {
+			line := scanner.Text()
 			if len(line) <= 1 || line[0] == '#' {
 				if m := versionRe.FindStringSubmatch(line); m != nil {
 					if m[1] != unicode.Version {
@@ -206,6 +200,9 @@ func loadTestData() []Test {
 				test.comment = append(test.comment, m[2])
 			}
 		}
+		if scanner.Err() != nil {
+			log.Fatal(scanner.Err())
+		}
 		tests = append(tests, test)
 	}
 	return tests
diff --git a/src/pkg/exp/norm/maketables.go b/src/pkg/exp/norm/maketables.go
index 03e1e2e33f..50c0c310a8 100644
--- a/src/pkg/exp/norm/maketables.go
+++ b/src/pkg/exp/norm/maketables.go
@@ -307,16 +307,12 @@ func parseCharacter(line string) {
 func loadUnicodeData() {
 	f := openReader("UnicodeData.txt")
 	defer f.Close()
-	input := bufio.NewReader(f)
-	for {
-		line, err := input.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			logger.Fatal(err)
-		}
-		parseCharacter(line[0 : len(line)-1])
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		parseCharacter(scanner.Text())
+	}
+	if scanner.Err() != nil {
+		logger.Fatal(scanner.Err())
 	}
 }
 
@@ -347,16 +343,9 @@ func parseExclusion(line string) int {
 func loadCompositionExclusions() {
 	f := openReader("CompositionExclusions.txt")
 	defer f.Close()
-	input := bufio.NewReader(f)
-	for {
-		line, err := input.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			logger.Fatal(err)
-		}
-		point := parseExclusion(line[0 : len(line)-1])
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		point := parseExclusion(scanner.Text())
 		if point == 0 {
 			continue
 		}
@@ -366,6 +355,9 @@ func loadCompositionExclusions() {
 		}
 		c.excludeInComp = true
 	}
+	if scanner.Err() != nil {
+		log.Fatal(scanner.Err())
+	}
 }
 
 // hasCompatDecomp returns true if any of the recursive
@@ -853,15 +845,9 @@ func testDerived() {
 	}
 	f := openReader("DerivedNormalizationProps.txt")
 	defer f.Close()
-	input := bufio.NewReader(f)
-	for {
-		line, err := input.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			logger.Fatal(err)
-		}
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
 		qc := qcRe.FindStringSubmatch(line)
 		if qc == nil {
 			continue
@@ -920,6 +906,9 @@ func testDerived() {
 			}
 		}
 	}
+	if scanner.Err() != nil {
+		logger.Fatal(scanner.Err())
+	}
 	// Any unspecified value must be QCYes. Verify this.
 	for i, c := range chars {
 		for j, fd := range c.forms {
diff --git a/src/pkg/exp/norm/normregtest.go b/src/pkg/exp/norm/normregtest.go
index d7ef9ba3ec..b77b5b545a 100644
--- a/src/pkg/exp/norm/normregtest.go
+++ b/src/pkg/exp/norm/normregtest.go
@@ -12,7 +12,6 @@ import (
 	"exp/norm"
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"net/http"
 	"os"
@@ -118,8 +117,8 @@ func (t Test) Name() string {
 	return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number)
 }
 
-var partRe = regexp.MustCompile(`@Part(\d) # (.*)\n$`)
-var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)\n?$`)
+var partRe = regexp.MustCompile(`@Part(\d) # (.*)$`)
+var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)$`)
 
 var counter int
 
@@ -141,15 +140,9 @@ func loadTestData() {
 	}
 	f := resp.Body
 	defer f.Close()
-	input := bufio.NewReader(f)
-	for {
-		line, err := input.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			logger.Fatal(err)
-		}
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
 		if len(line) == 0 || line[0] == '#' {
 			continue
 		}
@@ -190,6 +183,9 @@ func loadTestData() {
 		part := &part[len(part)-1]
 		part.tests = append(part.tests, test)
 	}
+	if scanner.Err() != nil {
+		logger.Fatal(scanner.Err())
+	}
 }
 
 var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}