math/big: fix Float.Float32 conversion for denormal corner cases

The existing code was incorrect for numbers that after rounding would
become the smallest denormal float32 (instead the result was 0). This
caused all.bash to fail if Float32() were used in the compiler for
constant arithmetic (there's currently a work-around - see also issue
10321.

This change fixes the implementation of Float.Float32 and adds
corresponding test cases. Float32 and Float64 diverge at this point.
For ease of review, this change only fixes Float32. Float64 will be
made to match in a subsequent change.

Fixes #10321.

Change-Id: Iccafe37c1593a4946bc552e4ad2045f69be62d80
Reviewed-on: https://go-review.googlesource.com/10350
Reviewed-by: Alan Donovan <adonovan@google.com>

1 files changed, 58 insertions, 35 deletions

diff --git a/src/math/big/float_test.go b/src/math/big/float_test.go
index 5d241a503b..4da145b652 100644
--- a/src/math/big/float_test.go
+++ b/src/math/big/float_test.go
@@ -203,6 +203,12 @@ func alike(x, y *Float) bool {
 	return x.Cmp(y) == 0 && x.Signbit() == y.Signbit()
 }
 
+func alike32(x, y float32) bool {
+	// we can ignore NaNs
+	return x == y && math.Signbit(float64(x)) == math.Signbit(float64(y))
+
+}
+
 func TestFloatMantExp(t *testing.T) {
 	for _, test := range []struct {
 		x    string
@@ -828,52 +834,69 @@ func TestFloatFloat32(t *testing.T) {
 		out float32
 		acc Accuracy
 	}{
-		{"-Inf", float32(math.Inf(-1)), Exact},
-		{"-0x1.ffffff0p2147483646", float32(-math.Inf(+1)), Below}, // overflow in rounding
-		{"-1e10000", float32(math.Inf(-1)), Below},                 // overflow
-		{"-0x1p128", float32(math.Inf(-1)), Below},                 // overflow
-		{"-0x1.ffffff0p127", float32(-math.Inf(+1)), Below},        // overflow
-		{"-0x1.fffffe8p127", -math.MaxFloat32, Above},
-		{"-0x1.fffffe0p127", -math.MaxFloat32, Exact},
-		{"-12345.000000000000000000001", -12345, Above},
-		{"-12345.0", -12345, Exact},
-		{"-1.000000000000000000001", -1, Above},
-		{"-1", -1, Exact},
-		{"-0x0.000002p-126", -math.SmallestNonzeroFloat32, Exact},
-		{"-0x0.000002p-127", -0, Above}, // underflow
-		{"-1e-1000", -0, Above},         // underflow
 		{"0", 0, Exact},
-		{"1e-1000", 0, Below},         // underflow
-		{"0x0.000002p-127", 0, Below}, // underflow
-		{"0x0.000002p-126", math.SmallestNonzeroFloat32, Exact},
+
+		// underflow
+		{"1e-1000", 0, Below},
+		{"0x0.000002p-127", 0, Below},
+		{"0x.0000010p-126", 0, Below},
+
+		// denormals
+		{"1.401298464e-45", math.SmallestNonzeroFloat32, Above}, // rounded up to smallest denormal
+		{"0x.ffffff8p-149", math.SmallestNonzeroFloat32, Above}, // rounded up to smallest denormal
+		{"0x.0000018p-126", math.SmallestNonzeroFloat32, Above}, // rounded up to smallest denormal
+		{"0x.0000020p-126", math.SmallestNonzeroFloat32, Exact},
+		{"0x.8p-148", math.SmallestNonzeroFloat32, Exact},
+		{"1p-149", math.SmallestNonzeroFloat32, Exact},
+		{"0x.fffffep-126", math.Float32frombits(0x7fffff), Exact}, // largest denormal
+
+		// normals
+		{"0x.ffffffp-126", math.Float32frombits(0x00800000), Above}, // rounded up to smallest normal
+		{"1p-126", math.Float32frombits(0x00800000), Exact},         // smallest normal
+		{"0x1.fffffep-126", math.Float32frombits(0x00ffffff), Exact},
+		{"0x1.ffffffp-126", math.Float32frombits(0x01000000), Above}, // rounded up
 		{"1", 1, Exact},
 		{"1.000000000000000000001", 1, Below},
 		{"12345.0", 12345, Exact},
 		{"12345.000000000000000000001", 12345, Below},
 		{"0x1.fffffe0p127", math.MaxFloat32, Exact},
 		{"0x1.fffffe8p127", math.MaxFloat32, Below},
-		{"0x1.ffffff0p127", float32(math.Inf(+1)), Above},        // overflow
-		{"0x1p128", float32(math.Inf(+1)), Above},                // overflow
-		{"1e10000", float32(math.Inf(+1)), Above},                // overflow
+
+		// overflow
+		{"0x1.ffffff0p127", float32(math.Inf(+1)), Above},
+		{"0x1p128", float32(math.Inf(+1)), Above},
+		{"1e10000", float32(math.Inf(+1)), Above},
 		{"0x1.ffffff0p2147483646", float32(math.Inf(+1)), Above}, // overflow in rounding
-		{"+Inf", float32(math.Inf(+1)), Exact},
+
+		// inf
+		{"Inf", float32(math.Inf(+1)), Exact},
 	} {
-		// conversion should match strconv where syntax is agreeable
-		if f, err := strconv.ParseFloat(test.x, 32); err == nil && float32(f) != test.out {
-			t.Errorf("%s: got %g; want %g (incorrect test data)", test.x, f, test.out)
-		}
+		for i := 0; i < 2; i++ {
+			// test both signs
+			tx, tout, tacc := test.x, test.out, test.acc
+			if i != 0 {
+				tx = "-" + tx
+				tout = -tout
+				tacc = -tacc
+			}
 
-		x := makeFloat(test.x)
-		out, acc := x.Float32()
-		if out != test.out || acc != test.acc {
-			t.Errorf("%s: got %g (%#x, %s); want %g (%#x, %s)", test.x, out, math.Float32bits(out), acc, test.out, math.Float32bits(test.out), test.acc)
-		}
+			// conversion should match strconv where syntax is agreeable
+			if f, err := strconv.ParseFloat(tx, 32); err == nil && !alike32(float32(f), tout) {
+				t.Errorf("%s: got %g; want %g (incorrect test data)", tx, f, tout)
+			}
 
-		// test that x.SetFloat64(float64(f)).Float32() == f
-		var x2 Float
-		out2, acc2 := x2.SetFloat64(float64(out)).Float32()
-		if out2 != out || acc2 != Exact {
-			t.Errorf("idempotency test: got %g (%s); want %g (Exact)", out2, acc2, out)
+			x := makeFloat(tx)
+			out, acc := x.Float32()
+			if !alike32(out, tout) || acc != tacc {
+				t.Errorf("%s: got %g (%#x, %s); want %g (%#x, %s)", tx, out, math.Float32bits(out), acc, test.out, math.Float32bits(test.out), tacc)
+			}
+
+			// test that x.SetFloat64(float64(f)).Float32() == f
+			var x2 Float
+			out2, acc2 := x2.SetFloat64(float64(out)).Float32()
+			if !alike32(out2, out) || acc2 != Exact {
+				t.Errorf("idempotency test: got %g (%s); want %g (Exact)", out2, acc2, out)
+			}
 		}
 	}
 }