From 6439010e52610650f8aa048173832f94006ebdbd Mon Sep 17 00:00:00 2001 From: Rob Pike Date: Tue, 12 May 2015 10:29:53 -0700 Subject: encoding/gob: add "too big" check when writing a message Messages that are too big are rejected when read, so they should be rejected when written too. Fixes #10518. Change-Id: I96678fbe2d94f51b957fe26faef33cd8df3823dd Reviewed-on: https://go-review.googlesource.com/9965 Reviewed-by: Brad Fitzpatrick --- src/encoding/gob/encoder.go | 6 ++++++ src/encoding/gob/encoder_test.go | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) (limited to 'src/encoding') diff --git a/src/encoding/gob/encoder.go b/src/encoding/gob/encoder.go index a340e47b5e..62d0f42e81 100644 --- a/src/encoding/gob/encoder.go +++ b/src/encoding/gob/encoder.go @@ -5,6 +5,7 @@ package gob import ( + "errors" "io" "reflect" "sync" @@ -65,6 +66,11 @@ func (enc *Encoder) writeMessage(w io.Writer, b *encBuffer) { // it by hand. message := b.Bytes() messageLen := len(message) - maxLength + // Length cannot be bigger than the decoder can handle. + if messageLen >= tooBig { + enc.setError(errors.New("gob: encoder: message too big")) + return + } // Encode the length. enc.countState.b.Reset() enc.countState.encodeUint(uint64(messageLen)) diff --git a/src/encoding/gob/encoder_test.go b/src/encoding/gob/encoder_test.go index c0bd379c93..8a72a3118c 100644 --- a/src/encoding/gob/encoder_test.go +++ b/src/encoding/gob/encoder_test.go @@ -976,3 +976,21 @@ func TestBadData(t *testing.T) { } } } + +// TestHugeWriteFails tests that enormous messages trigger an error. +func TestHugeWriteFails(t *testing.T) { + if testing.Short() { + // Requires allocating a monster, so don't do this from all.bash. + t.Skip("skipping huge allocation in short mode") + } + huge := make([]byte, tooBig) + huge[0] = 7 // Make sure it's not all zeros. + buf := new(bytes.Buffer) + err := NewEncoder(buf).Encode(huge) + if err == nil { + t.Fatalf("expected error for huge slice") + } + if !strings.Contains(err.Error(), "message too big") { + t.Fatalf("expected 'too big' error; got %s\n", err.Error()) + } +} -- cgit v1.3 From dbf533a5460d7fcc7d7be77014fd74a8aff8c412 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Wed, 13 May 2015 19:27:59 -0700 Subject: encoding/json: make BenchmarkSkipValue more stable BenchmarkSkipValue was sensitive to the value of b.N due to its significant startup cost. Two adjacent runs before this CL: BenchmarkSkipValue 50 21047499 ns/op 93.37 MB/s BenchmarkSkipValue 100 17260554 ns/op 118.05 MB/s After this CL, using benchtime to recreate the difference in b.N: BenchmarkSkipValue 50 15204797 ns/op 131.67 MB/s BenchmarkSkipValue 100 15332319 ns/op 130.58 MB/s Change-Id: Iac86f86dd774d535302fa5e4c08f89f8da00be9e Reviewed-on: https://go-review.googlesource.com/10053 Reviewed-by: Andrew Gerrand --- src/encoding/json/scanner_test.go | 1 + 1 file changed, 1 insertion(+) (limited to 'src/encoding') diff --git a/src/encoding/json/scanner_test.go b/src/encoding/json/scanner_test.go index 7880342902..66383ef0ef 100644 --- a/src/encoding/json/scanner_test.go +++ b/src/encoding/json/scanner_test.go @@ -209,6 +209,7 @@ var benchScan scanner func BenchmarkSkipValue(b *testing.B) { initBig() + b.ResetTimer() for i := 0; i < b.N; i++ { nextValue(jsonBig, &benchScan) } -- cgit v1.3 From 4302fd0409da5e4f1d71471a6770dacdc3301197 Mon Sep 17 00:00:00 2001 From: Håvard Haugen Date: Sun, 26 Apr 2015 23:52:42 +0200 Subject: encoding/json: fix decoding of types with '[]byte' as underlying type All slice types which have elements of kind reflect.Uint8 are marshalled into base64 for compactness. When decoding such data into a custom type based on []byte the decoder checked the slice kind instead of the slice element kind, so no appropriate decoder was found. Fixed by letting the decoder check slice element kind like the encoder. This guarantees that already encoded data can still be successfully decoded. Fixes #8962. Change-Id: Ia320d4dc2c6e9e5fe6d8dc15788c81da23d20c4f Reviewed-on: https://go-review.googlesource.com/9371 Reviewed-by: Peter Waldschmidt Reviewed-by: Russ Cox --- src/encoding/json/decode.go | 2 +- src/encoding/json/decode_test.go | 21 +++++++++++++++++++++ src/encoding/json/encode.go | 2 -- 3 files changed, 22 insertions(+), 3 deletions(-) (limited to 'src/encoding') diff --git a/src/encoding/json/decode.go b/src/encoding/json/decode.go index f26a7d49f0..613641afbb 100644 --- a/src/encoding/json/decode.go +++ b/src/encoding/json/decode.go @@ -739,7 +739,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool default: d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) case reflect.Slice: - if v.Type() != byteSliceType { + if v.Type().Elem().Kind() != reflect.Uint8 { d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) break } diff --git a/src/encoding/json/decode_test.go b/src/encoding/json/decode_test.go index 7ecc8f4402..f208ee8a7c 100644 --- a/src/encoding/json/decode_test.go +++ b/src/encoding/json/decode_test.go @@ -1207,7 +1207,28 @@ func TestStringKind(t *testing.T) { if !reflect.DeepEqual(m1, m2) { t.Error("Items should be equal after encoding and then decoding") } +} + +// Custom types with []byte as underlying type could not be marshalled +// and then unmarshalled. +// Issue 8962. +func TestByteKind(t *testing.T) { + type byteKind []byte + + a := byteKind("hello") + data, err := Marshal(a) + if err != nil { + t.Error(err) + } + var b byteKind + err = Unmarshal(data, &b) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(a, b) { + t.Errorf("expected %v == %v", a, b) + } } var decodeTypeErrorTests = []struct { diff --git a/src/encoding/json/encode.go b/src/encoding/json/encode.go index 4db9f35e69..7789bb5141 100644 --- a/src/encoding/json/encode.go +++ b/src/encoding/json/encode.go @@ -275,8 +275,6 @@ func (e *encodeState) error(err error) { panic(err) } -var byteSliceType = reflect.TypeOf([]byte(nil)) - func isEmptyValue(v reflect.Value) bool { switch v.Kind() { case reflect.Array, reflect.Map, reflect.Slice, reflect.String: -- cgit v1.3 From 91191e7b7bc8c0e1a6d49c7a9b3adeb1ab39a423 Mon Sep 17 00:00:00 2001 From: Shenghou Ma Date: Tue, 19 May 2015 00:19:04 -0400 Subject: encoding/gob: fix docs Fixes #10908. Change-Id: I5ac4bd90204bc230610dcced47ce5b2253e5a004 Reviewed-on: https://go-review.googlesource.com/10250 Reviewed-by: Rob Pike --- src/encoding/gob/doc.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'src/encoding') diff --git a/src/encoding/gob/doc.go b/src/encoding/gob/doc.go index d0acaba1ad..31223b6d43 100644 --- a/src/encoding/gob/doc.go +++ b/src/encoding/gob/doc.go @@ -6,7 +6,7 @@ Package gob manages streams of gobs - binary values exchanged between an Encoder (transmitter) and a Decoder (receiver). A typical use is transporting arguments and results of remote procedure calls (RPCs) such as those provided by -package "rpc". +package "net/rpc". The implementation compiles a custom codec for each data type in the stream and is most efficient when a single Encoder is used to transmit a stream of values, @@ -83,7 +83,7 @@ allocated. Regardless, the length of the resulting slice reports the number of elements decoded. Functions and channels will not be sent in a gob. Attempting to encode such a value -at top the level will fail. A struct field of chan or func type is treated exactly +at the top level will fail. A struct field of chan or func type is treated exactly like an unexported field and is ignored. Gob can encode a value of any type implementing the GobEncoder or @@ -111,11 +111,11 @@ A signed integer, i, is encoded within an unsigned integer, u. Within u, bits 1 upward contain the value; bit 0 says whether they should be complemented upon receipt. The encode algorithm looks like this: - uint u; + var u uint if i < 0 { - u = (^i << 1) | 1 // complement i, bit 0 is 1 + u = (^uint(i) << 1) | 1 // complement i, bit 0 is 1 } else { - u = (i << 1) // do not complement i, bit 0 is 0 + u = (uint(i) << 1) // do not complement i, bit 0 is 0 } encodeUnsigned(u) @@ -137,9 +137,9 @@ All other slices and arrays are sent as an unsigned count followed by that many elements using the standard gob encoding for their type, recursively. Maps are sent as an unsigned count followed by that many key, element -pairs. Empty but non-nil maps are sent, so if the sender has allocated -a map, the receiver will allocate a map even if no elements are -transmitted. +pairs. Empty but non-nil maps are sent, so if the receiver has not allocated +one already, one will always be allocated on receipt unless the transmitted map +is nil and not at the top level. Structs are sent as a sequence of (field number, field value) pairs. The field value is sent using the standard gob encoding for its type, recursively. If a @@ -246,7 +246,7 @@ where * signifies zero or more repetitions and the type id of a value must be predefined or be defined before the value in the stream. See "Gobs of data" for a design discussion of the gob wire format: -http://golang.org/doc/articles/gobs_of_data.html +http://blog.golang.org/gobs-of-data */ package gob -- cgit v1.3