From 3259bc441957bf74f069cf7df961367a3472afb2 Mon Sep 17 00:00:00 2001 From: Martin Möhrmann Date: Sat, 13 Oct 2018 22:40:23 +0200 Subject: strings, bytes: add ToValidUTF8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The newly added functions create a copy of their input with all bytes in invalid UTF-8 byte sequences mapped to the UTF-8 byte sequence given as replacement parameter. Fixes #25805 Change-Id: Iaf65f65b40c0581c6bb000f1590408d6628321d0 Reviewed-on: https://go-review.googlesource.com/c/go/+/142003 Run-TryBot: Martin Möhrmann TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/bytes/bytes.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'src/bytes/bytes.go') diff --git a/src/bytes/bytes.go b/src/bytes/bytes.go index 9d586581f5..eb13212384 100644 --- a/src/bytes/bytes.go +++ b/src/bytes/bytes.go @@ -592,6 +592,35 @@ func ToTitleSpecial(c unicode.SpecialCase, s []byte) []byte { return Map(c.ToTitle, s) } +// ToValidUTF8 treats s as UTF-8-encoded bytes and returns a copy with each run of bytes +// representing invalid UTF-8 replaced with the bytes in replacement, which may be empty. +func ToValidUTF8(s, replacement []byte) []byte { + b := make([]byte, 0, len(s)+len(replacement)) + invalid := false // previous byte was from an invalid UTF-8 sequence + for i := 0; i < len(s); { + c := s[i] + if c < utf8.RuneSelf { + i++ + invalid = false + b = append(b, byte(c)) + continue + } + _, wid := utf8.DecodeRune(s[i:]) + if wid == 1 { + i++ + if !invalid { + invalid = true + b = append(b, replacement...) + } + continue + } + invalid = false + b = append(b, s[i:i+wid]...) + i += wid + } + return b +} + // isSeparator reports whether the rune could mark a word boundary. // TODO: update when package unicode captures more of the properties. func isSeparator(r rune) bool { -- cgit v1.3-5-g9baa