1 // Copyright 2015 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package unicode 6 7 import ( 8 "golang.org/x/text/transform" 9 ) 10 11 // BOMOverride returns a new decoder transformer that is identical to fallback, 12 // except that the presence of a Byte Order Mark at the start of the input 13 // causes it to switch to the corresponding Unicode decoding. It will only 14 // consider BOMs for UTF-8, UTF-16BE, and UTF-16LE. 15 // 16 // This differs from using ExpectBOM by allowing a BOM to switch to UTF-8, not 17 // just UTF-16 variants, and allowing falling back to any encoding scheme. 18 // 19 // This technique is recommended by the W3C for use in HTML 5: "For 20 // compatibility with deployed content, the byte order mark (also known as BOM) 21 // is considered more authoritative than anything else." 22 // http://www.w3.org/TR/encoding/#specification-hooks 23 // 24 // Using BOMOverride is mostly intended for use cases where the first characters 25 // of a fallback encoding are known to not be a BOM, for example, for valid HTML 26 // and most encodings. 27 func BOMOverride(fallback transform.Transformer) transform.Transformer { 28 // TODO: possibly allow a variadic argument of unicode encodings to allow 29 // specifying details of which fallbacks are supported as well as 30 // specifying the details of the implementations. This would also allow for 31 // support for UTF-32, which should not be supported by default. 32 return &bomOverride{fallback: fallback} 33 } 34 35 type bomOverride struct { 36 fallback transform.Transformer 37 current transform.Transformer 38 } 39 40 func (d *bomOverride) Reset() { 41 d.current = nil 42 d.fallback.Reset() 43 } 44 45 var ( 46 // TODO: we could use decode functions here, instead of allocating a new 47 // decoder on every NewDecoder as IgnoreBOM decoders can be stateless. 48 utf16le = UTF16(LittleEndian, IgnoreBOM) 49 utf16be = UTF16(BigEndian, IgnoreBOM) 50 ) 51 52 const utf8BOM = "\ufeff" 53 54 func (d *bomOverride) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { 55 if d.current != nil { 56 return d.current.Transform(dst, src, atEOF) 57 } 58 if len(src) < 3 && !atEOF { 59 return 0, 0, transform.ErrShortSrc 60 } 61 d.current = d.fallback 62 bomSize := 0 63 if len(src) >= 2 { 64 if src[0] == 0xFF && src[1] == 0xFE { 65 d.current = utf16le.NewDecoder() 66 bomSize = 2 67 } else if src[0] == 0xFE && src[1] == 0xFF { 68 d.current = utf16be.NewDecoder() 69 bomSize = 2 70 } else if len(src) >= 3 && 71 src[0] == utf8BOM[0] && 72 src[1] == utf8BOM[1] && 73 src[2] == utf8BOM[2] { 74 d.current = transform.Nop 75 bomSize = 3 76 } 77 } 78 if bomSize < len(src) { 79 nDst, nSrc, err = d.current.Transform(dst, src[bomSize:], atEOF) 80 } 81 return nDst, nSrc + bomSize, err 82 } 83