mirror of https://github.com/gogits/gogs.git
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
82 lines
2.6 KiB
82 lines
2.6 KiB
// Copyright 2015 The Go Authors. All rights reserved. |
|
// Use of this source code is governed by a BSD-style |
|
// license that can be found in the LICENSE file. |
|
|
|
package unicode |
|
|
|
import ( |
|
"golang.org/x/text/transform" |
|
) |
|
|
|
// BOMOverride returns a new decoder transformer that is identical to fallback, |
|
// except that the presence of a Byte Order Mark at the start of the input |
|
// causes it to switch to the corresponding Unicode decoding. It will only |
|
// consider BOMs for UTF-8, UTF-16BE, and UTF-16LE. |
|
// |
|
// This differs from using ExpectBOM by allowing a BOM to switch to UTF-8, not |
|
// just UTF-16 variants, and allowing falling back to any encoding scheme. |
|
// |
|
// This technique is recommended by the W3C for use in HTML 5: "For |
|
// compatibility with deployed content, the byte order mark (also known as BOM) |
|
// is considered more authoritative than anything else." |
|
// http://www.w3.org/TR/encoding/#specification-hooks |
|
// |
|
// Using BOMOverride is mostly intended for use cases where the first characters |
|
// of a fallback encoding are known to not be a BOM, for example, for valid HTML |
|
// and most encodings. |
|
func BOMOverride(fallback transform.Transformer) transform.Transformer { |
|
// TODO: possibly allow a variadic argument of unicode encodings to allow |
|
// specifying details of which fallbacks are supported as well as |
|
// specifying the details of the implementations. This would also allow for |
|
// support for UTF-32, which should not be supported by default. |
|
return &bomOverride{fallback: fallback} |
|
} |
|
|
|
type bomOverride struct { |
|
fallback transform.Transformer |
|
current transform.Transformer |
|
} |
|
|
|
func (d *bomOverride) Reset() { |
|
d.current = nil |
|
d.fallback.Reset() |
|
} |
|
|
|
var ( |
|
// TODO: we could use decode functions here, instead of allocating a new |
|
// decoder on every NewDecoder as IgnoreBOM decoders can be stateless. |
|
utf16le = UTF16(LittleEndian, IgnoreBOM) |
|
utf16be = UTF16(BigEndian, IgnoreBOM) |
|
) |
|
|
|
const utf8BOM = "\ufeff" |
|
|
|
func (d *bomOverride) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { |
|
if d.current != nil { |
|
return d.current.Transform(dst, src, atEOF) |
|
} |
|
if len(src) < 3 && !atEOF { |
|
return 0, 0, transform.ErrShortSrc |
|
} |
|
d.current = d.fallback |
|
bomSize := 0 |
|
if len(src) >= 2 { |
|
if src[0] == 0xFF && src[1] == 0xFE { |
|
d.current = utf16le.NewDecoder() |
|
bomSize = 2 |
|
} else if src[0] == 0xFE && src[1] == 0xFF { |
|
d.current = utf16be.NewDecoder() |
|
bomSize = 2 |
|
} else if len(src) >= 3 && |
|
src[0] == utf8BOM[0] && |
|
src[1] == utf8BOM[1] && |
|
src[2] == utf8BOM[2] { |
|
d.current = transform.Nop |
|
bomSize = 3 |
|
} |
|
} |
|
if bomSize < len(src) { |
|
nDst, nSrc, err = d.current.Transform(dst, src[bomSize:], atEOF) |
|
} |
|
return nDst, nSrc + bomSize, err |
|
}
|
|
|