feat(parsing): full encoding support (UTF-32, ISO-8859-1, Windows-1252)
Decode UTF-32 (both byte orders, BOM or sniffed) and the single-byte encodings ISO-8859-1 and Windows-1252, selected by the XML declaration's encoding name, in addition to UTF-8 and UTF-16. The streaming byte path also handles UTF-32. Standard library only, no Foundation. Closes #28. Reimplemented from the documented encoding algorithms; no upstream code copied. 4 tests; 94 total.
This commit is contained in:
parent
135e464652
commit
85172923e8
@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Added
|
||||
|
||||
- Full encoding support. Byte input now decodes UTF-32 (both byte orders, BOM or
|
||||
sniffed) and the single-byte encodings ISO-8859-1 and Windows-1252 selected by
|
||||
the XML declaration's `encoding` name, in addition to UTF-8 and UTF-16. The
|
||||
streaming byte path also decodes UTF-32. Standard library only, no Foundation.
|
||||
- Incremental XML writer (`PureXML.Emitting.Writer`, the libxml2 `xmlTextWriter`
|
||||
model): emit a document with start/end and write calls without building a tree.
|
||||
Escaping matches the tree serializer (extracted into a shared `Escaping` helper),
|
||||
|
||||
@ -59,7 +59,7 @@ flowchart TB
|
||||
Push["#1 Push / feed streaming API"]:::epic
|
||||
Sax["#19 SAX2 push callbacks"]:::done
|
||||
Html["#20 HTML parser and serializer"]:::epic
|
||||
Enc["#28 Full encoding support"]:::epic
|
||||
Enc["#28 Full encoding support"]:::done
|
||||
Ent["#29 External and parameter entities"]:::epic
|
||||
TreeApi["#31 Tree manipulation API"]:::epic
|
||||
Writer["#32 xmlTextWriter"]:::done
|
||||
|
||||
@ -1,11 +1,17 @@
|
||||
extension PureXML.Parsing {
|
||||
/// Decodes raw bytes into a string using the detected encoding, stripping any
|
||||
/// byte-order mark. UTF-8 and UTF-16 (both byte orders) are supported, which
|
||||
/// is the encoding set the XML specification requires of every processor.
|
||||
/// Uses only the Swift standard library (no Foundation), so it stays portable.
|
||||
/// byte-order mark. Covers UTF-8, UTF-16, and UTF-32 (both byte orders), plus
|
||||
/// ISO-8859-1 and Windows-1252 selected by the XML declaration's encoding
|
||||
/// name. Uses only the Swift standard library (no Foundation).
|
||||
enum ByteDecoder {
|
||||
static func decode(_ bytes: [UInt8]) throws -> String {
|
||||
let (encoding, bomLength) = InputEncoding.detectWithBOM(bytes)
|
||||
let (sniffed, bomLength) = InputEncoding.detectWithBOM(bytes)
|
||||
// With no BOM the byte sniff defaults to UTF-8; honor a declared
|
||||
// single-byte encoding if the XML declaration names one.
|
||||
var encoding = sniffed
|
||||
if sniffed == .utf8, bomLength == 0, let declared = declaredEncoding(in: bytes) {
|
||||
encoding = declared
|
||||
}
|
||||
let body = bytes.dropFirst(bomLength)
|
||||
switch encoding {
|
||||
case .utf8:
|
||||
@ -14,6 +20,14 @@ extension PureXML.Parsing {
|
||||
return try decodeUTF16(body, bigEndian: true)
|
||||
case .utf16LittleEndian:
|
||||
return try decodeUTF16(body, bigEndian: false)
|
||||
case .utf32BigEndian:
|
||||
return try decodeUTF32(body, bigEndian: true)
|
||||
case .utf32LittleEndian:
|
||||
return try decodeUTF32(body, bigEndian: false)
|
||||
case .latin1:
|
||||
return String(String.UnicodeScalarView(body.map { Unicode.Scalar($0) }))
|
||||
case .windows1252:
|
||||
return String(String.UnicodeScalarView(body.map { Self.windows1252Scalar($0) }))
|
||||
}
|
||||
}
|
||||
|
||||
@ -24,12 +38,78 @@ extension PureXML.Parsing {
|
||||
var units: [UInt16] = []
|
||||
units.reserveCapacity(bytes.count / 2)
|
||||
var iterator = bytes.makeIterator()
|
||||
while let first = iterator.next(), let second = iterator.next() {
|
||||
let high = UInt16(first)
|
||||
let low = UInt16(second)
|
||||
units.append(bigEndian ? (high << 8 | low) : (low << 8 | high))
|
||||
while let high = iterator.next(), let low = iterator.next() {
|
||||
let first = UInt16(high)
|
||||
let second = UInt16(low)
|
||||
units.append(bigEndian ? (first << 8 | second) : (second << 8 | first))
|
||||
}
|
||||
return String(decoding: units, as: UTF16.self)
|
||||
}
|
||||
|
||||
private static func decodeUTF32(_ bytes: ArraySlice<UInt8>, bigEndian: Bool) throws -> String {
|
||||
guard bytes.count.isMultiple(of: 4) else {
|
||||
throw PureXML.Parsing.ParseError.malformedEncoding
|
||||
}
|
||||
let array = Array(bytes)
|
||||
var scalars = String.UnicodeScalarView()
|
||||
var index = 0
|
||||
while index + 4 <= array.count {
|
||||
let quad = array[index ..< index + 4]
|
||||
let value = (bigEndian ? AnySequence(quad) : AnySequence(quad.reversed()))
|
||||
.reduce(UInt32(0)) { ($0 << 8) | UInt32($1) }
|
||||
scalars.append(Unicode.Scalar(value) ?? "\u{FFFD}")
|
||||
index += 4
|
||||
}
|
||||
return String(scalars)
|
||||
}
|
||||
|
||||
/// Reads the `encoding` pseudo-attribute from an XML declaration that is
|
||||
/// ASCII-compatible at the front (UTF-8 or a single-byte encoding).
|
||||
static func declaredEncoding(in bytes: [UInt8]) -> InputEncoding? {
|
||||
let header = Array(bytes.prefix(256))
|
||||
guard header.starts(with: Array("<?xml".utf8)) else { return nil }
|
||||
let limit = header.firstIndex(of: 0x3E) ?? header.count // ">"
|
||||
let scan = Array(header[..<limit])
|
||||
guard let position = indexOfSubsequence(Array("encoding".utf8), in: scan) else { return nil }
|
||||
var index = position + 8
|
||||
while index < scan.count, scan[index] != 0x22, scan[index] != 0x27 {
|
||||
index += 1
|
||||
}
|
||||
guard index < scan.count else { return nil }
|
||||
let quote = scan[index]
|
||||
index += 1
|
||||
var name: [UInt8] = []
|
||||
while index < scan.count, scan[index] != quote {
|
||||
name.append(scan[index])
|
||||
index += 1
|
||||
}
|
||||
switch String(decoding: name, as: UTF8.self).lowercased() {
|
||||
case "utf-8", "utf8", "us-ascii", "ascii": return .utf8
|
||||
case "iso-8859-1", "latin1", "latin-1", "l1": return .latin1
|
||||
case "windows-1252", "cp1252", "cp-1252": return .windows1252
|
||||
default: return nil
|
||||
}
|
||||
}
|
||||
|
||||
private static func indexOfSubsequence(_ needle: [UInt8], in haystack: [UInt8]) -> Int? {
|
||||
guard !needle.isEmpty, haystack.count >= needle.count else { return nil }
|
||||
for start in 0 ... (haystack.count - needle.count) where Array(haystack[start ..< start + needle.count]) == needle {
|
||||
return start
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/// Maps a Windows-1252 byte to its Unicode scalar. The 0x80...0x9F range
|
||||
/// is the published CP1252 table; everything else is ISO-8859-1.
|
||||
static func windows1252Scalar(_ byte: UInt8) -> Unicode.Scalar {
|
||||
guard (0x80 ... 0x9F).contains(byte) else { return Unicode.Scalar(byte) }
|
||||
let high: [UInt32] = [
|
||||
0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
|
||||
0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F,
|
||||
0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
|
||||
0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178,
|
||||
]
|
||||
return Unicode.Scalar(high[Int(byte - 0x80)]) ?? "\u{FFFD}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,35 +1,54 @@
|
||||
public extension PureXML.Parsing {
|
||||
/// A character encoding PureXML can decode from raw bytes. XML processors are
|
||||
/// required to support UTF-8 and UTF-16; those are covered here. Other declared
|
||||
/// encodings are out of scope (and reported as malformed rather than guessed).
|
||||
/// A character encoding PureXML can decode from raw bytes. Covers the
|
||||
/// Unicode transformation formats plus the common single-byte encodings
|
||||
/// (ISO-8859-1 and Windows-1252), toward libxml2 encoding parity.
|
||||
enum InputEncoding: String, Equatable, Sendable {
|
||||
case utf8
|
||||
case utf16BigEndian
|
||||
case utf16LittleEndian
|
||||
case utf32BigEndian
|
||||
case utf32LittleEndian
|
||||
case latin1
|
||||
case windows1252
|
||||
|
||||
/// Detects the encoding from the leading bytes, following the XML sniff
|
||||
/// order: a byte-order mark first, then the `<?` byte pattern of the XML
|
||||
/// declaration, defaulting to UTF-8 when neither is present.
|
||||
/// declaration, defaulting to UTF-8. Single-byte encodings are not
|
||||
/// detectable from bytes alone; they come from the declared encoding name.
|
||||
public static func detect(_ bytes: [UInt8]) -> InputEncoding {
|
||||
detectWithBOM(bytes).encoding
|
||||
}
|
||||
|
||||
/// Detects the encoding and the length of any byte-order mark to strip.
|
||||
static func detectWithBOM(_ bytes: [UInt8]) -> (encoding: InputEncoding, bomLength: Int) {
|
||||
if bytes.count >= 3, bytes[0] == 0xEF, bytes[1] == 0xBB, bytes[2] == 0xBF {
|
||||
if bytes.starts(with: [0xEF, 0xBB, 0xBF]) {
|
||||
return (.utf8, 3)
|
||||
}
|
||||
if bytes.count >= 2, bytes[0] == 0xFE, bytes[1] == 0xFF {
|
||||
// UTF-32 marks are four bytes and must be checked before UTF-16, since
|
||||
// the UTF-32LE mark begins with the UTF-16LE mark.
|
||||
if bytes.starts(with: [0x00, 0x00, 0xFE, 0xFF]) {
|
||||
return (.utf32BigEndian, 4)
|
||||
}
|
||||
if bytes.starts(with: [0xFF, 0xFE, 0x00, 0x00]) {
|
||||
return (.utf32LittleEndian, 4)
|
||||
}
|
||||
if bytes.starts(with: [0xFE, 0xFF]) {
|
||||
return (.utf16BigEndian, 2)
|
||||
}
|
||||
if bytes.count >= 2, bytes[0] == 0xFF, bytes[1] == 0xFE {
|
||||
if bytes.starts(with: [0xFF, 0xFE]) {
|
||||
return (.utf16LittleEndian, 2)
|
||||
}
|
||||
// No BOM: sniff the "<?" pattern of an XML declaration.
|
||||
if bytes.count >= 4, bytes[0] == 0x00, bytes[1] == 0x3C, bytes[2] == 0x00, bytes[3] == 0x3F {
|
||||
// No BOM: sniff the "<?" byte pattern of an XML declaration.
|
||||
if bytes.starts(with: [0x00, 0x00, 0x00, 0x3C]) {
|
||||
return (.utf32BigEndian, 0)
|
||||
}
|
||||
if bytes.starts(with: [0x3C, 0x00, 0x00, 0x00]) {
|
||||
return (.utf32LittleEndian, 0)
|
||||
}
|
||||
if bytes.starts(with: [0x00, 0x3C, 0x00, 0x3F]) {
|
||||
return (.utf16BigEndian, 0)
|
||||
}
|
||||
if bytes.count >= 4, bytes[0] == 0x3C, bytes[1] == 0x00, bytes[2] == 0x3F, bytes[3] == 0x00 {
|
||||
if bytes.starts(with: [0x3C, 0x00, 0x3F, 0x00]) {
|
||||
return (.utf16LittleEndian, 0)
|
||||
}
|
||||
return (.utf8, 0)
|
||||
|
||||
@ -35,9 +35,39 @@ extension PureXML.Parsing {
|
||||
return nextUTF16(bigEndian: true)
|
||||
case .utf16LittleEndian:
|
||||
return nextUTF16(bigEndian: false)
|
||||
case .utf32BigEndian:
|
||||
return nextUTF32(bigEndian: true)
|
||||
case .utf32LittleEndian:
|
||||
return nextUTF32(bigEndian: false)
|
||||
case .latin1:
|
||||
return nextSingleByte { Unicode.Scalar($0) }
|
||||
case .windows1252:
|
||||
return nextSingleByte(PureXML.Parsing.ByteDecoder.windows1252Scalar)
|
||||
}
|
||||
}
|
||||
|
||||
private mutating func nextUTF32(bigEndian: Bool) -> Character? {
|
||||
var ordered: [UInt32] = []
|
||||
for _ in 0 ..< 4 {
|
||||
guard let byte = nextByte() else {
|
||||
finished = true
|
||||
return nil
|
||||
}
|
||||
ordered.append(UInt32(byte))
|
||||
}
|
||||
if !bigEndian { ordered.reverse() }
|
||||
let value = ordered.reduce(UInt32(0)) { ($0 << 8) | $1 }
|
||||
return Character(Unicode.Scalar(value) ?? "\u{FFFD}")
|
||||
}
|
||||
|
||||
private mutating func nextSingleByte(_ map: (UInt8) -> Unicode.Scalar) -> Character? {
|
||||
guard let byte = nextByte() else {
|
||||
finished = true
|
||||
return nil
|
||||
}
|
||||
return Character(map(byte))
|
||||
}
|
||||
|
||||
private mutating func detect() {
|
||||
while prefix.count < 4, let byte = pull() {
|
||||
prefix.append(byte)
|
||||
|
||||
@ -51,6 +51,34 @@ struct EncodingTests {
|
||||
}
|
||||
}
|
||||
|
||||
@Test("Detects UTF-32 byte-order marks")
|
||||
func test_detectUTF32() {
|
||||
#expect(PureXML.Parsing.InputEncoding.detect([0x00, 0x00, 0xFE, 0xFF, 0x00]) == .utf32BigEndian)
|
||||
#expect(PureXML.Parsing.InputEncoding.detect([0xFF, 0xFE, 0x00, 0x00, 0x00]) == .utf32LittleEndian)
|
||||
}
|
||||
|
||||
@Test("Parses UTF-32 bytes in both byte orders")
|
||||
func test_parseUTF32() throws {
|
||||
try #expect(rootText(PureXML.parse(bytes: utf32("<r>hi</r>", bigEndian: true))) == "hi")
|
||||
try #expect(rootText(PureXML.parse(bytes: utf32("<r>hi</r>", bigEndian: false))) == "hi")
|
||||
}
|
||||
|
||||
@Test("Honors a declared ISO-8859-1 encoding")
|
||||
func test_latin1() throws {
|
||||
var bytes = Array("<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?><r>caf".utf8)
|
||||
bytes.append(0xE9) // small letter e with acute, in Latin-1
|
||||
bytes += Array("</r>".utf8)
|
||||
try #expect(rootText(PureXML.parse(bytes: bytes)) == "caf\u{E9}")
|
||||
}
|
||||
|
||||
@Test("Honors a declared Windows-1252 encoding")
|
||||
func test_windows1252() throws {
|
||||
var bytes = Array("<?xml version=\"1.0\" encoding=\"windows-1252\"?><r>".utf8)
|
||||
bytes.append(0x80) // euro sign in CP1252
|
||||
bytes += Array("</r>".utf8)
|
||||
try #expect(rootText(PureXML.parse(bytes: bytes)) == "\u{20AC}")
|
||||
}
|
||||
|
||||
private func byteSource(_ bytes: [UInt8]) -> () -> UInt8? {
|
||||
var index = 0
|
||||
return {
|
||||
@ -70,6 +98,21 @@ struct EncodingTests {
|
||||
return bytes
|
||||
}
|
||||
|
||||
private func utf32(_ string: String, bigEndian: Bool) -> [UInt8] {
|
||||
var bytes: [UInt8] = bigEndian ? [0x00, 0x00, 0xFE, 0xFF] : [0xFF, 0xFE, 0x00, 0x00]
|
||||
for scalar in string.unicodeScalars {
|
||||
let value = scalar.value
|
||||
let bigEndianBytes = [
|
||||
UInt8(value >> 24 & 0xFF),
|
||||
UInt8(value >> 16 & 0xFF),
|
||||
UInt8(value >> 8 & 0xFF),
|
||||
UInt8(value & 0xFF),
|
||||
]
|
||||
bytes += bigEndian ? bigEndianBytes : bigEndianBytes.reversed()
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
|
||||
private func rootText(_ node: PureXML.Model.Node) -> String? {
|
||||
guard case let .document(children) = node else { return nil }
|
||||
for child in children {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user