From 85172923e8a0463f7ab9f3524b28cd91e89e1071 Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Mon, 8 Jun 2026 02:39:25 +0200 Subject: [PATCH] feat(parsing): full encoding support (UTF-32, ISO-8859-1, Windows-1252) Decode UTF-32 (both byte orders, BOM or sniffed) and the single-byte encodings ISO-8859-1 and Windows-1252, selected by the XML declaration's encoding name, in addition to UTF-8 and UTF-16. The streaming byte path also handles UTF-32. Standard library only, no Foundation. Closes #28. Reimplemented from the documented encoding algorithms; no upstream code copied. 4 tests; 94 total. --- CHANGELOG.md | 4 ++ README.md | 2 +- Sources/Parsing/ByteDecoder.swift | 96 +++++++++++++++++++++++--- Sources/Parsing/InputEncoding.swift | 39 ++++++++--- Sources/Parsing/StreamingDecoder.swift | 30 ++++++++ Tests/EncodingTests.swift | 43 ++++++++++++ 6 files changed, 195 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8913f18..abc7141 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Full encoding support. Byte input now decodes UTF-32 (both byte orders, BOM or + sniffed) and the single-byte encodings ISO-8859-1 and Windows-1252 selected by + the XML declaration's `encoding` name, in addition to UTF-8 and UTF-16. The + streaming byte path also decodes UTF-32. Standard library only, no Foundation. - Incremental XML writer (`PureXML.Emitting.Writer`, the libxml2 `xmlTextWriter` model): emit a document with start/end and write calls without building a tree. Escaping matches the tree serializer (extracted into a shared `Escaping` helper), diff --git a/README.md b/README.md index 916954f..b3e622d 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ flowchart TB Push["#1 Push / feed streaming API"]:::epic Sax["#19 SAX2 push callbacks"]:::done Html["#20 HTML parser and serializer"]:::epic - Enc["#28 Full encoding support"]:::epic + Enc["#28 Full encoding support"]:::done Ent["#29 External and parameter entities"]:::epic TreeApi["#31 Tree manipulation API"]:::epic Writer["#32 xmlTextWriter"]:::done diff --git a/Sources/Parsing/ByteDecoder.swift b/Sources/Parsing/ByteDecoder.swift index 843a2bd..f55e297 100644 --- a/Sources/Parsing/ByteDecoder.swift +++ b/Sources/Parsing/ByteDecoder.swift @@ -1,11 +1,17 @@ extension PureXML.Parsing { /// Decodes raw bytes into a string using the detected encoding, stripping any - /// byte-order mark. UTF-8 and UTF-16 (both byte orders) are supported, which - /// is the encoding set the XML specification requires of every processor. - /// Uses only the Swift standard library (no Foundation), so it stays portable. + /// byte-order mark. Covers UTF-8, UTF-16, and UTF-32 (both byte orders), plus + /// ISO-8859-1 and Windows-1252 selected by the XML declaration's encoding + /// name. Uses only the Swift standard library (no Foundation). enum ByteDecoder { static func decode(_ bytes: [UInt8]) throws -> String { - let (encoding, bomLength) = InputEncoding.detectWithBOM(bytes) + let (sniffed, bomLength) = InputEncoding.detectWithBOM(bytes) + // With no BOM the byte sniff defaults to UTF-8; honor a declared + // single-byte encoding if the XML declaration names one. + var encoding = sniffed + if sniffed == .utf8, bomLength == 0, let declared = declaredEncoding(in: bytes) { + encoding = declared + } let body = bytes.dropFirst(bomLength) switch encoding { case .utf8: @@ -14,6 +20,14 @@ extension PureXML.Parsing { return try decodeUTF16(body, bigEndian: true) case .utf16LittleEndian: return try decodeUTF16(body, bigEndian: false) + case .utf32BigEndian: + return try decodeUTF32(body, bigEndian: true) + case .utf32LittleEndian: + return try decodeUTF32(body, bigEndian: false) + case .latin1: + return String(String.UnicodeScalarView(body.map { Unicode.Scalar($0) })) + case .windows1252: + return String(String.UnicodeScalarView(body.map { Self.windows1252Scalar($0) })) } } @@ -24,12 +38,78 @@ extension PureXML.Parsing { var units: [UInt16] = [] units.reserveCapacity(bytes.count / 2) var iterator = bytes.makeIterator() - while let first = iterator.next(), let second = iterator.next() { - let high = UInt16(first) - let low = UInt16(second) - units.append(bigEndian ? (high << 8 | low) : (low << 8 | high)) + while let high = iterator.next(), let low = iterator.next() { + let first = UInt16(high) + let second = UInt16(low) + units.append(bigEndian ? (first << 8 | second) : (second << 8 | first)) } return String(decoding: units, as: UTF16.self) } + + private static func decodeUTF32(_ bytes: ArraySlice, bigEndian: Bool) throws -> String { + guard bytes.count.isMultiple(of: 4) else { + throw PureXML.Parsing.ParseError.malformedEncoding + } + let array = Array(bytes) + var scalars = String.UnicodeScalarView() + var index = 0 + while index + 4 <= array.count { + let quad = array[index ..< index + 4] + let value = (bigEndian ? AnySequence(quad) : AnySequence(quad.reversed())) + .reduce(UInt32(0)) { ($0 << 8) | UInt32($1) } + scalars.append(Unicode.Scalar(value) ?? "\u{FFFD}") + index += 4 + } + return String(scalars) + } + + /// Reads the `encoding` pseudo-attribute from an XML declaration that is + /// ASCII-compatible at the front (UTF-8 or a single-byte encoding). + static func declaredEncoding(in bytes: [UInt8]) -> InputEncoding? { + let header = Array(bytes.prefix(256)) + guard header.starts(with: Array("" + let scan = Array(header[.. Int? { + guard !needle.isEmpty, haystack.count >= needle.count else { return nil } + for start in 0 ... (haystack.count - needle.count) where Array(haystack[start ..< start + needle.count]) == needle { + return start + } + return nil + } + + /// Maps a Windows-1252 byte to its Unicode scalar. The 0x80...0x9F range + /// is the published CP1252 table; everything else is ISO-8859-1. + static func windows1252Scalar(_ byte: UInt8) -> Unicode.Scalar { + guard (0x80 ... 0x9F).contains(byte) else { return Unicode.Scalar(byte) } + let high: [UInt32] = [ + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, + 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, + 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178, + ] + return Unicode.Scalar(high[Int(byte - 0x80)]) ?? "\u{FFFD}" + } } } diff --git a/Sources/Parsing/InputEncoding.swift b/Sources/Parsing/InputEncoding.swift index b4288dc..666df75 100644 --- a/Sources/Parsing/InputEncoding.swift +++ b/Sources/Parsing/InputEncoding.swift @@ -1,35 +1,54 @@ public extension PureXML.Parsing { - /// A character encoding PureXML can decode from raw bytes. XML processors are - /// required to support UTF-8 and UTF-16; those are covered here. Other declared - /// encodings are out of scope (and reported as malformed rather than guessed). + /// A character encoding PureXML can decode from raw bytes. Covers the + /// Unicode transformation formats plus the common single-byte encodings + /// (ISO-8859-1 and Windows-1252), toward libxml2 encoding parity. enum InputEncoding: String, Equatable, Sendable { case utf8 case utf16BigEndian case utf16LittleEndian + case utf32BigEndian + case utf32LittleEndian + case latin1 + case windows1252 /// Detects the encoding from the leading bytes, following the XML sniff /// order: a byte-order mark first, then the ` InputEncoding { detectWithBOM(bytes).encoding } /// Detects the encoding and the length of any byte-order mark to strip. static func detectWithBOM(_ bytes: [UInt8]) -> (encoding: InputEncoding, bomLength: Int) { - if bytes.count >= 3, bytes[0] == 0xEF, bytes[1] == 0xBB, bytes[2] == 0xBF { + if bytes.starts(with: [0xEF, 0xBB, 0xBF]) { return (.utf8, 3) } - if bytes.count >= 2, bytes[0] == 0xFE, bytes[1] == 0xFF { + // UTF-32 marks are four bytes and must be checked before UTF-16, since + // the UTF-32LE mark begins with the UTF-16LE mark. + if bytes.starts(with: [0x00, 0x00, 0xFE, 0xFF]) { + return (.utf32BigEndian, 4) + } + if bytes.starts(with: [0xFF, 0xFE, 0x00, 0x00]) { + return (.utf32LittleEndian, 4) + } + if bytes.starts(with: [0xFE, 0xFF]) { return (.utf16BigEndian, 2) } - if bytes.count >= 2, bytes[0] == 0xFF, bytes[1] == 0xFE { + if bytes.starts(with: [0xFF, 0xFE]) { return (.utf16LittleEndian, 2) } - // No BOM: sniff the "= 4, bytes[0] == 0x00, bytes[1] == 0x3C, bytes[2] == 0x00, bytes[3] == 0x3F { + // No BOM: sniff the "= 4, bytes[0] == 0x3C, bytes[1] == 0x00, bytes[2] == 0x3F, bytes[3] == 0x00 { + if bytes.starts(with: [0x3C, 0x00, 0x3F, 0x00]) { return (.utf16LittleEndian, 0) } return (.utf8, 0) diff --git a/Sources/Parsing/StreamingDecoder.swift b/Sources/Parsing/StreamingDecoder.swift index c0cb57e..b97b293 100644 --- a/Sources/Parsing/StreamingDecoder.swift +++ b/Sources/Parsing/StreamingDecoder.swift @@ -35,9 +35,39 @@ extension PureXML.Parsing { return nextUTF16(bigEndian: true) case .utf16LittleEndian: return nextUTF16(bigEndian: false) + case .utf32BigEndian: + return nextUTF32(bigEndian: true) + case .utf32LittleEndian: + return nextUTF32(bigEndian: false) + case .latin1: + return nextSingleByte { Unicode.Scalar($0) } + case .windows1252: + return nextSingleByte(PureXML.Parsing.ByteDecoder.windows1252Scalar) } } + private mutating func nextUTF32(bigEndian: Bool) -> Character? { + var ordered: [UInt32] = [] + for _ in 0 ..< 4 { + guard let byte = nextByte() else { + finished = true + return nil + } + ordered.append(UInt32(byte)) + } + if !bigEndian { ordered.reverse() } + let value = ordered.reduce(UInt32(0)) { ($0 << 8) | $1 } + return Character(Unicode.Scalar(value) ?? "\u{FFFD}") + } + + private mutating func nextSingleByte(_ map: (UInt8) -> Unicode.Scalar) -> Character? { + guard let byte = nextByte() else { + finished = true + return nil + } + return Character(map(byte)) + } + private mutating func detect() { while prefix.count < 4, let byte = pull() { prefix.append(byte) diff --git a/Tests/EncodingTests.swift b/Tests/EncodingTests.swift index 61d3db4..1f78ecf 100644 --- a/Tests/EncodingTests.swift +++ b/Tests/EncodingTests.swift @@ -51,6 +51,34 @@ struct EncodingTests { } } + @Test("Detects UTF-32 byte-order marks") + func test_detectUTF32() { + #expect(PureXML.Parsing.InputEncoding.detect([0x00, 0x00, 0xFE, 0xFF, 0x00]) == .utf32BigEndian) + #expect(PureXML.Parsing.InputEncoding.detect([0xFF, 0xFE, 0x00, 0x00, 0x00]) == .utf32LittleEndian) + } + + @Test("Parses UTF-32 bytes in both byte orders") + func test_parseUTF32() throws { + try #expect(rootText(PureXML.parse(bytes: utf32("hi", bigEndian: true))) == "hi") + try #expect(rootText(PureXML.parse(bytes: utf32("hi", bigEndian: false))) == "hi") + } + + @Test("Honors a declared ISO-8859-1 encoding") + func test_latin1() throws { + var bytes = Array("caf".utf8) + bytes.append(0xE9) // small letter e with acute, in Latin-1 + bytes += Array("".utf8) + try #expect(rootText(PureXML.parse(bytes: bytes)) == "caf\u{E9}") + } + + @Test("Honors a declared Windows-1252 encoding") + func test_windows1252() throws { + var bytes = Array("".utf8) + bytes.append(0x80) // euro sign in CP1252 + bytes += Array("".utf8) + try #expect(rootText(PureXML.parse(bytes: bytes)) == "\u{20AC}") + } + private func byteSource(_ bytes: [UInt8]) -> () -> UInt8? { var index = 0 return { @@ -70,6 +98,21 @@ struct EncodingTests { return bytes } + private func utf32(_ string: String, bigEndian: Bool) -> [UInt8] { + var bytes: [UInt8] = bigEndian ? [0x00, 0x00, 0xFE, 0xFF] : [0xFF, 0xFE, 0x00, 0x00] + for scalar in string.unicodeScalars { + let value = scalar.value + let bigEndianBytes = [ + UInt8(value >> 24 & 0xFF), + UInt8(value >> 16 & 0xFF), + UInt8(value >> 8 & 0xFF), + UInt8(value & 0xFF), + ] + bytes += bigEndian ? bigEndianBytes : bigEndianBytes.reversed() + } + return bytes + } + private func rootText(_ node: PureXML.Model.Node) -> String? { guard case let .document(children) = node else { return nil } for child in children {