From e62995a7a84bd68bbdb82dd51045df5cb8071c2d Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Tue, 9 Jun 2026 09:32:39 +0200 Subject: [PATCH] feat(emitting): output encoding for UTF and single-byte families Add PureXML.Parsing.ByteEncoder, the inverse of ByteDecoder, and a Serializer.serialize(_:encoding:) / PureXML.serialize(_:encoding:) that emits bytes in a chosen encoding (the libxml2 save-with-encoding model). Covers the Unicode transformation formats and the single-byte legacy families, built by inverting the existing vendored to-Unicode tables, so no new data is introduced. The declaration carries the canonical encoding name, a UTF-16/32 stream is preceded by a byte-order mark, and any unrepresentable scalar becomes a decimal numeric character reference, so output round-trips. CJK output encodings remain (tracked on #106). Part of #106, #105 --- CHANGELOG.md | 1 + README.md | 2 +- Sources/API.swift | 12 +++ Sources/Emitting/Serializer.swift | 14 ++++ Sources/Parsing/ByteEncoder.swift | 121 ++++++++++++++++++++++++++++++ Tests/OutputEncodingTests.swift | 54 +++++++++++++ 6 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 Sources/Parsing/ByteEncoder.swift create mode 100644 Tests/OutputEncodingTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 2669131..79b2732 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Output encoding (#106, in progress): `Serializer.serialize(_:encoding:)` and `PureXML.serialize(_:encoding:)` now emit bytes in a chosen encoding (the libxml2 `xmlSaveToFilename(..., encoding)` model), not only UTF-8. A new `PureXML.Parsing.ByteEncoder` inverts the existing vendored to-Unicode tables (no new data) to cover the Unicode transformation formats and the single-byte legacy families (ISO-8859, Windows code pages, KOI8). The declaration carries the encoding's canonical name, a UTF-16/32 stream is preceded by a byte-order mark, and any scalar the target encoding cannot represent is written as a decimal numeric character reference, so the output always round-trips. CJK output encodings are tracked separately. - Big5 (#103): the byte decoder gains the WHATWG `big5` algorithm, for both the whole-buffer and streaming decoders, selected by the `big5` declaration name (and the `big5-hkscs` / `cn-big5` aliases). A lead in `0x81`-`0xFE` followed by a trail in `0x40`-`0x7E` or `0xA1`-`0xFE` resolves through the index, covering Traditional Chinese and the HKSCS extensions including the astral CJK ideographs (Extension B and the Compatibility Supplement). Four pointers decode to a base letter plus a combining mark (a two-scalar sequence). The 18,590-entry index is vendored verbatim from the WHATWG `index-big5.txt`. Big5 completes the major CJK families libxml2 reaches through iconv (Japanese, Korean, Simplified and Traditional Chinese). - Full GB18030 (#102): the byte decoder gains the GB18030 four-byte sequences on top of the GBK two-byte subset, for both the whole-buffer and streaming decoders, selected by the `gb18030` declaration name. A lead in `0x81`-`0xFE` followed by a digit opens a four-byte form whose pointer is mapped through the WHATWG `index-gb18030-ranges` linear ranges, reaching the supplementary planes (the `189000` range maps to `U+10000` and beyond). The 207 ranges are vendored verbatim from the WHATWG `index-gb18030-ranges.txt`. GB18030 is China's mandatory standard, supported in libxml2 through iconv. - CJK multi-byte encodings (#99, in progress): the byte decoder gains a multi-byte path and the WHATWG **Shift-JIS** and **EUC-JP** algorithms, for both the whole-buffer and streaming decoders, selected by the XML declaration's `encoding` name. Shift-JIS covers ASCII, half-width katakana, and the JIS X 0208 plane; EUC-JP adds the `0x8E` half-width katakana and `0x8F` JIS X 0212 leads. EUC-KR (CP949/UHC) decodes Hangul and Hanja through the WHATWG `index-euc-kr`, and GBK (which also covers GB2312) decodes Chinese through the two-byte range of `index-gb18030` with `0x80` as the euro sign. The JIS X 0208/0212, CP949, and GBK indexes are all vendored verbatim from the corresponding WHATWG `index-*.txt` files. This completes the CJK multi-byte encodings. diff --git a/README.md b/README.md index 723d8be..80604cf 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ flowchart TB PureXML is a working, dependency-free XML library today: parse, emit, validate, query, and stream documents on macOS, Linux, Windows, and WASM. The test suite -currently runs **768 tests in 117 suites** (`swift test`). +currently runs **773 tests in 118 suites** (`swift test`). ### Shipped (libxml2-aligned surface) diff --git a/Sources/API.swift b/Sources/API.swift index 0b3e8cf..b9ad2ee 100644 --- a/Sources/API.swift +++ b/Sources/API.swift @@ -126,6 +126,18 @@ public extension PureXML { Emitting.Serializer(options: options).serialize(node) } + /// Serializes a ``Model/Node`` tree into bytes in `encoding` (the libxml2 + /// save-with-encoding model): the declaration carries the encoding name, a + /// UTF-16/32 stream is preceded by a byte-order mark, and any unrepresentable + /// scalar becomes a numeric character reference. + static func serialize( + _ node: Model.Node, + encoding: Parsing.InputEncoding, + options: Emitting.Options = .default, + ) -> [UInt8] { + Emitting.Serializer(options: options).serialize(node, encoding: encoding) + } + /// Returns a pull cursor (the libxml2 `xmlTextReader` model) over an XML /// string. Call `read()` to advance node by node, reading `nodeKind`, `name`, /// `value`, `depth`, and `attributes` at each step. diff --git a/Sources/Emitting/Serializer.swift b/Sources/Emitting/Serializer.swift index 2f3c70d..a6a6ec1 100644 --- a/Sources/Emitting/Serializer.swift +++ b/Sources/Emitting/Serializer.swift @@ -28,6 +28,20 @@ public extension PureXML.Emitting { self.options = options } + /// Serializes a node into bytes in `encoding`, the libxml2 + /// `xmlSaveToFilename(..., encoding)` model. The XML declaration carries the + /// encoding's canonical name, a byte-order mark precedes a UTF-16/32 stream, + /// and any scalar the encoding cannot represent becomes a numeric character + /// reference. Falls back to UTF-8 for an encoding without an output table. + public func serialize(_ node: PureXML.Model.Node, encoding: PureXML.Parsing.InputEncoding) -> [UInt8] { + var encodingOptions = options + encodingOptions.includeXMLDeclaration = true + encodingOptions.encodingName = PureXML.Parsing.ByteEncoder.canonicalName(encoding) + let text = Serializer(options: encodingOptions).serialize(node) + return PureXML.Parsing.ByteEncoder.byteOrderMark(encoding) + + PureXML.Parsing.ByteEncoder.encode(text, as: encoding) + } + /// Serializes a node into an XML string. public func serialize(_ node: PureXML.Model.Node) -> String { var output = "" diff --git a/Sources/Parsing/ByteEncoder.swift b/Sources/Parsing/ByteEncoder.swift new file mode 100644 index 0000000..b2178d0 --- /dev/null +++ b/Sources/Parsing/ByteEncoder.swift @@ -0,0 +1,121 @@ +public extension PureXML.Parsing { + /// Encodes a Unicode string into bytes for output, the inverse of + /// ``ByteDecoder``. Covers the Unicode transformation formats and the + /// single-byte legacy families (built by inverting the same vendored + /// to-Unicode tables, so no new data is introduced). A scalar the target + /// encoding cannot represent is written as a decimal numeric character + /// reference, matching libxml2's save behavior. + enum ByteEncoder { + /// Whether `encoding` can be produced by ``encode(_:as:)``. + public static func supports(_ encoding: InputEncoding) -> Bool { + switch encoding { + case .utf8, .utf16BigEndian, .utf16LittleEndian, .utf32BigEndian, .utf32LittleEndian: + true + default: + ByteDecoder.singleByteMap(encoding) != nil + } + } + + /// The bytes of `string` in `encoding`. Unrepresentable scalars become + /// `&#NNNN;` character references (whose own bytes are ASCII, encodable + /// everywhere). Returns the UTF-8 bytes for an unsupported encoding. + public static func encode(_ string: String, as encoding: InputEncoding) -> [UInt8] { + switch encoding { + case .utf8: Array(string.utf8) + case .utf16BigEndian: utf16(string, bigEndian: true) + case .utf16LittleEndian: utf16(string, bigEndian: false) + case .utf32BigEndian: utf32(string, bigEndian: true) + case .utf32LittleEndian: utf32(string, bigEndian: false) + default: singleByte(string, encoding: encoding) + } + } + + /// The byte-order mark for `encoding` (empty for the byte-oriented ones). + public static func byteOrderMark(_ encoding: InputEncoding) -> [UInt8] { + switch encoding { + case .utf16BigEndian: [0xFE, 0xFF] + case .utf16LittleEndian: [0xFF, 0xFE] + case .utf32BigEndian: [0x00, 0x00, 0xFE, 0xFF] + case .utf32LittleEndian: [0xFF, 0xFE, 0x00, 0x00] + default: [] + } + } + + // MARK: Unicode transformation formats + + private static func utf16(_ string: String, bigEndian: Bool) -> [UInt8] { + var bytes: [UInt8] = [] + for unit in string.utf16 { + let pair: [UInt8] = [UInt8(unit >> 8), UInt8(unit & 0xFF)] + bytes.append(contentsOf: bigEndian ? pair : pair.reversed()) + } + return bytes + } + + private static func utf32(_ string: String, bigEndian: Bool) -> [UInt8] { + var bytes: [UInt8] = [] + for scalar in string.unicodeScalars { + let value = scalar.value + let quad: [UInt8] = [ + UInt8(value >> 24 & 0xFF), + UInt8(value >> 16 & 0xFF), + UInt8(value >> 8 & 0xFF), + UInt8(value & 0xFF), + ] + bytes.append(contentsOf: bigEndian ? quad : quad.reversed()) + } + return bytes + } + + // MARK: Single-byte families + + private static func singleByte(_ string: String, encoding: InputEncoding) -> [UInt8] { + guard let inverse = singleByteInverse(encoding) else { return Array(string.utf8) } + var bytes: [UInt8] = [] + for scalar in string.unicodeScalars { + if let byte = inverse[scalar.value] { + bytes.append(byte) + } else { + bytes.append(contentsOf: Array("&#\(scalar.value);".utf8)) + } + } + return bytes + } + + /// The scalar-to-byte map for a single-byte encoding, inverted from its + /// forward table. The lowest byte wins a collision, and the `0xFFFD` + /// sentinel for an unmapped byte is skipped. + private static func singleByteInverse(_ encoding: InputEncoding) -> [UInt32: UInt8]? { + guard let forward = ByteDecoder.singleByteMap(encoding) else { return nil } + var map: [UInt32: UInt8] = [:] + for byte in UInt8.min ... UInt8.max { + let value = forward(byte).value + if value == 0xFFFD { continue } + if map[value] == nil { map[value] = byte } + } + return map + } + + // MARK: Declaration names + + /// The canonical encoding name to write in the XML declaration. + public static func canonicalName(_ encoding: InputEncoding) -> String { + canonicalNames[encoding] ?? "UTF-8" + } + + private static let canonicalNames: [InputEncoding: String] = [ + .utf8: "UTF-8", + .utf16BigEndian: "UTF-16", .utf16LittleEndian: "UTF-16", + .utf32BigEndian: "UTF-32", .utf32LittleEndian: "UTF-32", + .latin1: "ISO-8859-1", .latin2: "ISO-8859-2", .latin3: "ISO-8859-3", + .latin4: "ISO-8859-4", .latinCyrillic: "ISO-8859-5", .arabic: "ISO-8859-6", + .greek: "ISO-8859-7", .hebrew: "ISO-8859-8", .latin5: "ISO-8859-9", + .latin6: "ISO-8859-10", .thai: "ISO-8859-11", .latin7: "ISO-8859-13", + .latin8: "ISO-8859-14", .latin9: "ISO-8859-15", .latin10: "ISO-8859-16", + .windows1250: "windows-1250", .windows1251: "windows-1251", .windows1252: "windows-1252", + .windows1253: "windows-1253", .windows1254: "windows-1254", .windows1255: "windows-1255", + .windows1256: "windows-1256", .windows1257: "windows-1257", .windows1258: "windows-1258", + .koi8r: "KOI8-R", .koi8u: "KOI8-U", + ] + } +} diff --git a/Tests/OutputEncodingTests.swift b/Tests/OutputEncodingTests.swift new file mode 100644 index 0000000..cba7252 --- /dev/null +++ b/Tests/OutputEncodingTests.swift @@ -0,0 +1,54 @@ +@testable import PureXML +import Testing + +@Suite("Output encoding") +struct OutputEncodingTests { + private func rootText(_ node: PureXML.Model.Node) -> String? { + guard case let .document(children) = node else { return nil } + for child in children { + if case let .element(element) = child { return element.text } + } + return nil + } + + private func serialized(_ xml: String, _ encoding: PureXML.Parsing.InputEncoding) throws -> [UInt8] { + try PureXML.serialize(PureXML.parse(xml), encoding: encoding) + } + + @Test("ISO-8859-1 emits one byte per Latin-1 scalar and round-trips") + func test_latin1() throws { + let bytes = try serialized("caf\u{E9}", .latin1) // café + #expect(bytes.contains(0xE9)) // é is a single 0xE9 byte + #expect(!bytes.contains(0xC3)) // not the UTF-8 lead byte of é + try #expect(rootText(PureXML.parse(bytes: bytes)) == "caf\u{E9}") + } + + @Test("ISO-8859-7 round-trips Greek and windows-1252 round-trips its punctuation") + func test_greekAndWindows() throws { + try #expect(rootText(PureXML.parse(bytes: serialized("\u{0391}", .greek))) == "\u{0391}") // Α + try #expect(rootText(PureXML.parse(bytes: serialized("\u{20AC}\u{201C}", .windows1252))) == "\u{20AC}\u{201C}") + } + + @Test("An unrepresentable scalar becomes a numeric character reference") + func test_characterReferenceFallback() throws { + // U+4E2D (中) is not in Latin-1, so it is written as 中 (decimal). + let bytes = try serialized("\u{4E2D}", .latin1) + let text = String(decoding: bytes, as: UTF8.self) + #expect(text.contains("中")) + // The reference bytes are ASCII, so they re-parse to the original scalar. + try #expect(rootText(PureXML.parse(bytes: bytes)) == "\u{4E2D}") + } + + @Test("The declaration carries the canonical encoding name") + func test_declarationName() throws { + let text = try String(decoding: serialized("x", .latin1), as: UTF8.self) + #expect(text.hasPrefix("")) + } + + @Test("UTF-16 output carries a byte-order mark and round-trips") + func test_utf16() throws { + let bytes = try serialized("hi\u{4E2D}", .utf16BigEndian) + #expect(Array(bytes.prefix(2)) == [0xFE, 0xFF]) // BOM + try #expect(rootText(PureXML.parse(bytes: bytes)) == "hi\u{4E2D}") + } +}