From 191d2245de4bfc8d5578510f5ed65a44dd5c8cac Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Tue, 9 Jun 2026 17:33:52 +0200 Subject: [PATCH] feat(parsing): streaming honors the declared encoding; streaming ISO-2022-JP The streaming decoder previously selected an encoding only from a byte-order mark or the UTF sniff, so its single-byte and CJK decode paths were latent. It now reads the encoding from the XML declaration (the libxml2 behavior) when there is no BOM, activating those paths over an incremental byte source. Add the streaming form of ISO-2022-JP, carrying the stateful mode across reads (extracted to its own file to stay under the type-body cap). A test asserts streaming agrees with whole-buffer decode for a single-byte (ISO-8859-1), a CJK (Shift_JIS), and an ISO-2022-JP document. Part of #108, #105 --- .claude/commands/exp | 1 + .claude/commands/openapi | 1 + .claude/commands/swift | 1 + .claude/commands/ui | 1 + CHANGELOG.md | 2 +- README.md | 2 +- Sources/Parsing/ISO2022JPDecoder.swift | 4 +- Sources/Parsing/StreamingDecoder.swift | 22 ++++-- .../Parsing/StreamingDecoderISO2022JP.swift | 71 +++++++++++++++++++ Tests/EncodingTests.swift | 25 +++++++ 10 files changed, 122 insertions(+), 8 deletions(-) create mode 120000 .claude/commands/exp create mode 120000 .claude/commands/openapi create mode 120000 .claude/commands/swift create mode 120000 .claude/commands/ui create mode 100644 Sources/Parsing/StreamingDecoderISO2022JP.swift diff --git a/.claude/commands/exp b/.claude/commands/exp new file mode 120000 index 0000000..491b78f --- /dev/null +++ b/.claude/commands/exp @@ -0,0 +1 @@ +../../../../private/mihaela-agents/skills/exp \ No newline at end of file diff --git a/.claude/commands/openapi b/.claude/commands/openapi new file mode 120000 index 0000000..773aa45 --- /dev/null +++ b/.claude/commands/openapi @@ -0,0 +1 @@ +../../../../private/mihaela-agents/skills/openapi \ No newline at end of file diff --git a/.claude/commands/swift b/.claude/commands/swift new file mode 120000 index 0000000..a0877ee --- /dev/null +++ b/.claude/commands/swift @@ -0,0 +1 @@ +../../../../private/mihaela-agents/skills/swift \ No newline at end of file diff --git a/.claude/commands/ui b/.claude/commands/ui new file mode 120000 index 0000000..5dafd44 --- /dev/null +++ b/.claude/commands/ui @@ -0,0 +1 @@ +../../../../private/mihaela-agents/skills/ui \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index d44cb32..e2602b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- ISO-2022-JP decoding (#108, in progress): the byte decoder gains the WHATWG `iso-2022-jp` algorithm, selected by the `iso-2022-jp` declaration name. Unlike the other CJK encodings this one is stateful, escape sequences switch between ASCII, JIS X 0201 Roman (with the yen sign and overline), JIS X 0201 katakana, and the JIS X 0208 plane, which is resolved through the shared `jis0208` table (no new data). EUC-TW and the streaming form of ISO-2022-JP remain on the issue. +- ISO-2022-JP decoding (#108, in progress): the byte decoder gains the WHATWG `iso-2022-jp` algorithm, selected by the `iso-2022-jp` declaration name. Unlike the other CJK encodings this one is stateful, escape sequences switch between ASCII, JIS X 0201 Roman (with the yen sign and overline), JIS X 0201 katakana, and the JIS X 0208 plane, which is resolved through the shared `jis0208` table (no new data). The streaming decoder now honors the declared encoding from the XML declaration (the libxml2 behavior), not only a byte-order mark, so the single-byte and multi-byte CJK decoders, including the stateful ISO-2022-JP whose mode is carried across reads, work over an incremental byte source; a test asserts streaming agrees with whole-buffer decode for a single-byte, a CJK, and an ISO-2022-JP document. EUC-TW remains on the issue. - Streaming (reader-driven) DTD validation (#107, in progress): `PureXML.validate(streaming:dtd:)` and `PureXML.Validation.StreamingDTDValidator` validate a document against a DTD while it is pulled event by event (the libxml2 `xmlTextReader` validation model), holding only the open-element stack rather than the whole tree. Each element's content model and attributes are checked the moment it closes, by applying the exact per-element DTD rules to a shallow element synthesized from the streamed child names. Document-scoped ID/IDREF integrity is tracked in bounded sets (the identifier values, not the tree): a duplicate ID is reported at its repeat occurrence, and forward IDREFs resolve at the end of the stream. Streaming and tree validation report the same problems (a test asserts matching reasons), and errors carry a libxml2 `xmlGetNodePath`-style coding path. RELAX NG streaming is also available via `RelaxNG.validate(streaming:)`: the Brzozowski derivative is itself incremental, so it is driven directly from the event stream, retaining only the residual pattern (the `after`-threaded chain is the validation stack) and a light per-element frame for ignorable-whitespace handling; a differential test asserts it agrees with the tree engine across valid, invalid, whitespace, and mixed-content documents. XSD streaming is available via `Schema.Document.validate(streaming:)`: each element's type is resolved from its parent's content model (and `xsi:type` / `typeReference`) as it opens, and its attributes and content-model structure are checked shallowly as it closes (`ComplexValidator.validateShallow`), with a differential test asserting agreement with the tree validator. Wildcard `processContents` and substitution-group resolution in the streaming form stay on the tree validator. - Output encoding (#106): `Serializer.serialize(_:encoding:)` and `PureXML.serialize(_:encoding:)` now emit bytes in a chosen encoding (the libxml2 `xmlSaveToFilename(..., encoding)` model), not only UTF-8. A new `PureXML.Parsing.ByteEncoder` covers the Unicode transformation formats, the single-byte legacy families (ISO-8859, Windows code pages, KOI8), and the multi-byte CJK encodings (Shift-JIS, EUC-JP, EUC-KR, GBK, GB18030, Big5). The CJK and single-byte inverses are built by running the existing, tested decoders over every valid byte sequence and recording the shortest sequence per scalar, so each encoder is by construction the exact inverse of its decoder (no new data, no separately maintained encode tables); GB18030's astral range uses the four-byte pointer formula. The declaration carries the encoding's canonical name, a UTF-16/32 stream is preceded by a byte-order mark, and any scalar the target encoding cannot represent is written as a decimal numeric character reference, so output always round-trips. - Big5 (#103): the byte decoder gains the WHATWG `big5` algorithm, for both the whole-buffer and streaming decoders, selected by the `big5` declaration name (and the `big5-hkscs` / `cn-big5` aliases). A lead in `0x81`-`0xFE` followed by a trail in `0x40`-`0x7E` or `0xA1`-`0xFE` resolves through the index, covering Traditional Chinese and the HKSCS extensions including the astral CJK ideographs (Extension B and the Compatibility Supplement). Four pointers decode to a base letter plus a combining mark (a two-scalar sequence). The 18,590-entry index is vendored verbatim from the WHATWG `index-big5.txt`. Big5 completes the major CJK families libxml2 reaches through iconv (Japanese, Korean, Simplified and Traditional Chinese). diff --git a/README.md b/README.md index 23a5df1..91bbe40 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ flowchart TB PureXML is a working, dependency-free XML library today: parse, emit, validate, query, and stream documents on macOS, Linux, Windows, and WASM. The test suite -currently runs **788 tests in 121 suites** (`swift test`). +currently runs **789 tests in 121 suites** (`swift test`). ### Shipped (libxml2-aligned surface) diff --git a/Sources/Parsing/ISO2022JPDecoder.swift b/Sources/Parsing/ISO2022JPDecoder.swift index 665ef7e..ee7a40c 100644 --- a/Sources/Parsing/ISO2022JPDecoder.swift +++ b/Sources/Parsing/ISO2022JPDecoder.swift @@ -97,8 +97,8 @@ extension PureXML.Parsing.ByteDecoder { } /// The active character set while decoding ISO-2022-JP, switched by escape -/// sequences. -private enum ISO2022JPMode { +/// sequences. Held by the streaming decoder across reads, so it is module-visible. +enum ISO2022JPMode { case ascii case roman case katakana diff --git a/Sources/Parsing/StreamingDecoder.swift b/Sources/Parsing/StreamingDecoder.swift index fb03514..100342a 100644 --- a/Sources/Parsing/StreamingDecoder.swift +++ b/Sources/Parsing/StreamingDecoder.swift @@ -12,9 +12,10 @@ extension PureXML.Parsing { private var encoding: InputEncoding? private var prefix: [UInt8] = [] private var prefixIndex = 0 - private var finished = false + var finished = false + var iso2022jpMode: ISO2022JPMode = .ascii - private static let replacement: Character = "\u{FFFD}" + static let replacement: Character = "\u{FFFD}" init(pullingBytes pull: @escaping () -> UInt8?) { self.pull = pull @@ -55,6 +56,7 @@ extension PureXML.Parsing { case .gbk: nextGBK() case .gb18030: nextGB18030() case .big5: nextBig5() + case .iso2022jp: nextISO2022JP() default: nextUTF8() } } @@ -201,11 +203,23 @@ extension PureXML.Parsing { prefix.append(byte) } let detected = InputEncoding.detectWithBOM(prefix) - encoding = detected.encoding prefixIndex = detected.bomLength + encoding = detected.encoding + // With no byte-order mark and an ASCII-compatible default, the XML + // declaration may name another encoding (the libxml2 behavior). Buffer + // through the declaration and honor it; the ASCII declaration bytes + // decode identically under the chosen encoding when replayed. + if detected.bomLength == 0, detected.encoding == .utf8 { + while prefix.count < 256, !prefix.contains(0x3E), let byte = pull() { + prefix.append(byte) + } + if let declared = PureXML.Parsing.ByteDecoder.declaredEncoding(in: prefix) { + encoding = declared + } + } } - private mutating func nextByte() -> UInt8? { + mutating func nextByte() -> UInt8? { if prefixIndex < prefix.count { let byte = prefix[prefixIndex] prefixIndex += 1 diff --git a/Sources/Parsing/StreamingDecoderISO2022JP.swift b/Sources/Parsing/StreamingDecoderISO2022JP.swift new file mode 100644 index 0000000..7b60eb1 --- /dev/null +++ b/Sources/Parsing/StreamingDecoderISO2022JP.swift @@ -0,0 +1,71 @@ +extension PureXML.Parsing.StreamingDecoder { + /// Streams one ISO-2022-JP character, carrying the active mode across calls. + /// Escape sequences switch mode and yield no character, so it loops until a + /// character is produced or the stream ends. + mutating func nextISO2022JP() -> Character? { + while true { + guard let byte = nextByte() else { + finished = true + return nil + } + if byte == 0x1B { + guard applyISO2022JPEscape() else { return Self.replacement } + continue + } + return decodeISO2022JPByte(byte) + } + } + + private mutating func decodeISO2022JPByte(_ byte: UInt8) -> Character { + switch iso2022jpMode { + case .ascii: + return byte <= 0x7F ? Character(Unicode.Scalar(byte)) : Self.replacement + case .roman: + return romanScalar(byte) + case .katakana: + guard (0x21 ... 0x5F).contains(byte) else { return Self.replacement } + return Character(Unicode.Scalar(0xFF61 + UInt32(byte) - 0x21) ?? "\u{FFFD}") + case .jis0208: + return jis0208Character(byte) + } + } + + private func romanScalar(_ byte: UInt8) -> Character { + switch byte { + case 0x5C: "\u{00A5}" // yen + case 0x7E: "\u{203E}" // overline + case 0 ... 0x7F: Character(Unicode.Scalar(byte)) + default: Self.replacement + } + } + + private mutating func jis0208Character(_ lead: UInt8) -> Character { + guard (0x21 ... 0x7E).contains(lead) else { + return lead == 0x0A || lead == 0x0D ? Character(Unicode.Scalar(lead)) : Self.replacement + } + guard let trail = nextByte() else { + finished = true + return Self.replacement + } + guard (0x21 ... 0x7E).contains(trail) else { return Self.replacement } + let pointer = (Int(lead) - 0x21) * 94 + Int(trail) - 0x21 + let table = PureXML.Parsing.ByteDecoder.jis0208 + guard pointer >= 0, pointer < table.count, table[pointer] != 0xFFFF else { return Self.replacement } + return Character(Unicode.Scalar(UInt32(table[pointer])) ?? "\u{FFFD}") + } + + private mutating func applyISO2022JPEscape() -> Bool { + guard let first = nextByte(), let second = nextByte() else { + finished = true + return false + } + switch (first, second) { + case (0x28, 0x42): iso2022jpMode = .ascii + case (0x28, 0x4A): iso2022jpMode = .roman + case (0x28, 0x49): iso2022jpMode = .katakana + case (0x24, 0x40), (0x24, 0x42): iso2022jpMode = .jis0208 + default: return false + } + return true + } +} diff --git a/Tests/EncodingTests.swift b/Tests/EncodingTests.swift index 2269838..0c416ee 100644 --- a/Tests/EncodingTests.swift +++ b/Tests/EncodingTests.swift @@ -44,6 +44,31 @@ struct EncodingTests { #expect(rootText(littleEndianNode) == "héllo") } + @Test("Streaming honors the declared encoding and agrees with whole-buffer decode") + func test_streamingDeclaredEncoding() throws { + func doc(_ encoding: String, _ content: [UInt8]) -> [UInt8] { + Array("".utf8) + content + Array("".utf8) + } + // Single-byte, CJK, and the stateful ISO-2022-JP, none detectable from + // bytes alone, so each exercises declaration-driven streaming. + struct Case { let encoding: String + let content: [UInt8] + let expected: String + } + let cases = [ + Case(encoding: "ISO-8859-1", content: [0xE9], expected: "\u{00E9}"), // é + Case(encoding: "Shift_JIS", content: [0xB1], expected: "\u{FF71}"), // ｱ half-width katakana + Case(encoding: "ISO-2022-JP", content: [0x1B, 0x28, 0x49, 0x31, 0x1B, 0x28, 0x42], expected: "\u{FF71}"), // ｱ via escape + ] + for testCase in cases { + let bytes = doc(testCase.encoding, testCase.content) + let whole = try rootText(PureXML.parse(bytes: bytes)) + let streamed = try rootText(PureXML.parse(pullingBytes: byteSource(bytes))) + #expect(whole == testCase.expected, "whole-buffer wrong for \(testCase.encoding)") + #expect(streamed == whole, "streaming disagreed with whole-buffer for \(testCase.encoding)") + } + } + @Test("Odd-length UTF-16 input is rejected as malformed") func test_malformedUTF16() { #expect(throws: PureXML.Parsing.ParseError.malformedEncoding) {