diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c03609..169db9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Streaming byte decode. `PureXML.parse(pullingBytes:)` and + `events(pullingBytes:)` accept an incremental byte source (`() -> UInt8?`) and + decode UTF-8 or UTF-16 on the fly, so the bytes are never fully buffered. + Encoding is detected from the leading bytes; invalid sequences become the + Unicode replacement character rather than failing the stream. - Namespace resolution. Qualified names now carry an optional resolved `namespaceURI`, populated by the parser from in-scope `xmlns` declarations at the start-element boundary (the libxml2 SAX2 model). A default namespace applies diff --git a/README.md b/README.md index 07579bc..8b18da8 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ flowchart TB Limits["Bounded-by-default limits"]:::done Bytes["Byte input + encoding detection"]:::done Namespaces["Namespace resolution"]:::done - ByteStream["Streaming byte decode"]:::todo + ByteStream["Streaming byte decode"]:::done DTD["Optional DTD + entities"]:::todo Validation["Schema validation"]:::todo XPath["XPath 1.0"]:::todo diff --git a/Sources/API.swift b/Sources/API.swift index d81f33e..73be113 100644 --- a/Sources/API.swift +++ b/Sources/API.swift @@ -45,6 +45,25 @@ public extension PureXML { Parsing.EventReader(pulling: pull, limits: limits) } + /// Parses an XML document from an incremental byte source, decoding the + /// encoding on the fly so the bytes are never fully buffered. + static func parse( + pullingBytes pull: @escaping () -> UInt8?, + limits: Parsing.Limits = .default, + ) throws -> Model.Node { + try Parsing.Parser().parse(pullingBytes: pull, limits: limits) + } + + /// Returns a streaming ``Parsing/EventReader`` over an incremental byte + /// source, decoding the encoding on the fly. + static func events( + pullingBytes pull: @escaping () -> UInt8?, + limits: Parsing.Limits = .default, + ) -> Parsing.EventReader { + var decoder = Parsing.StreamingDecoder(pullingBytes: pull) + return Parsing.EventReader(pulling: { decoder.next() }, limits: limits) + } + /// Serializes a ``Model/Node`` tree into XML with the selected options. static func serialize( _ node: Model.Node, diff --git a/Sources/Parsing/Parser.swift b/Sources/Parsing/Parser.swift index 72d2ad9..ec46658 100644 --- a/Sources/Parsing/Parser.swift +++ b/Sources/Parsing/Parser.swift @@ -36,6 +36,17 @@ public extension PureXML.Parsing { try build(EventReader(pulling: pull, limits: limits)) } + /// Parses a single XML document from an incremental BYTE source, decoding + /// the encoding (UTF-8 or UTF-16) on the fly so the bytes are never fully + /// buffered. The closure returns the next byte or nil at end of input. + public func parse( + pullingBytes pull: @escaping () -> UInt8?, + limits: Limits = .default, + ) throws -> PureXML.Model.Node { + var decoder = StreamingDecoder(pullingBytes: pull) + return try build(EventReader(pulling: { decoder.next() }, limits: limits)) + } + private func build(_ source: EventReader) throws -> PureXML.Model.Node { var reader = source var roots: [PureXML.Model.Node] = [] diff --git a/Sources/Parsing/StreamingDecoder.swift b/Sources/Parsing/StreamingDecoder.swift new file mode 100644 index 0000000..c0cb57e --- /dev/null +++ b/Sources/Parsing/StreamingDecoder.swift @@ -0,0 +1,121 @@ +extension PureXML.Parsing { + /// Decodes a byte stream into characters incrementally. It detects the + /// encoding from the leading bytes (buffering only those few), strips any + /// byte-order mark, then yields one Unicode scalar at a time as a Character, + /// so a byte source is never fully materialized. Invalid sequences yield the + /// Unicode replacement character rather than failing the stream. + /// + /// This works at scalar granularity; a multi-scalar grapheme cluster is + /// surfaced as separate characters, which is harmless for XML tokenization. + struct StreamingDecoder { + private let pull: () -> UInt8? + private var encoding: InputEncoding? + private var prefix: [UInt8] = [] + private var prefixIndex = 0 + private var finished = false + + private static let replacement: Character = "\u{FFFD}" + + init(pullingBytes pull: @escaping () -> UInt8?) { + self.pull = pull + } + + /// Returns the next decoded character, or nil at the end of the stream. + mutating func next() -> Character? { + if encoding == nil { + detect() + } + guard !finished, let encoding else { + return nil + } + switch encoding { + case .utf8: + return nextUTF8() + case .utf16BigEndian: + return nextUTF16(bigEndian: true) + case .utf16LittleEndian: + return nextUTF16(bigEndian: false) + } + } + + private mutating func detect() { + while prefix.count < 4, let byte = pull() { + prefix.append(byte) + } + let detected = InputEncoding.detectWithBOM(prefix) + encoding = detected.encoding + prefixIndex = detected.bomLength + } + + private mutating func nextByte() -> UInt8? { + if prefixIndex < prefix.count { + let byte = prefix[prefixIndex] + prefixIndex += 1 + return byte + } + return pull() + } + + private mutating func nextUTF8() -> Character? { + guard let first = nextByte() else { + finished = true + return nil + } + if first < 0x80 { + return Character(Unicode.Scalar(first)) + } + let length: Int + var value: UInt32 + if first & 0xE0 == 0xC0 { + length = 2 + value = UInt32(first & 0x1F) + } else if first & 0xF0 == 0xE0 { + length = 3 + value = UInt32(first & 0x0F) + } else if first & 0xF8 == 0xF0 { + length = 4 + value = UInt32(first & 0x07) + } else { + return Self.replacement + } + for _ in 1 ..< length { + guard let continuation = nextByte(), continuation & 0xC0 == 0x80 else { + return Self.replacement + } + value = (value << 6) | UInt32(continuation & 0x3F) + } + guard let scalar = Unicode.Scalar(value) else { + return Self.replacement + } + return Character(scalar) + } + + private mutating func nextUTF16(bigEndian: Bool) -> Character? { + guard let unit = nextUnit(bigEndian: bigEndian) else { + finished = true + return nil + } + if (0xD800 ... 0xDBFF).contains(unit) { + guard let low = nextUnit(bigEndian: bigEndian), (0xDC00 ... 0xDFFF).contains(low) else { + return Self.replacement + } + let value = 0x10000 + (UInt32(unit - 0xD800) << 10) + UInt32(low - 0xDC00) + guard let scalar = Unicode.Scalar(value) else { + return Self.replacement + } + return Character(scalar) + } + guard let scalar = Unicode.Scalar(unit) else { + return Self.replacement + } + return Character(scalar) + } + + private mutating func nextUnit(bigEndian: Bool) -> UInt16? { + guard let high = nextByte(), let low = nextByte() else { + return nil + } + return bigEndian ? (UInt16(high) << 8 | UInt16(low)) : (UInt16(low) << 8 | UInt16(high)) + } + } +} diff --git a/Tests/EncodingTests.swift b/Tests/EncodingTests.swift index 36a5ddc..61d3db4 100644 --- a/Tests/EncodingTests.swift +++ b/Tests/EncodingTests.swift @@ -30,6 +30,20 @@ struct EncodingTests { #expect(rootText(littleEndianNode) == "hi") } + @Test("Decodes a UTF-8 byte stream incrementally (multi-byte scalars)") + func test_streamingUTF8() throws { + let node = try PureXML.parse(pullingBytes: byteSource(Array("café \u{1F600}".utf8))) + #expect(rootText(node) == "café \u{1F600}") + } + + @Test("Decodes a UTF-16 byte stream incrementally, both byte orders") + func test_streamingUTF16() throws { + let bigEndianNode = try PureXML.parse(pullingBytes: byteSource(utf16("héllo", bigEndian: true))) + #expect(rootText(bigEndianNode) == "héllo") + let littleEndianNode = try PureXML.parse(pullingBytes: byteSource(utf16("héllo", bigEndian: false))) + #expect(rootText(littleEndianNode) == "héllo") + } + @Test("Odd-length UTF-16 input is rejected as malformed") func test_malformedUTF16() { #expect(throws: PureXML.Parsing.ParseError.malformedEncoding) { @@ -37,6 +51,15 @@ struct EncodingTests { } } + private func byteSource(_ bytes: [UInt8]) -> () -> UInt8? { + var index = 0 + return { + guard index < bytes.count else { return nil } + defer { index += 1 } + return bytes[index] + } + } + private func utf16(_ string: String, bigEndian: Bool) -> [UInt8] { var bytes: [UInt8] = bigEndian ? [0xFE, 0xFF] : [0xFF, 0xFE] for unit in string.utf16 {