diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eb3e6e..1b78a38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Push / feed streaming API (#1), the last epic. A resumable scanner reports a + token only once its terminator is fully buffered and otherwise signals + need-more (the Expat `XML_TOK_PARTIAL` model), so `PureXML.Parsing.PushParser` + parses input fed in arbitrary chunks with `feed(_:)`/`finish()`, driving a + `SAXHandler`, while retaining only the current incomplete token plus the + open-element stack. `PureXML.events(feeding:)` exposes an `AsyncThrowingStream` + of events over any async sequence of text chunks. Splitting a document at any + byte boundary, down to one character per feed, yields identical events. - HTML parser and serializer (#20, the libxml2 `HTMLparser.h`/`HTMLtree.h` model). `PureXML.HTML.parse` reads tag-soup HTML leniently (case-insensitive tags; quoted, unquoted, and boolean attributes; comments; doctype; character diff --git a/README.md b/README.md index ea2b2ef..c7570bd 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ flowchart TB LReview["PR in review"]:::review LEpic["Epic grouping"]:::epic LTodo["Open issue, no PR"]:::todo + Core["Core shipped: #1 push, #19 SAX, #28 encoding, #29 entities, #31 tree, #32 writer, #33 reader, #34 save, #35 chars"]:::done + Query["Query + validation shipped: #21 #22 #23 query, #30 regex, #2 XSD/RELAX NG, #25 Schematron"]:::done + Transform["Transform + HTML shipped: #3 XSLT, #20 HTML, #24 XInclude, #26 C14N, #27 Catalog"]:::done LDone --> LReview LReview --> LEpic LEpic --> LTodo @@ -50,38 +53,7 @@ remaining libxml2 surface is tracked as epics. Deliberate non-goals: network fetching (`nanohttp`/`nanoftp`) and the threading/memory infrastructure stay out, and external resolution is opt-in through an injected resolver so XXE stays closed. -Parsing and I/O: - -```mermaid -flowchart TB - classDef done fill:#ddf9e4,stroke:#34c759,color:#111827 - classDef review fill:#fff7d6,stroke:#ffcc00,color:#111827 - classDef epic fill:#f2e5ff,stroke:#af52de,color:#111827 - classDef todo fill:#f2f4f7,stroke:#8e8e93,color:#111827 - Core["Streaming core + DTD validation (shipped)"]:::done - Push["#1 Push / feed streaming API"]:::epic - Sax["#19 SAX2 push callbacks"]:::done - Html["#20 HTML parser and serializer"]:::done - Enc["#28 Full encoding support"]:::done - Ent["#29 External and parameter entities"]:::done - TreeApi["#31 Tree manipulation API"]:::done - Writer["#32 xmlTextWriter"]:::done - Reader["#33 xmlTextReader with validation"]:::done - Save["#34 Serialization option parity"]:::done - Chars["#35 Character and name validity"]:::done - Query["Query + validation shipped: #21 #22 #23 query, #30 regex, #2 XSD/RELAX NG, #25 Schematron"]:::done - Transform["Transform shipped: #3 XSLT, #24 XInclude, #26 C14N, #27 Catalog"]:::done - Core --> Push - Core --> Sax - Core --> Html - Core --> Enc - Core --> Ent - Core --> TreeApi - Core --> Writer - Core --> Reader - Core --> Save - Core --> Chars -``` +Every tracked epic is now shipped; the diagram above summarizes the surface. ## Status diff --git a/Sources/Parsing/PushAsync.swift b/Sources/Parsing/PushAsync.swift new file mode 100644 index 0000000..d0b5bcc --- /dev/null +++ b/Sources/Parsing/PushAsync.swift @@ -0,0 +1,39 @@ +public extension PureXML { + /// Parses an asynchronous sequence of text chunks (a socket, a file read loop) + /// into a stream of ``Parsing/Event`` values, driving the push parser as each + /// chunk arrives. The document is never held whole. + @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *) + static func events( + feeding chunks: Chunks, + ) -> AsyncThrowingStream where Chunks.Element == String { + AsyncThrowingStream { continuation in + let task = Task { + do { + var parser = Parsing.PushParser(sax: Self.streamingHandler(continuation)) + for try await chunk in chunks { + try parser.feed(chunk) + } + try parser.finish() + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + } + + @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *) + private static func streamingHandler( + _ continuation: AsyncThrowingStream.Continuation, + ) -> Parsing.SAXHandler { + Parsing.SAXHandler( + startElement: { continuation.yield(.startElement(name: $0, attributes: $1)) }, + endElement: { continuation.yield(.endElement(name: $0)) }, + characters: { continuation.yield(.characters($0)) }, + cdata: { continuation.yield(.cdata($0)) }, + comment: { continuation.yield(.comment($0)) }, + processingInstruction: { continuation.yield(.processingInstruction(target: $0, data: $1)) }, + ) + } +} diff --git a/Sources/Parsing/PushParser.swift b/Sources/Parsing/PushParser.swift new file mode 100644 index 0000000..a4181c9 --- /dev/null +++ b/Sources/Parsing/PushParser.swift @@ -0,0 +1,86 @@ +public extension PureXML.Parsing { + /// A push-style XML parser: feed input as it arrives with ``feed(_:)`` and end + /// the document with ``finish()``. It drives a ``SAXHandler``, holding only a + /// small buffer (the current incomplete token) plus the open-element stack, so + /// it never requires the whole document and resumes cleanly across chunk + /// boundaries (the Expat `XML_TOK_PARTIAL` model). + struct PushParser { + private var buffer: [Character] = [] + private var namespaces = NamespaceContext() + private var open: [PureXML.Model.QualifiedName] = [] + private let handler: SAXHandler + private var started = false + + /// Creates a push parser delivering events to `sax`. + public init(sax: SAXHandler) { + handler = sax + } + + /// Feeds the next chunk of input, delivering events for every token that is + /// now complete. + public mutating func feed(_ text: String) throws { + buffer += Array(text) + try drain(final: false) + } + + /// Ends the document, flushing any trailing token and the document-end + /// callback. + public mutating func finish() throws { + ensureStarted() + try drain(final: true) + handler.endDocument?() + } + + private mutating func ensureStarted() { + guard !started else { return } + started = true + handler.startDocument?() + } + + private mutating func drain(final: Bool) throws { + ensureStarted() + while case let .token(token, consumed) = PushScanner.scan(buffer, final: final) { + buffer.removeFirst(consumed) + try deliver(token) + } + } + + private mutating func deliver(_ token: PushToken) throws { + switch token { + case let .open(name, attributes, selfClosing): + try open(name, attributes, selfClosing: selfClosing) + case let .close(name): + close(name) + case let .text(value): + if !value.isEmpty { handler.characters?(value) } + case let .comment(value): + handler.comment?(value) + case let .cdata(value): + handler.cdata?(value) + case let .processingInstruction(target, data): + if target.lowercased() != "xml" { handler.processingInstruction?(target, data) } + case .ignorable: + break + } + } + + private mutating func open(_ name: String, _ attributes: [PushAttribute], selfClosing: Bool) throws { + let rawName = PureXML.Model.QualifiedName(name) + let rawAttributes = attributes.map { PureXML.Model.Attribute(name: .init($0.name), value: $0.value) } + let resolved = try namespaces.enterElement(name: rawName, attributes: rawAttributes, at: .start) + handler.startElement?(resolved.name, resolved.attributes) + if selfClosing { + namespaces.leaveElement() + handler.endElement?(resolved.name) + } else { + open.append(resolved.name) + } + } + + private mutating func close(_: String) { + guard let top = open.popLast() else { return } + namespaces.leaveElement() + handler.endElement?(top) + } + } +} diff --git a/Sources/Parsing/PushScanner.swift b/Sources/Parsing/PushScanner.swift new file mode 100644 index 0000000..585fa87 --- /dev/null +++ b/Sources/Parsing/PushScanner.swift @@ -0,0 +1,242 @@ +/// Reads the name and attributes out of a start tag's interior. File-scope and +/// private: an internal detail of ``PureXML/Parsing/PushScanner``. +private struct TagCursor { + private let chars: [Character] + private var index = 0 + + init(_ chars: [Character]) { + self.chars = chars + } + + mutating func readName() -> String { + skipSpace() + return readNameToken() + } + + mutating func readAttribute() -> PureXML.Parsing.PushAttribute? { + skipSpace() + let name = readNameToken() + guard !name.isEmpty else { return nil } + skipSpace() + guard peek() == "=" else { return .init(name: name, value: "") } + advance() + skipSpace() + return .init(name: name, value: readValue()) + } + + private mutating func readValue() -> String { + guard let quote = peek(), quote == "\"" || quote == "'" else { return readUnquoted() } + advance() + var value = "" + while let character = peek(), character != quote { + value.append(character) + advance() + } + advance() + return decode(value) + } + + private mutating func readUnquoted() -> String { + var value = "" + while let character = peek(), !character.isXMLWhitespace { + value.append(character) + advance() + } + return decode(value) + } + + private mutating func readNameToken() -> String { + var name = "" + while let character = peek(), character.isXMLNameContinuation { + name.append(character) + advance() + } + return name + } + + private func decode(_ value: String) -> String { + var budget = Int.max + return (try? PureXML.Parsing.EntityDecoder.decode(value, entities: [:], budget: &budget, at: .start)) ?? value + } + + private func peek() -> Character? { + index < chars.count ? chars[index] : nil + } + + private mutating func advance() { + if index < chars.count { index += 1 } + } + + private mutating func skipSpace() { + while let character = peek(), character.isXMLWhitespace { + advance() + } + } +} + +extension PureXML.Parsing { + /// One token recognized by the resumable scanner. + enum PushToken: Equatable, Sendable { + case open(name: String, attributes: [PushAttribute], selfClosing: Bool) + case close(name: String) + case text(String) + case comment(String) + case cdata(String) + case processingInstruction(target: String, data: String) + /// An ignorable construct (XML declaration or doctype). + case ignorable + } + + /// A raw start-tag attribute (name and value, before namespace resolution). + struct PushAttribute: Equatable, Sendable { + var name: String + var value: String + } + + /// The outcome of one resumable scan. + enum Scanned: Equatable, Sendable { + /// A complete token and the number of leading characters it consumed. + case token(PushToken, consumed: Int) + /// The buffer ends mid-token; feed more input and scan again from the + /// same position (the Expat `XML_TOK_PARTIAL` model). + case needMore + } + + /// A resumable XML tokenizer. It only reports a token once that token's + /// terminator is fully present in the buffer; otherwise it returns + /// ``Scanned/needMore`` and leaves the buffer untouched so the caller can feed + /// more input and rescan. The only retained state across feeds is the + /// unconsumed buffer, which is bounded by the largest single token. + enum PushScanner { + static func scan(_ buffer: [Character], final: Bool) -> Scanned { + guard let first = buffer.first else { return .needMore } + return first == "<" ? scanMarkup(buffer, final: final) : scanText(buffer, final: final) + } + + // MARK: Text + + private static func scanText(_ buffer: [Character], final: Bool) -> Scanned { + if let lessThan = buffer.firstIndex(of: "<") { + return lessThan == 0 ? scanMarkup(buffer, final: final) + : .token(.text(decode(buffer[0 ..< lessThan])), consumed: lessThan) + } + // No markup ahead: a text run is complete only at the end of input, + // so an entity reference is never split across a feed. + return final ? .token(.text(decode(buffer[...])), consumed: buffer.count) : .needMore + } + + // MARK: Markup + + private static func scanMarkup(_ buffer: [Character], final: Bool) -> Scanned { + if isPrefix(buffer, ofAny: ["") { .comment(String($0)) } } + if startsWith(buffer, "") { .cdata(String($0)) } } + if startsWith(buffer, "") { _ in .ignorable } } + if startsWith(buffer, " Scanned { + until(buffer, terminator: ">") { inner in + .close(name: String(inner.drop { $0 == "/" }).trimmingXMLWhitespace()) + } + } + + private static func scanProcessingInstruction(_ buffer: [Character]) -> Scanned { + delimited(buffer, open: 2, close: "?>") { inner in + let text = String(inner) + guard let space = text.firstIndex(where: \.isXMLWhitespace) else { + return .processingInstruction(target: text, data: "") + } + return .processingInstruction( + target: String(text[.. Scanned { + guard let end = tagEnd(buffer) else { return final ? .token(.text("<"), consumed: 1) : .needMore } + var inner = Array(buffer[1 ..< end]) + let selfClosing = inner.last == "/" + if selfClosing { inner.removeLast() } + let (name, attributes) = parseTag(inner) + return .token(.open(name: name, attributes: attributes, selfClosing: selfClosing), consumed: end + 1) + } + + // MARK: Terminator scanning + + private static func delimited( + _ buffer: [Character], + open: Int, + close: String, + _ make: (ArraySlice) -> PushToken, + ) -> Scanned { + let target = Array(close) + var index = open + while index + target.count <= buffer.count { + if Array(buffer[index ..< index + target.count]) == target { + return .token(make(buffer[open ..< index]), consumed: index + target.count) + } + index += 1 + } + return .needMore + } + + private static func until( + _ buffer: [Character], + terminator: Character, + _ make: (ArraySlice) -> PushToken, + ) -> Scanned { + guard let end = buffer.firstIndex(of: terminator) else { return .needMore } + return .token(make(buffer[2 ..< end]), consumed: end + 1) + } + + /// The index of the `>` that closes a start tag, skipping any inside quotes. + private static func tagEnd(_ buffer: [Character]) -> Int? { + var quote: Character? + for index in 1 ..< buffer.count { + let character = buffer[index] + if let active = quote { + if character == active { quote = nil } + } else if character == "\"" || character == "'" { + quote = character + } else if character == ">" { + return index + } + } + return nil + } + + // MARK: Tag and entity parsing + + private static func parseTag(_ inner: [Character]) -> (name: String, attributes: [PushAttribute]) { + var cursor = TagCursor(inner) + let name = cursor.readName() + var attributes: [PushAttribute] = [] + while let attribute = cursor.readAttribute() { + attributes.append(attribute) + } + return (name, attributes) + } + + private static func decode(_ slice: ArraySlice) -> String { + var budget = Int.max + return (try? EntityDecoder.decode(String(slice), entities: [:], budget: &budget, at: .start)) ?? String(slice) + } + + private static func startsWith(_ buffer: [Character], _ prefix: String) -> Bool { + let target = Array(prefix) + guard buffer.count >= target.count else { return false } + return Array(buffer[0 ..< target.count]) == target + } + + /// Whether the buffer is too short to decide which markup kind it is. + private static func isPrefix(_ buffer: [Character], ofAny _: [String], decidedAt: Int) -> Bool { + buffer.count < decidedAt + } + } +} diff --git a/Tests/PushParserTests.swift b/Tests/PushParserTests.swift new file mode 100644 index 0000000..fdde41a --- /dev/null +++ b/Tests/PushParserTests.swift @@ -0,0 +1,98 @@ +@testable import PureXML +import Testing + +@Suite("Push parser") +struct PushParserTests { + /// Feeds the chunks through a push parser and returns a flat list of event + /// descriptions. + private func collect(_ chunks: [String]) throws -> [String] { + var events: [String] = [] + let handler = PureXML.Parsing.SAXHandler( + startElement: { name, attributes in + let attrs = attributes.map { " \($0.name.description)=\($0.value)" }.joined() + events.append("start:\(name.description)\(attrs)") + }, + endElement: { events.append("end:\($0.description)") }, + characters: { events.append("text:\($0)") }, + cdata: { events.append("cdata:\($0)") }, + comment: { events.append("comment:\($0)") }, + processingInstruction: { events.append("pi:\($0):\($1)") }, + ) + var parser = PureXML.Parsing.PushParser(sax: handler) + for chunk in chunks { + try parser.feed(chunk) + } + try parser.finish() + return events + } + + private let document = "" + + "hi & byetail" + + @Test("A whole document produces the expected events") + func test_whole() throws { + #expect(try collect([document]) == [ + "start:root a=1 xmlns:n=urn:x", + "start:n:b", + "text:hi & bye", + "end:n:b", + "comment: c ", + "cdata:x", + "", + "", + "", + "a < b", + ] + for fragment in cases { + let whole = try collect([fragment]) + let mid = fragment.index(fragment.startIndex, offsetBy: fragment.count / 2) + let parts = [String(fragment[..", "a", "b", ""] + let stream = AsyncStream { continuation in + for chunk in chunks { + continuation.yield(chunk) + } + continuation.finish() + } + var names: [String] = [] + for try await event in PureXML.events(feeding: stream) { + if case let .startElement(name, _) = event { names.append(name.description) } + } + #expect(names == ["doc", "item", "item"]) + } +}