feat(html): lenient HTML parser and serializer

Add the PureXML.HTML namespace: a lenient tokenizer for tag-soup input
(case-insensitive tags; quoted, unquoted, and boolean attributes; comments;
doctype; character references), a tree builder handling void elements,
optional end tags (li, p, table rows and cells), raw-text elements (script,
style), and unmatched end tags, and a serializer that writes void elements
unclosed and raw-text content unescaped.

Closes #36
Closes #37
Closes #38
Closes #39
Closes #20

CHANGELOG.md

@@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- HTML parser and serializer (#20, the libxml2 `HTMLparser.h`/`HTMLtree.h`
+  model). `PureXML.HTML.parse` reads tag-soup HTML leniently (case-insensitive
+  tags; quoted, unquoted, and boolean attributes; comments; doctype; character
+  references) and builds a node tree handling void elements (no end tag),
+  optional end tags (implicit close of `li`, `p`, table rows and cells, and the
+  like), raw-text elements (`script`, `style`), and unmatched end tags.
+  `PureXML.HTML.serialize` writes it back with void elements unclosed and
+  raw-text content unescaped.
 - XSLT 1.0 transformation engine, completing #3. `PureXML.XSLT.transform`
   applies a stylesheet to a source document by the XSLT processing model: it
   matches each node to the highest-priority template (or the built-in rules) and

README.md

@@ -61,7 +61,7 @@ flowchart TB
   Core["Streaming core + DTD validation (shipped)"]:::done
   Push["#1 Push / feed streaming API"]:::epic
   Sax["#19 SAX2 push callbacks"]:::done
-  Html["#20 HTML parser and serializer"]:::epic
+  Html["#20 HTML parser and serializer"]:::done
   Enc["#28 Full encoding support"]:::done
   Ent["#29 External and parameter entities"]:::done
   TreeApi["#31 Tree manipulation API"]:::done

Sources/HTML/HTMLEntities.swift

@@ -0,0 +1,54 @@
+extension PureXML.HTML.Tokenizer {
+    private static let named: [String: Character] = [
+        "amp": "&", "lt": "<", "gt": ">", "quot": "\"", "apos": "'",
+        "nbsp": "\u{A0}", "copy": "\u{A9}", "reg": "\u{AE}", "trade": "\u{2122}",
+        "mdash": "\u{2014}", "ndash": "\u{2013}", "hellip": "\u{2026}",
+        "laquo": "\u{AB}", "raquo": "\u{BB}", "deg": "\u{B0}", "euro": "\u{20AC}",
+    ]
+
+    /// Decodes the common HTML character references in `value`, leaving any
+    /// unrecognized `&...;` sequence as written (the lenient HTML behavior).
+    static func decodeEntities(_ value: String) -> String {
+        guard value.contains("&") else { return value }
+        var result = ""
+        var characters = Array(value)
+        var index = 0
+        while index < characters.count {
+            guard characters[index] == "&", let semicolon = nextSemicolon(characters, after: index) else {
+                result.append(characters[index])
+                index += 1
+                continue
+            }
+            let body = String(characters[(index + 1) ..< semicolon])
+            if let decoded = decode(body) {
+                result.append(decoded)
+                index = semicolon + 1
+            } else {
+                result.append("&")
+                index += 1
+            }
+        }
+        return result
+    }
+
+    private static func nextSemicolon(_ characters: [Character], after start: Int) -> Int? {
+        var index = start + 1
+        while index < characters.count, index - start <= 10 {
+            if characters[index] == ";" { return index }
+            index += 1
+        }
+        return nil
+    }
+
+    private static func decode(_ body: String) -> Character? {
+        if let named = named[body] { return named }
+        guard body.hasPrefix("#") else { return nil }
+        let digits = body.dropFirst()
+        let scalar: UInt32? = if digits.hasPrefix("x") || digits.hasPrefix("X") {
+            UInt32(digits.dropFirst(), radix: 16)
+        } else {
+            UInt32(digits, radix: 10)
+        }
+        return scalar.flatMap(Unicode.Scalar.init).map(Character.init)
+    }
+}

Sources/HTML/HTMLSerializer.swift

@@ -0,0 +1,95 @@
+extension PureXML.HTML {
+    /// Serializes a node tree as HTML (the libxml2 `HTMLtree.h` model): void
+    /// elements are written without an end tag or self-closing slash, raw-text
+    /// elements (`script`, `style`) keep their content unescaped, and text and
+    /// attribute values are escaped per HTML.
+    enum Serializer {
+        static func serialize(_ node: PureXML.Model.Node) -> String {
+            switch node {
+            case let .document(children):
+                children.map(serialize).joined()
+            case let .element(element):
+                serializeElement(element)
+            case let .text(value), let .cdata(value):
+                escapeText(value)
+            case let .comment(value):
+                "<!--\(value)-->"
+            case let .processingInstruction(target, data):
+                data.isEmpty ? "<?\(target)>" : "<?\(target) \(data)>"
+            }
+        }
+
+        private static func serializeElement(_ element: PureXML.Model.Element) -> String {
+            let name = element.name.description
+            var output = "<" + name
+            for attribute in element.attributes {
+                output += attributeText(attribute)
+            }
+            output += ">"
+            if Elements.void.contains(name) {
+                return output
+            }
+            output += content(of: element, name: name)
+            return output + "</\(name)>"
+        }
+
+        private static func content(of element: PureXML.Model.Element, name: String) -> String {
+            if Elements.rawText.contains(name) {
+                return element.children.map(rawText).joined()
+            }
+            return element.children.map(serialize).joined()
+        }
+
+        private static func rawText(_ node: PureXML.Model.Node) -> String {
+            switch node {
+            case let .text(value), let .cdata(value): value
+            default: serialize(node)
+            }
+        }
+
+        private static func attributeText(_ attribute: PureXML.Model.Attribute) -> String {
+            let name = attribute.name.description
+            return attribute.value.isEmpty ? " \(name)" : " \(name)=\"\(escapeAttribute(attribute.value))\""
+        }
+
+        private static func escapeText(_ value: String) -> String {
+            var result = ""
+            for character in value {
+                switch character {
+                case "&": result += "&amp;"
+                case "<": result += "&lt;"
+                case ">": result += "&gt;"
+                default: result.append(character)
+                }
+            }
+            return result
+        }
+
+        private static func escapeAttribute(_ value: String) -> String {
+            var result = ""
+            for character in value {
+                switch character {
+                case "&": result += "&amp;"
+                case "\"": result += "&quot;"
+                case "<": result += "&lt;"
+                default: result.append(character)
+                }
+            }
+            return result
+        }
+    }
+}
+
+public extension PureXML.HTML {
+    /// Parses HTML into a ``PureXML/Model/Node`` document, leniently handling
+    /// tag-soup input, void elements, optional end tags, and raw-text elements.
+    static func parse(_ html: String) -> PureXML.Model.Node {
+        var tokenizer = Tokenizer(html)
+        return TreeBuilder.build(tokenizer.tokenize())
+    }
+
+    /// Serializes a node tree as HTML.
+    static func serialize(_ node: PureXML.Model.Node) -> String {
+        Serializer.serialize(node)
+    }
+}

Sources/HTML/HTMLToken.swift

@@ -0,0 +1,54 @@
+extension PureXML.HTML {
+    /// A lexical token from the lenient HTML tokenizer.
+    enum Token: Equatable, Sendable {
+        case startTag(name: String, attributes: [(String, String)], selfClosing: Bool)
+        case endTag(name: String)
+        case text(String)
+        case comment(String)
+        case doctype(String)
+
+        static func == (lhs: Token, rhs: Token) -> Bool {
+            switch (lhs, rhs) {
+            case let (.startTag(leftName, leftAttributes, leftClosing), .startTag(rightName, rightAttributes, rightClosing)):
+                leftName == rightName && leftClosing == rightClosing
+                    && leftAttributes.map(\.0) == rightAttributes.map(\.0)
+                    && leftAttributes.map(\.1) == rightAttributes.map(\.1)
+            case let (.endTag(left), .endTag(right)): left == right
+            case let (.text(left), .text(right)): left == right
+            case let (.comment(left), .comment(right)): left == right
+            case let (.doctype(left), .doctype(right)): left == right
+            default: false
+            }
+        }
+    }
+
+    /// The HTML element categories that drive lenient tree building and
+    /// serialization.
+    enum Elements {
+        /// Void elements: no content and no end tag.
+        static let void: Set<String> = [
+            "area", "base", "br", "col", "embed", "hr", "img", "input",
+            "link", "meta", "param", "source", "track", "wbr",
+        ]
+
+        /// Raw-text elements: their content is taken verbatim up to the matching
+        /// end tag, not parsed as markup.
+        static let rawText: Set<String> = ["script", "style", "textarea", "title"]
+
+        /// For an opening tag (key), the set of currently-open elements it
+        /// implicitly closes first (the common optional-end-tag rules).
+        static let impliedClose: [String: Set<String>] = [
+            "li": ["li"],
+            "dt": ["dt", "dd"],
+            "dd": ["dt", "dd"],
+            "p": ["p"],
+            "option": ["option"],
+            "tr": ["tr", "td", "th"],
+            "td": ["td", "th"],
+            "th": ["td", "th"],
+            "thead": ["tbody", "thead", "tfoot"],
+            "tbody": ["tbody", "thead", "tfoot"],
+            "tfoot": ["tbody", "thead", "tfoot"],
+        ]
+    }
+}

Sources/HTML/HTMLTokenizer.swift

@@ -0,0 +1,215 @@
+extension PureXML.HTML {
+    /// A lenient HTML tokenizer for tag-soup input: case-insensitive tag names,
+    /// quoted/unquoted/boolean attributes, comments, doctype, and raw-text
+    /// elements (`script`, `style`, `textarea`, `title`) whose content is taken
+    /// verbatim. Malformed markup degrades to text rather than failing.
+    struct Tokenizer {
+        private let chars: [Character]
+        private var index = 0
+        private var rawTextName: String?
+
+        init(_ html: String) {
+            chars = Array(html)
+        }
+
+        mutating func tokenize() -> [Token] {
+            var tokens: [Token] = []
+            while let token = next() {
+                tokens.append(token)
+            }
+            return tokens
+        }
+
+        private mutating func next() -> Token? {
+            guard !isAtEnd else { return nil }
+            if let raw = rawTextName {
+                rawTextName = nil
+                return rawText(until: raw)
+            }
+            guard peek() == "<" else { return text() }
+            if matches("<!--") { return comment() }
+            if matches("<!") { return doctype() }
+            if matches("</") { return endTag() }
+            if peek(1)?.isLetter == true { return startTag() }
+            return text()
+        }
+
+        // MARK: Tokens
+
+        private mutating func text() -> Token {
+            var value = ""
+            if let first = advance() { value.append(first) }
+            while let character = peek(), character != "<" {
+                value.append(character)
+                advance()
+            }
+            return .text(Self.decodeEntities(value))
+        }
+
+        private mutating func startTag() -> Token {
+            advance()
+            let name = readName().lowercased()
+            var attributes: [(String, String)] = []
+            var selfClosing = false
+            while !isAtEnd {
+                skipSpace()
+                if consume("/>") { selfClosing = true
+                    break
+                }
+                if consume(">") { break }
+                guard let attribute = readAttribute() else { advance()
+                    continue
+                }
+                attributes.append(attribute)
+            }
+            if Elements.rawText.contains(name) { rawTextName = name }
+            return .startTag(name: name, attributes: attributes, selfClosing: selfClosing)
+        }
+
+        private mutating func readAttribute() -> (String, String)? {
+            let name = readAttributeName().lowercased()
+            guard !name.isEmpty else { return nil }
+            skipSpace()
+            guard consume("=") else { return (name, "") }
+            skipSpace()
+            return (name, readAttributeValue())
+        }
+
+        private mutating func readAttributeValue() -> String {
+            guard let quote = peek(), quote == "\"" || quote == "'" else {
+                var value = ""
+                while let character = peek(), !character.isWhitespace, character != ">" {
+                    value.append(character)
+                    advance()
+                }
+                return Self.decodeEntities(value)
+            }
+            advance()
+            var value = ""
+            while let character = peek(), character != quote {
+                value.append(character)
+                advance()
+            }
+            consume(String(quote))
+            return Self.decodeEntities(value)
+        }
+
+        private mutating func endTag() -> Token {
+            advance()
+            advance()
+            let name = readName().lowercased()
+            skipUntil(">")
+            return .endTag(name: name)
+        }
+
+        private mutating func comment() -> Token {
+            index += 4
+            var value = ""
+            while !isAtEnd, !matches("-->") {
+                value.append(chars[index])
+                index += 1
+            }
+            _ = consume("-->")
+            return .comment(value)
+        }
+
+        private mutating func doctype() -> Token {
+            index += 2
+            var value = ""
+            while let character = peek(), character != ">" {
+                value.append(character)
+                advance()
+            }
+            consume(">")
+            return .doctype(value)
+        }
+
+        private mutating func rawText(until name: String) -> Token {
+            let target = Array("</" + name)
+            var value = ""
+            while !isAtEnd, !matchesCaseInsensitive(target) {
+                value.append(chars[index])
+                index += 1
+            }
+            return .text(value)
+        }
+
+        // MARK: Cursor
+
+        private mutating func readName() -> String {
+            var name = ""
+            while let character = peek(), character.isLetter || character.isNumber || character == "-" {
+                name.append(character)
+                advance()
+            }
+            return name
+        }
+
+        private mutating func readAttributeName() -> String {
+            var name = ""
+            while let character = peek(), Self.isAttributeNameCharacter(character) {
+                name.append(character)
+                advance()
+            }
+            return name
+        }
+
+        private static func isAttributeNameCharacter(_ character: Character) -> Bool {
+            !character.isWhitespace && character != "=" && character != ">" && character != "/"
+        }
+
+        private var isAtEnd: Bool {
+            index >= chars.count
+        }
+
+        private func peek(_ ahead: Int = 0) -> Character? {
+            let target = index + ahead
+            return target < chars.count ? chars[target] : nil
+        }
+
+        private func matches(_ literal: String) -> Bool {
+            let target = Array(literal)
+            guard index + target.count <= chars.count else { return false }
+            return Array(chars[index ..< index + target.count]) == target
+        }
+
+        private func matchesCaseInsensitive(_ target: [Character]) -> Bool {
+            guard index + target.count <= chars.count else { return false }
+            for (offset, character) in target.enumerated() where !Self.sameLetter(character, chars[index + offset]) {
+                return false
+            }
+            return true
+        }
+
+        private static func sameLetter(_ lhs: Character, _ rhs: Character) -> Bool {
+            lhs.lowercased() == rhs.lowercased()
+        }
+
+        @discardableResult
+        private mutating func advance() -> Character? {
+            guard !isAtEnd else { return nil }
+            defer { index += 1 }
+            return chars[index]
+        }
+
+        @discardableResult
+        private mutating func consume(_ literal: String) -> Bool {
+            guard matches(literal) else { return false }
+            index += literal.count
+            return true
+        }
+
+        private mutating func skipSpace() {
+            while let character = peek(), character.isWhitespace {
+                advance()
+            }
+        }
+
+        private mutating func skipUntil(_ character: Character) {
+            while let current = peek(), current != character {
+                advance()
+            }
+            advance()
+        }
+    }
+}

Sources/HTML/HTMLTreeBuilder.swift

@@ -0,0 +1,86 @@
+/// A partially-built element on the open-element stack. File-scope and private.
+private struct HTMLFrame {
+    let name: String
+    let attributes: [PureXML.Model.Attribute]
+    var children: [PureXML.Model.Node] = []
+}
+
+extension PureXML.HTML {
+    /// Builds a lenient tree from HTML tokens (the libxml2 `HTMLtree.h` model):
+    /// void elements get no children and no end tag, optional-end-tag elements are
+    /// closed implicitly when a conflicting tag opens, and an unmatched end tag is
+    /// ignored. The result is a ``PureXML/Model/Node`` document.
+    enum TreeBuilder {
+        static func build(_ tokens: [Token]) -> PureXML.Model.Node {
+            var stack: [HTMLFrame] = []
+            var roots: [PureXML.Model.Node] = []
+            for token in tokens {
+                apply(token, stack: &stack, roots: &roots)
+            }
+            while !stack.isEmpty {
+                pop(&stack, &roots)
+            }
+            return .document(roots)
+        }
+
+        private static func apply(_ token: Token, stack: inout [HTMLFrame], roots: inout [PureXML.Model.Node]) {
+            switch token {
+            case let .startTag(name, attributes, selfClosing):
+                openElement(name, attributes, selfClosing: selfClosing, &stack, &roots)
+            case let .endTag(name):
+                closeElement(name, &stack, &roots)
+            case let .text(value):
+                attach(.text(value), &stack, &roots)
+            case let .comment(value):
+                attach(.comment(value), &stack, &roots)
+            case .doctype:
+                break
+            }
+        }
+
+        private static func openElement(
+            _ name: String,
+            _ attributes: [(String, String)],
+            selfClosing: Bool,
+            _ stack: inout [HTMLFrame],
+            _ roots: inout [PureXML.Model.Node],
+        ) {
+            closeImplied(by: name, &stack, &roots)
+            let modelAttributes = attributes.map { PureXML.Model.Attribute($0.0, $0.1) }
+            if Elements.void.contains(name) || selfClosing {
+                let element = PureXML.Model.Element(name: .init(name), attributes: modelAttributes)
+                attach(.element(element), &stack, &roots)
+            } else {
+                stack.append(HTMLFrame(name: name, attributes: modelAttributes))
+            }
+        }
+
+        private static func closeElement(_ name: String, _ stack: inout [HTMLFrame], _ roots: inout [PureXML.Model.Node]) {
+            guard let index = stack.lastIndex(where: { $0.name == name }) else { return }
+            while stack.count > index {
+                pop(&stack, &roots)
+            }
+        }
+
+        private static func closeImplied(by name: String, _ stack: inout [HTMLFrame], _ roots: inout [PureXML.Model.Node]) {
+            guard let closes = Elements.impliedClose[name] else { return }
+            while let top = stack.last, closes.contains(top.name) {
+                pop(&stack, &roots)
+            }
+        }
+
+        private static func pop(_ stack: inout [HTMLFrame], _ roots: inout [PureXML.Model.Node]) {
+            guard let frame = stack.popLast() else { return }
+            let element = PureXML.Model.Element(name: .init(frame.name), attributes: frame.attributes, children: frame.children)
+            attach(.element(element), &stack, &roots)
+        }
+
+        private static func attach(_ node: PureXML.Model.Node, _ stack: inout [HTMLFrame], _ roots: inout [PureXML.Model.Node]) {
+            if stack.isEmpty {
+                roots.append(node)
+            } else {
+                stack[stack.count - 1].children.append(node)
+            }
+        }
+    }
+}

Sources/PureXML.swift

@@ -48,4 +48,7 @@ public enum PureXML {
 
     /// XSLT 1.0 transformation types.
     public enum XSLT {}
+
+    /// Lenient HTML parsing and serialization types.
+    public enum HTML {}
 }

Tests/HTMLTests.swift

@@ -0,0 +1,69 @@
+@testable import PureXML
+import Testing
+
+@Suite("HTML")
+struct HTMLTests {
+    private func roundTrip(_ html: String) -> String {
+        PureXML.HTML.serialize(PureXML.HTML.parse(html))
+    }
+
+    @Test("Void elements need no end tag and serialize without a slash")
+    func test_voidElements() {
+        #expect(roundTrip("<p>a<br>b<img src=\"x\">c</p>") == "<p>a<br>b<img src=\"x\">c</p>")
+    }
+
+    @Test("Optional end tags close implicitly")
+    func test_optionalEndTags() {
+        #expect(roundTrip("<ul><li>one<li>two</ul>") == "<ul><li>one</li><li>two</li></ul>")
+        #expect(roundTrip("<p>first<p>second") == "<p>first</p><p>second</p>")
+    }
+
+    @Test("Table rows and cells close implicitly")
+    func test_tableImpliedClose() {
+        let html = "<table><tr><td>a<td>b<tr><td>c</table>"
+        #expect(roundTrip(html) == "<table><tr><td>a</td><td>b</td></tr><tr><td>c</td></tr></table>")
+    }
+
+    @Test("Raw-text elements keep their content unescaped")
+    func test_rawText() {
+        let html = "<script>if (a < b && c > d) {}</script>"
+        let parsed = PureXML.HTML.parse(html)
+        #expect(PureXML.HTML.serialize(parsed) == html)
+    }
+
+    @Test("Attributes parse quoted, unquoted, and boolean forms")
+    func test_attributes() {
+        let parsed = PureXML.HTML.parse("<input type=text value='hi' disabled>")
+        guard case let .document(children) = parsed, case let .element(element) = children.first else {
+            Issue.record("expected an element")
+            return
+        }
+        #expect(element.attributes.count == 3)
+        #expect(element.attributes.contains { $0.name.description == "type" && $0.value == "text" })
+        #expect(element.attributes.contains { $0.name.description == "disabled" && $0.value == "" })
+    }
+
+    @Test("Tag names are lowercased")
+    func test_caseInsensitive() {
+        #expect(roundTrip("<DIV><SPAN>x</SPAN></DIV>") == "<div><span>x</span></div>")
+    }
+
+    @Test("Character references are decoded and re-escaped")
+    func test_entities() {
+        #expect(roundTrip("<p>a &amp; b &lt; c &#65;</p>") == "<p>a &amp; b &lt; c A</p>")
+        #expect(roundTrip("<p>&nbsp;&copy;</p>") == "<p>\u{A0}\u{A9}</p>")
+    }
+
+    @Test("Comments and an unmatched end tag are handled leniently")
+    func test_lenient() {
+        #expect(roundTrip("<div><!-- note -->text</div>") == "<div><!-- note -->text</div>")
+        #expect(roundTrip("<b>bold</i></b>") == "<b>bold</b>")
+    }
+
+    @Test("A representative document round-trips")
+    func test_document() {
+        let html = "<html><head><title>T</title></head>"
+            + "<body><h1>Hi</h1><p>Para with <a href=\"/x\">link</a>.</p></body></html>"
+        #expect(roundTrip(html) == html)
+    }
+}