From cfba3ed7d2b745ccb847618b3161118f5734a112 Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Mon, 8 Jun 2026 07:12:42 +0200 Subject: [PATCH] feat(html): lenient HTML parser and serializer Add the PureXML.HTML namespace: a lenient tokenizer for tag-soup input (case-insensitive tags; quoted, unquoted, and boolean attributes; comments; doctype; character references), a tree builder handling void elements, optional end tags (li, p, table rows and cells), raw-text elements (script, style), and unmatched end tags, and a serializer that writes void elements unclosed and raw-text content unescaped. Closes #36 Closes #37 Closes #38 Closes #39 Closes #20 --- CHANGELOG.md | 8 ++ README.md | 2 +- Sources/HTML/HTMLEntities.swift | 54 ++++++++ Sources/HTML/HTMLSerializer.swift | 95 +++++++++++++ Sources/HTML/HTMLToken.swift | 54 ++++++++ Sources/HTML/HTMLTokenizer.swift | 215 +++++++++++++++++++++++++++++ Sources/HTML/HTMLTreeBuilder.swift | 86 ++++++++++++ Sources/PureXML.swift | 3 + Tests/HTMLTests.swift | 69 +++++++++ 9 files changed, 585 insertions(+), 1 deletion(-) create mode 100644 Sources/HTML/HTMLEntities.swift create mode 100644 Sources/HTML/HTMLSerializer.swift create mode 100644 Sources/HTML/HTMLToken.swift create mode 100644 Sources/HTML/HTMLTokenizer.swift create mode 100644 Sources/HTML/HTMLTreeBuilder.swift create mode 100644 Tests/HTMLTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c6edc2..3eb3e6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- HTML parser and serializer (#20, the libxml2 `HTMLparser.h`/`HTMLtree.h` + model). `PureXML.HTML.parse` reads tag-soup HTML leniently (case-insensitive + tags; quoted, unquoted, and boolean attributes; comments; doctype; character + references) and builds a node tree handling void elements (no end tag), + optional end tags (implicit close of `li`, `p`, table rows and cells, and the + like), raw-text elements (`script`, `style`), and unmatched end tags. + `PureXML.HTML.serialize` writes it back with void elements unclosed and + raw-text content unescaped. - XSLT 1.0 transformation engine, completing #3. `PureXML.XSLT.transform` applies a stylesheet to a source document by the XSLT processing model: it matches each node to the highest-priority template (or the built-in rules) and diff --git a/README.md b/README.md index d368c3c..ea2b2ef 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ flowchart TB Core["Streaming core + DTD validation (shipped)"]:::done Push["#1 Push / feed streaming API"]:::epic Sax["#19 SAX2 push callbacks"]:::done - Html["#20 HTML parser and serializer"]:::epic + Html["#20 HTML parser and serializer"]:::done Enc["#28 Full encoding support"]:::done Ent["#29 External and parameter entities"]:::done TreeApi["#31 Tree manipulation API"]:::done diff --git a/Sources/HTML/HTMLEntities.swift b/Sources/HTML/HTMLEntities.swift new file mode 100644 index 0000000..49d42c5 --- /dev/null +++ b/Sources/HTML/HTMLEntities.swift @@ -0,0 +1,54 @@ +extension PureXML.HTML.Tokenizer { + private static let named: [String: Character] = [ + "amp": "&", "lt": "<", "gt": ">", "quot": "\"", "apos": "'", + "nbsp": "\u{A0}", "copy": "\u{A9}", "reg": "\u{AE}", "trade": "\u{2122}", + "mdash": "\u{2014}", "ndash": "\u{2013}", "hellip": "\u{2026}", + "laquo": "\u{AB}", "raquo": "\u{BB}", "deg": "\u{B0}", "euro": "\u{20AC}", + ] + + /// Decodes the common HTML character references in `value`, leaving any + /// unrecognized `&...;` sequence as written (the lenient HTML behavior). + static func decodeEntities(_ value: String) -> String { + guard value.contains("&") else { return value } + var result = "" + var characters = Array(value) + var index = 0 + while index < characters.count { + guard characters[index] == "&", let semicolon = nextSemicolon(characters, after: index) else { + result.append(characters[index]) + index += 1 + continue + } + let body = String(characters[(index + 1) ..< semicolon]) + if let decoded = decode(body) { + result.append(decoded) + index = semicolon + 1 + } else { + result.append("&") + index += 1 + } + } + return result + } + + private static func nextSemicolon(_ characters: [Character], after start: Int) -> Int? { + var index = start + 1 + while index < characters.count, index - start <= 10 { + if characters[index] == ";" { return index } + index += 1 + } + return nil + } + + private static func decode(_ body: String) -> Character? { + if let named = named[body] { return named } + guard body.hasPrefix("#") else { return nil } + let digits = body.dropFirst() + let scalar: UInt32? = if digits.hasPrefix("x") || digits.hasPrefix("X") { + UInt32(digits.dropFirst(), radix: 16) + } else { + UInt32(digits, radix: 10) + } + return scalar.flatMap(Unicode.Scalar.init).map(Character.init) + } +} diff --git a/Sources/HTML/HTMLSerializer.swift b/Sources/HTML/HTMLSerializer.swift new file mode 100644 index 0000000..630ed94 --- /dev/null +++ b/Sources/HTML/HTMLSerializer.swift @@ -0,0 +1,95 @@ +extension PureXML.HTML { + /// Serializes a node tree as HTML (the libxml2 `HTMLtree.h` model): void + /// elements are written without an end tag or self-closing slash, raw-text + /// elements (`script`, `style`) keep their content unescaped, and text and + /// attribute values are escaped per HTML. + enum Serializer { + static func serialize(_ node: PureXML.Model.Node) -> String { + switch node { + case let .document(children): + children.map(serialize).joined() + case let .element(element): + serializeElement(element) + case let .text(value), let .cdata(value): + escapeText(value) + case let .comment(value): + "" + case let .processingInstruction(target, data): + data.isEmpty ? "" : "" + } + } + + private static func serializeElement(_ element: PureXML.Model.Element) -> String { + let name = element.name.description + var output = "<" + name + for attribute in element.attributes { + output += attributeText(attribute) + } + output += ">" + if Elements.void.contains(name) { + return output + } + output += content(of: element, name: name) + return output + "" + } + + private static func content(of element: PureXML.Model.Element, name: String) -> String { + if Elements.rawText.contains(name) { + return element.children.map(rawText).joined() + } + return element.children.map(serialize).joined() + } + + private static func rawText(_ node: PureXML.Model.Node) -> String { + switch node { + case let .text(value), let .cdata(value): value + default: serialize(node) + } + } + + private static func attributeText(_ attribute: PureXML.Model.Attribute) -> String { + let name = attribute.name.description + return attribute.value.isEmpty ? " \(name)" : " \(name)=\"\(escapeAttribute(attribute.value))\"" + } + + private static func escapeText(_ value: String) -> String { + var result = "" + for character in value { + switch character { + case "&": result += "&" + case "<": result += "<" + case ">": result += ">" + default: result.append(character) + } + } + return result + } + + private static func escapeAttribute(_ value: String) -> String { + var result = "" + for character in value { + switch character { + case "&": result += "&" + case "\"": result += """ + case "<": result += "<" + default: result.append(character) + } + } + return result + } + } +} + +public extension PureXML.HTML { + /// Parses HTML into a ``PureXML/Model/Node`` document, leniently handling + /// tag-soup input, void elements, optional end tags, and raw-text elements. + static func parse(_ html: String) -> PureXML.Model.Node { + var tokenizer = Tokenizer(html) + return TreeBuilder.build(tokenizer.tokenize()) + } + + /// Serializes a node tree as HTML. + static func serialize(_ node: PureXML.Model.Node) -> String { + Serializer.serialize(node) + } +} diff --git a/Sources/HTML/HTMLToken.swift b/Sources/HTML/HTMLToken.swift new file mode 100644 index 0000000..1cb50f7 --- /dev/null +++ b/Sources/HTML/HTMLToken.swift @@ -0,0 +1,54 @@ +extension PureXML.HTML { + /// A lexical token from the lenient HTML tokenizer. + enum Token: Equatable, Sendable { + case startTag(name: String, attributes: [(String, String)], selfClosing: Bool) + case endTag(name: String) + case text(String) + case comment(String) + case doctype(String) + + static func == (lhs: Token, rhs: Token) -> Bool { + switch (lhs, rhs) { + case let (.startTag(leftName, leftAttributes, leftClosing), .startTag(rightName, rightAttributes, rightClosing)): + leftName == rightName && leftClosing == rightClosing + && leftAttributes.map(\.0) == rightAttributes.map(\.0) + && leftAttributes.map(\.1) == rightAttributes.map(\.1) + case let (.endTag(left), .endTag(right)): left == right + case let (.text(left), .text(right)): left == right + case let (.comment(left), .comment(right)): left == right + case let (.doctype(left), .doctype(right)): left == right + default: false + } + } + } + + /// The HTML element categories that drive lenient tree building and + /// serialization. + enum Elements { + /// Void elements: no content and no end tag. + static let void: Set = [ + "area", "base", "br", "col", "embed", "hr", "img", "input", + "link", "meta", "param", "source", "track", "wbr", + ] + + /// Raw-text elements: their content is taken verbatim up to the matching + /// end tag, not parsed as markup. + static let rawText: Set = ["script", "style", "textarea", "title"] + + /// For an opening tag (key), the set of currently-open elements it + /// implicitly closes first (the common optional-end-tag rules). + static let impliedClose: [String: Set] = [ + "li": ["li"], + "dt": ["dt", "dd"], + "dd": ["dt", "dd"], + "p": ["p"], + "option": ["option"], + "tr": ["tr", "td", "th"], + "td": ["td", "th"], + "th": ["td", "th"], + "thead": ["tbody", "thead", "tfoot"], + "tbody": ["tbody", "thead", "tfoot"], + "tfoot": ["tbody", "thead", "tfoot"], + ] + } +} diff --git a/Sources/HTML/HTMLTokenizer.swift b/Sources/HTML/HTMLTokenizer.swift new file mode 100644 index 0000000..25d95d8 --- /dev/null +++ b/Sources/HTML/HTMLTokenizer.swift @@ -0,0 +1,215 @@ +extension PureXML.HTML { + /// A lenient HTML tokenizer for tag-soup input: case-insensitive tag names, + /// quoted/unquoted/boolean attributes, comments, doctype, and raw-text + /// elements (`script`, `style`, `textarea`, `title`) whose content is taken + /// verbatim. Malformed markup degrades to text rather than failing. + struct Tokenizer { + private let chars: [Character] + private var index = 0 + private var rawTextName: String? + + init(_ html: String) { + chars = Array(html) + } + + mutating func tokenize() -> [Token] { + var tokens: [Token] = [] + while let token = next() { + tokens.append(token) + } + return tokens + } + + private mutating func next() -> Token? { + guard !isAtEnd else { return nil } + if let raw = rawTextName { + rawTextName = nil + return rawText(until: raw) + } + guard peek() == "<" else { return text() } + if matches("") { + value.append(chars[index]) + index += 1 + } + _ = consume("-->") + return .comment(value) + } + + private mutating func doctype() -> Token { + index += 2 + var value = "" + while let character = peek(), character != ">" { + value.append(character) + advance() + } + consume(">") + return .doctype(value) + } + + private mutating func rawText(until name: String) -> Token { + let target = Array(" String { + var name = "" + while let character = peek(), character.isLetter || character.isNumber || character == "-" { + name.append(character) + advance() + } + return name + } + + private mutating func readAttributeName() -> String { + var name = "" + while let character = peek(), Self.isAttributeNameCharacter(character) { + name.append(character) + advance() + } + return name + } + + private static func isAttributeNameCharacter(_ character: Character) -> Bool { + !character.isWhitespace && character != "=" && character != ">" && character != "/" + } + + private var isAtEnd: Bool { + index >= chars.count + } + + private func peek(_ ahead: Int = 0) -> Character? { + let target = index + ahead + return target < chars.count ? chars[target] : nil + } + + private func matches(_ literal: String) -> Bool { + let target = Array(literal) + guard index + target.count <= chars.count else { return false } + return Array(chars[index ..< index + target.count]) == target + } + + private func matchesCaseInsensitive(_ target: [Character]) -> Bool { + guard index + target.count <= chars.count else { return false } + for (offset, character) in target.enumerated() where !Self.sameLetter(character, chars[index + offset]) { + return false + } + return true + } + + private static func sameLetter(_ lhs: Character, _ rhs: Character) -> Bool { + lhs.lowercased() == rhs.lowercased() + } + + @discardableResult + private mutating func advance() -> Character? { + guard !isAtEnd else { return nil } + defer { index += 1 } + return chars[index] + } + + @discardableResult + private mutating func consume(_ literal: String) -> Bool { + guard matches(literal) else { return false } + index += literal.count + return true + } + + private mutating func skipSpace() { + while let character = peek(), character.isWhitespace { + advance() + } + } + + private mutating func skipUntil(_ character: Character) { + while let current = peek(), current != character { + advance() + } + advance() + } + } +} diff --git a/Sources/HTML/HTMLTreeBuilder.swift b/Sources/HTML/HTMLTreeBuilder.swift new file mode 100644 index 0000000..e4de205 --- /dev/null +++ b/Sources/HTML/HTMLTreeBuilder.swift @@ -0,0 +1,86 @@ +/// A partially-built element on the open-element stack. File-scope and private. +private struct HTMLFrame { + let name: String + let attributes: [PureXML.Model.Attribute] + var children: [PureXML.Model.Node] = [] +} + +extension PureXML.HTML { + /// Builds a lenient tree from HTML tokens (the libxml2 `HTMLtree.h` model): + /// void elements get no children and no end tag, optional-end-tag elements are + /// closed implicitly when a conflicting tag opens, and an unmatched end tag is + /// ignored. The result is a ``PureXML/Model/Node`` document. + enum TreeBuilder { + static func build(_ tokens: [Token]) -> PureXML.Model.Node { + var stack: [HTMLFrame] = [] + var roots: [PureXML.Model.Node] = [] + for token in tokens { + apply(token, stack: &stack, roots: &roots) + } + while !stack.isEmpty { + pop(&stack, &roots) + } + return .document(roots) + } + + private static func apply(_ token: Token, stack: inout [HTMLFrame], roots: inout [PureXML.Model.Node]) { + switch token { + case let .startTag(name, attributes, selfClosing): + openElement(name, attributes, selfClosing: selfClosing, &stack, &roots) + case let .endTag(name): + closeElement(name, &stack, &roots) + case let .text(value): + attach(.text(value), &stack, &roots) + case let .comment(value): + attach(.comment(value), &stack, &roots) + case .doctype: + break + } + } + + private static func openElement( + _ name: String, + _ attributes: [(String, String)], + selfClosing: Bool, + _ stack: inout [HTMLFrame], + _ roots: inout [PureXML.Model.Node], + ) { + closeImplied(by: name, &stack, &roots) + let modelAttributes = attributes.map { PureXML.Model.Attribute($0.0, $0.1) } + if Elements.void.contains(name) || selfClosing { + let element = PureXML.Model.Element(name: .init(name), attributes: modelAttributes) + attach(.element(element), &stack, &roots) + } else { + stack.append(HTMLFrame(name: name, attributes: modelAttributes)) + } + } + + private static func closeElement(_ name: String, _ stack: inout [HTMLFrame], _ roots: inout [PureXML.Model.Node]) { + guard let index = stack.lastIndex(where: { $0.name == name }) else { return } + while stack.count > index { + pop(&stack, &roots) + } + } + + private static func closeImplied(by name: String, _ stack: inout [HTMLFrame], _ roots: inout [PureXML.Model.Node]) { + guard let closes = Elements.impliedClose[name] else { return } + while let top = stack.last, closes.contains(top.name) { + pop(&stack, &roots) + } + } + + private static func pop(_ stack: inout [HTMLFrame], _ roots: inout [PureXML.Model.Node]) { + guard let frame = stack.popLast() else { return } + let element = PureXML.Model.Element(name: .init(frame.name), attributes: frame.attributes, children: frame.children) + attach(.element(element), &stack, &roots) + } + + private static func attach(_ node: PureXML.Model.Node, _ stack: inout [HTMLFrame], _ roots: inout [PureXML.Model.Node]) { + if stack.isEmpty { + roots.append(node) + } else { + stack[stack.count - 1].children.append(node) + } + } + } +} diff --git a/Sources/PureXML.swift b/Sources/PureXML.swift index b762c8c..96f4aa8 100644 --- a/Sources/PureXML.swift +++ b/Sources/PureXML.swift @@ -48,4 +48,7 @@ public enum PureXML { /// XSLT 1.0 transformation types. public enum XSLT {} + + /// Lenient HTML parsing and serialization types. + public enum HTML {} } diff --git a/Tests/HTMLTests.swift b/Tests/HTMLTests.swift new file mode 100644 index 0000000..ae32d9a --- /dev/null +++ b/Tests/HTMLTests.swift @@ -0,0 +1,69 @@ +@testable import PureXML +import Testing + +@Suite("HTML") +struct HTMLTests { + private func roundTrip(_ html: String) -> String { + PureXML.HTML.serialize(PureXML.HTML.parse(html)) + } + + @Test("Void elements need no end tag and serialize without a slash") + func test_voidElements() { + #expect(roundTrip("

a
bc

") == "

a
bc

") + } + + @Test("Optional end tags close implicitly") + func test_optionalEndTags() { + #expect(roundTrip("
  • one
  • two
") == "
  • one
  • two
") + #expect(roundTrip("

first

second") == "

first

second

") + } + + @Test("Table rows and cells close implicitly") + func test_tableImpliedClose() { + let html = "
ab
c
" + #expect(roundTrip(html) == "
ab
c
") + } + + @Test("Raw-text elements keep their content unescaped") + func test_rawText() { + let html = "" + let parsed = PureXML.HTML.parse(html) + #expect(PureXML.HTML.serialize(parsed) == html) + } + + @Test("Attributes parse quoted, unquoted, and boolean forms") + func test_attributes() { + let parsed = PureXML.HTML.parse("") + guard case let .document(children) = parsed, case let .element(element) = children.first else { + Issue.record("expected an element") + return + } + #expect(element.attributes.count == 3) + #expect(element.attributes.contains { $0.name.description == "type" && $0.value == "text" }) + #expect(element.attributes.contains { $0.name.description == "disabled" && $0.value == "" }) + } + + @Test("Tag names are lowercased") + func test_caseInsensitive() { + #expect(roundTrip("
x
") == "
x
") + } + + @Test("Character references are decoded and re-escaped") + func test_entities() { + #expect(roundTrip("

a & b < c A

") == "

a & b < c A

") + #expect(roundTrip("

 ©

") == "

\u{A0}\u{A9}

") + } + + @Test("Comments and an unmatched end tag are handled leniently") + func test_lenient() { + #expect(roundTrip("
text
") == "
text
") + #expect(roundTrip("bold") == "bold") + } + + @Test("A representative document round-trips") + func test_document() { + let html = "T" + + "

Hi

Para with link.

" + #expect(roundTrip(html) == html) + } +}