From e86c87af3cb5cfd1efca2da215ad09a9ca694148 Mon Sep 17 00:00:00 2001
From: Mihaela Mihaljevic <mihaelamj@me.com>
Date: Tue, 9 Jun 2026 05:11:18 +0200
Subject: [PATCH] feat(html): numeric-ref validation, NUL, RCDATA decoding,
 semicolon-less refs (#84)

---
 CHANGELOG.md                     |   1 +
 README.md                        |   2 +-
 Sources/HTML/HTMLEntities.swift  | 107 ++++++++++++++++++++++++-------
 Sources/HTML/HTMLToken.swift     |   5 ++
 Sources/HTML/HTMLTokenizer.swift |   7 +-
 Tests/HTMLTokenizerTests.swift   |  53 +++++++++++++++
 6 files changed, 150 insertions(+), 25 deletions(-)
 create mode 100644 Tests/HTMLTokenizerTests.swift

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f345098..8ccb748 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- HTML tokenizer correctness (#84, partial): numeric character references are now range/surrogate-validated (zero, surrogates, and values above U+10FFFF yield U+FFFD) with the Windows-1252 C1 fixup table (`&#x80;` -> euro, etc.); named references are decoded by longest match with or without a trailing semicolon; a literal NUL byte becomes U+FFFD; RCDATA elements (`title`, `textarea`) decode character references while raw-text elements (`script`, `style`) take their content verbatim; and a single leading newline after `<textarea>` is stripped. (The full ~2,000-entry named-entity table remains deferred to vendoring from the authoritative WHATWG data, the same policy as the legacy encodings.)
 - HTML conformance validator in the composable `Validation` idiom. `PureXML.HTML.validationErrors(in:)` (backed by `PureXML.Validation.HTML`) judges a parsed tree against the intrinsic HTML5 content-model invariants as named, removable, isolation-testable `Validation` rules, the same idiom as the DTD/XSD/structural validators: `voidElementsAreEmpty` (void elements carry no content), `requiredParent` (`li`/`td`/`option`/... appear inside an allowed parent, with the allowed parents named in the finding), and `uniqueIdentifiers` (`id` values are unique). Failures are located `ValidationError`s; this is the validation layer HTML previously lacked, so every parsing/schema subsystem now exposes its conformance checks as composable rules.
 - XInclude range inclusion (#87), completing the XInclude audit. An `xi:include` whose `xpointer` selects a range (`xpointer(range(...))`, `range-to`, or `string-range`) now includes that range's content: when no node-selecting scheme matches, the processor falls back to the XPointer range model and substitutes each range's spanned nodes (or the matched text for a `string-range`).
 - XPointer `range()` model (#88), completing the XPath/XPointer audit. `PureXML.XPointer.evaluateRanges(_:over:)` (and `Pointer.evaluateRanges(over:)`) evaluate the range forms inside an `xpointer()`/`xpath1()` scheme: `range(expr)` (the covering range of each location), `start/range-to(end)` (one range spanning from the first location of `start` to the first of `end`, the sibling run when they share a parent), and `string-range(location, "search"[, offset[, length]])` (a character range per occurrence, with the XPointer offset/length adjustment). Each `Range` materializes to the whole `nodes` it spans and the `text` it covers, and `xmlns()` bindings apply to the range parts that follow, so this also unblocks XInclude range inclusion (#87).
diff --git a/README.md b/README.md
index 5ebc4d4..1818c7b 100644
--- a/README.md
+++ b/README.md
@@ -89,7 +89,7 @@ flowchart TB
 
 PureXML is a working, dependency-free XML library today: parse, emit, validate,
 query, and stream documents on macOS, Linux, Windows, and WASM. The test suite
-currently runs **710 tests in 112 suites** (`swift test`).
+currently runs **717 tests in 113 suites** (`swift test`).
 
 ### Shipped (libxml2-aligned surface)
 
diff --git a/Sources/HTML/HTMLEntities.swift b/Sources/HTML/HTMLEntities.swift
index 49ac622..8f933f5 100644
--- a/Sources/HTML/HTMLEntities.swift
+++ b/Sources/HTML/HTMLEntities.swift
@@ -6,23 +6,45 @@ extension PureXML.HTML.Tokenizer {
         "laquo": "\u{AB}", "raquo": "\u{BB}", "deg": "\u{B0}", "euro": "\u{20AC}",
     ]
 
-    /// Decodes the common HTML character references in `value`, leaving any
-    /// unrecognized `&...;` sequence as written (the lenient HTML behavior).
+    /// The HTML numeric-character-reference fixups for the Windows-1252 C1 range
+    /// (`0x80`–`0x9F`): a reference to one of these code points yields the mapped
+    /// character rather than the C1 control, per the HTML standard.
+    private static let c1Replacements: [UInt32: UInt32] = [
+        0x80: 0x20AC, 0x82: 0x201A, 0x83: 0x0192, 0x84: 0x201E, 0x85: 0x2026,
+        0x86: 0x2020, 0x87: 0x2021, 0x88: 0x02C6, 0x89: 0x2030, 0x8A: 0x0160,
+        0x8B: 0x2039, 0x8C: 0x0152, 0x8E: 0x017D, 0x91: 0x2018, 0x92: 0x2019,
+        0x93: 0x201C, 0x94: 0x201D, 0x95: 0x2022, 0x96: 0x2013, 0x97: 0x2014,
+        0x98: 0x02DC, 0x99: 0x2122, 0x9A: 0x0161, 0x9B: 0x203A, 0x9C: 0x0153,
+        0x9E: 0x017E, 0x9F: 0x0178,
+    ]
+
+    private static let replacementCharacter: Character = "\u{FFFD}"
+
+    /// Replaces NUL with U+FFFD, the HTML rule for the literal null byte in
+    /// character data (it never reaches the document as U+0000).
+    static func replacingNull(_ value: String) -> String {
+        guard value.contains("\u{0}") else { return value }
+        return String(value.map { $0 == "\u{0}" ? replacementCharacter : $0 })
+    }
+
+    /// Decodes the HTML character references in `value`: named references (with or
+    /// without a trailing semicolon, longest match wins) and numeric references
+    /// (decimal `&#...;` or hex `&#x...;`) with range, surrogate, and C1 fixups.
+    /// An unrecognized `&...` sequence is left as written (the lenient behavior).
     static func decodeEntities(_ value: String) -> String {
         guard value.contains("&") else { return value }
-        var result = ""
         let characters = Array(value)
+        var result = ""
         var index = 0
         while index < characters.count {
-            guard characters[index] == "&", let semicolon = nextSemicolon(characters, after: index) else {
+            guard characters[index] == "&" else {
                 result.append(characters[index])
                 index += 1
                 continue
             }
-            let body = String(characters[(index + 1) ..< semicolon])
-            if let decoded = decode(body) {
-                result.append(decoded)
-                index = semicolon + 1
+            if let (text, consumed) = decodeReference(characters, at: index) {
+                result += text
+                index += consumed
             } else {
                 result.append("&")
                 index += 1
@@ -31,24 +53,63 @@ extension PureXML.HTML.Tokenizer {
         return result
     }
 
-    private static func nextSemicolon(_ characters: [Character], after start: Int) -> Int? {
-        var index = start + 1
-        while index < characters.count, index - start <= 10 {
-            if characters[index] == ";" { return index }
-            index += 1
+    /// Decodes a single reference starting at the `&` in `characters[start]`,
+    /// returning its replacement text and how many characters it consumed, or nil
+    /// when there is no valid reference there.
+    private static func decodeReference(_ characters: [Character], at start: Int) -> (String, Int)? {
+        if start + 1 < characters.count, characters[start + 1] == "#" {
+            return decodeNumeric(characters, at: start)
+        }
+        return decodeNamed(characters, at: start)
+    }
+
+    private static func decodeNumeric(_ characters: [Character], at start: Int) -> (String, Int)? {
+        var cursor = start + 2 // past "&#"
+        let isHex = cursor < characters.count && (characters[cursor] == "x" || characters[cursor] == "X")
+        if isHex { cursor += 1 }
+        let digitsStart = cursor
+        while cursor < characters.count, isDigit(characters[cursor], hex: isHex) {
+            cursor += 1
+        }
+        guard cursor > digitsStart, let number = UInt32(String(characters[digitsStart ..< cursor]), radix: isHex ? 16 : 10) else { return nil }
+        if cursor < characters.count, characters[cursor] == ";" { cursor += 1 }
+        return (String(numericCharacter(number)), cursor - start)
+    }
+
+    /// The character a numeric reference resolves to: U+FFFD for zero, surrogates,
+    /// and out-of-range values; the C1 fixup for `0x80`–`0x9F`; otherwise the code
+    /// point itself.
+    private static func numericCharacter(_ number: UInt32) -> Character {
+        if let replacement = c1Replacements[number] {
+            return Unicode.Scalar(replacement).map(Character.init) ?? replacementCharacter
+        }
+        if number == 0 || number > 0x10FFFF || (0xD800 ... 0xDFFF).contains(number) {
+            return replacementCharacter
+        }
+        return Unicode.Scalar(number).map(Character.init) ?? replacementCharacter
+    }
+
+    private static func decodeNamed(_ characters: [Character], at start: Int) -> (String, Int)? {
+        var cursor = start + 1
+        var name = ""
+        // The longest name in the table is short; cap the scan accordingly.
+        while cursor < characters.count, characters[cursor].isLetter || characters[cursor].isNumber, name.count < 12 {
+            name.append(characters[cursor])
+            cursor += 1
+        }
+        // Longest match wins: shrink the candidate until it names an entity.
+        while !name.isEmpty {
+            if let character = named[name] {
+                let end = start + 1 + name.count
+                let semicolon = end < characters.count && characters[end] == ";"
+                return (String(character), name.count + 1 + (semicolon ? 1 : 0))
+            }
+            name.removeLast()
         }
         return nil
     }
 
-    private static func decode(_ body: String) -> Character? {
-        if let named = named[body] { return named }
-        guard body.hasPrefix("#") else { return nil }
-        let digits = body.dropFirst()
-        let scalar: UInt32? = if digits.hasPrefix("x") || digits.hasPrefix("X") {
-            UInt32(digits.dropFirst(), radix: 16)
-        } else {
-            UInt32(digits, radix: 10)
-        }
-        return scalar.flatMap(Unicode.Scalar.init).map(Character.init)
+    private static func isDigit(_ character: Character, hex: Bool) -> Bool {
+        hex ? character.isHexDigit : character.isNumber
     }
 }
diff --git a/Sources/HTML/HTMLToken.swift b/Sources/HTML/HTMLToken.swift
index 1cb50f7..d3964ae 100644
--- a/Sources/HTML/HTMLToken.swift
+++ b/Sources/HTML/HTMLToken.swift
@@ -35,6 +35,11 @@ extension PureXML.HTML {
         /// end tag, not parsed as markup.
         static let rawText: Set<String> = ["script", "style", "textarea", "title"]
 
+        /// RCDATA elements: like raw-text, but character references in their
+        /// content are decoded (`title`, `textarea`). The remaining raw-text
+        /// elements (`script`, `style`) take their content with no decoding.
+        static let rcdata: Set<String> = ["textarea", "title"]
+
         /// For an opening tag (key), the set of currently-open elements it
         /// implicitly closes first (the common optional-end-tag rules).
         static let impliedClose: [String: Set<String>] = [
diff --git a/Sources/HTML/HTMLTokenizer.swift b/Sources/HTML/HTMLTokenizer.swift
index 25d95d8..d3da723 100644
--- a/Sources/HTML/HTMLTokenizer.swift
+++ b/Sources/HTML/HTMLTokenizer.swift
@@ -43,7 +43,7 @@ extension PureXML.HTML {
                 value.append(character)
                 advance()
             }
-            return .text(Self.decodeEntities(value))
+            return .text(Self.decodeEntities(Self.replacingNull(value)))
         }
 
         private mutating func startTag() -> Token {
@@ -131,6 +131,11 @@ extension PureXML.HTML {
                 value.append(chars[index])
                 index += 1
             }
+            value = Self.replacingNull(value)
+            // RCDATA decodes character references; raw text (script/style) does not.
+            if Elements.rcdata.contains(name) { value = Self.decodeEntities(value) }
+            // A single leading newline after <textarea> is stripped (the HTML rule).
+            if name == "textarea", value.first == "\n" { value.removeFirst() }
             return .text(value)
         }
 
diff --git a/Tests/HTMLTokenizerTests.swift b/Tests/HTMLTokenizerTests.swift
new file mode 100644
index 0000000..4bdacaa
--- /dev/null
+++ b/Tests/HTMLTokenizerTests.swift
@@ -0,0 +1,53 @@
+@testable import PureXML
+import Testing
+
+@Suite("HTML tokenizer: references, NUL, RCDATA")
+struct HTMLTokenizerTests {
+    /// The concatenated text content of the first parsed element.
+    private func text(_ html: String) -> String {
+        guard case let .document(children) = PureXML.HTML.parse(html), case let .element(element) = children.first else { return "" }
+        return element.children.compactMap { if case let .text(value) = $0 { value } else { nil } }.joined()
+    }
+
+    @Test("A numeric reference to zero, a surrogate, or an out-of-range value yields U+FFFD")
+    func test_numericInvalid() {
+        #expect(text("<p>&#0;</p>") == "\u{FFFD}")
+        #expect(text("<p>&#xD800;</p>") == "\u{FFFD}")
+        #expect(text("<p>&#x110000;</p>") == "\u{FFFD}")
+    }
+
+    @Test("A numeric reference in the C1 range is mapped to its Windows-1252 character")
+    func test_numericC1() {
+        #expect(text("<p>&#x80;</p>") == "\u{20AC}") // euro
+        #expect(text("<p>&#153;</p>") == "\u{2122}") // trade mark
+    }
+
+    @Test("A valid numeric reference resolves to its code point")
+    func test_numericValid() {
+        #expect(text("<p>&#65;&#x42;</p>") == "AB")
+    }
+
+    @Test("A literal NUL byte becomes U+FFFD")
+    func test_nullReplacement() {
+        #expect(text("<p>a\u{0}b</p>") == "a\u{FFFD}b")
+    }
+
+    @Test("A named reference without a trailing semicolon is still decoded")
+    func test_semicolonless() {
+        #expect(text("<p>&amp</p>") == "&")
+        #expect(text("<p>&copy</p>") == "\u{A9}")
+    }
+
+    @Test("RCDATA content (title, textarea) decodes references; raw text (style) does not")
+    func test_rcdataDecoding() {
+        #expect(text("<title>a &amp; b</title>") == "a & b")
+        #expect(text("<textarea>&lt;x&gt;</textarea>") == "<x>")
+        #expect(text("<style>a &amp; b</style>") == "a &amp; b")
+    }
+
+    @Test("A single leading newline after <textarea> is stripped")
+    func test_textareaLeadingNewline() {
+        #expect(text("<textarea>\nhello</textarea>") == "hello")
+        #expect(text("<textarea>\n\nhello</textarea>") == "\nhello")
+    }
+}