From 2b2c1fb692836adf47b474fb2f4995fa763f4585 Mon Sep 17 00:00:00 2001
From: Mihaela Mihaljevic <mihaelamj@me.com>
Date: Tue, 9 Jun 2026 08:24:00 +0200
Subject: [PATCH] feat(parsing): full GB18030 four-byte sequences

Extend the GBK two-byte subset to full GB18030: a lead byte followed by
a digit opens a four-byte form whose pointer is mapped through the WHATWG
index-gb18030-ranges linear ranges, reaching the supplementary planes.
The 207 ranges are vendored verbatim. Wired into the whole-buffer and
streaming decoders and the encoding-name table (gb18030/gb-18030).

Closes #102
---
 CHANGELOG.md                           |  1 +
 README.md                              |  4 +-
 Sources/Parsing/ByteDecoder.swift      |  2 +
 Sources/Parsing/GB18030Decoder.swift   | 94 ++++++++++++++++++++++++++
 Sources/Parsing/GB18030Ranges.swift    | 52 ++++++++++++++
 Sources/Parsing/InputEncoding.swift    |  1 +
 Sources/Parsing/StreamingDecoder.swift | 25 +++++++
 Tests/EncodingTests.swift              | 12 ++++
 8 files changed, 189 insertions(+), 2 deletions(-)
 create mode 100644 Sources/Parsing/GB18030Decoder.swift
 create mode 100644 Sources/Parsing/GB18030Ranges.swift

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d644f57..00753e6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Full GB18030 (#102): the byte decoder gains the GB18030 four-byte sequences on top of the GBK two-byte subset, for both the whole-buffer and streaming decoders, selected by the `gb18030` declaration name. A lead in `0x81`-`0xFE` followed by a digit opens a four-byte form whose pointer is mapped through the WHATWG `index-gb18030-ranges` linear ranges, reaching the supplementary planes (the `189000` range maps to `U+10000` and beyond). The 207 ranges are vendored verbatim from the WHATWG `index-gb18030-ranges.txt`. GB18030 is China's mandatory standard, supported in libxml2 through iconv.
 - CJK multi-byte encodings (#99, in progress): the byte decoder gains a multi-byte path and the WHATWG **Shift-JIS** and **EUC-JP** algorithms, for both the whole-buffer and streaming decoders, selected by the XML declaration's `encoding` name. Shift-JIS covers ASCII, half-width katakana, and the JIS X 0208 plane; EUC-JP adds the `0x8E` half-width katakana and `0x8F` JIS X 0212 leads. EUC-KR (CP949/UHC) decodes Hangul and Hanja through the WHATWG `index-euc-kr`, and GBK (which also covers GB2312) decodes Chinese through the two-byte range of `index-gb18030` with `0x80` as the euro sign. The JIS X 0208/0212, CP949, and GBK indexes are all vendored verbatim from the corresponding WHATWG `index-*.txt` files. This completes the CJK multi-byte encodings.
 - HTML named character references: the complete WHATWG named character reference set (#84), all 2,125 semicolon-form references including the astral mathematical alphanumerics and the 93 two-codepoint references, vendored verbatim from the authoritative `entities.json` (replacing the previous 16-entry stopgap). Decoding stays longest-match with an optional trailing semicolon, on top of the existing numeric-reference validation.
 - HTML5 select content model (#83, stage 10): a `<select>` now follows the in-select rules. `<optgroup>` closes an open `<option>`, a nested `<select>` closes the open one rather than nesting, and an `<input>`/`<keygen>`/`<textarea>` closes the select and is placed after it. In the "in select in table" case, a table-structural tag (`caption`/`table`/`tbody`/`tfoot`/`thead`/`tr`/`td`/`th`) closes a select that is inside a table before being processed.
diff --git a/README.md b/README.md
index 523ae41..f72718d 100644
--- a/README.md
+++ b/README.md
@@ -88,7 +88,7 @@ flowchart TB
 
 PureXML is a working, dependency-free XML library today: parse, emit, validate,
 query, and stream documents on macOS, Linux, Windows, and WASM. The test suite
-currently runs **763 tests in 117 suites** (`swift test`).
+currently runs **764 tests in 117 suites** (`swift test`).
 
 ### Shipped (libxml2-aligned surface)
 
@@ -163,7 +163,7 @@ under epic **#71** (and its parser-breadth sub-epic **#74**):
 | XSLT 1.0 breadth | #81 #82 | Missing top-level elements and functions, html output |
 | HTML5 | #83 #84 | Full tree construction, tokenizer states, entity table |
 | C14N | #85 | Node-subset canonicalization, `xml:*` inheritance, 2.0 prefix rewrite |
-| Legacy encodings | #97 #99 | Single-byte and CJK multi-byte to-Unicode tables |
+| Legacy encodings | #97 #99 #102 | Single-byte, CJK multi-byte, and full GB18030 to-Unicode tables |
 | Validator decomposition | #101 | Inner XSD/DTD validators as composable rules |
 | Partial follow-ups | #79 #80 #86 #87 #88 #90 #91 | Remaining bullets on already-advanced areas |
 
diff --git a/Sources/Parsing/ByteDecoder.swift b/Sources/Parsing/ByteDecoder.swift
index fa14fec..7ccd85f 100644
--- a/Sources/Parsing/ByteDecoder.swift
+++ b/Sources/Parsing/ByteDecoder.swift
@@ -35,6 +35,7 @@ extension PureXML.Parsing {
             case .eucJP: EUCJP.decode(body)
             case .eucKR: EUCKR.decode(body)
             case .gbk: GBK.decode(body)
+            case .gb18030: GB18030.decode(body)
             default: nil
             }
         }
@@ -160,6 +161,7 @@ extension PureXML.Parsing {
             "euc-jp": .eucJP, "eucjp": .eucJP, "x-euc-jp": .eucJP, "cseucpkdfmtjapanese": .eucJP,
             "euc-kr": .eucKR, "euckr": .eucKR, "cp949": .eucKR, "uhc": .eucKR, "ks_c_5601-1987": .eucKR, "korean": .eucKR,
             "gbk": .gbk, "gb2312": .gbk, "gb_2312-80": .gbk, "csgb2312": .gbk, "x-gbk": .gbk, "chinese": .gbk,
+            "gb18030": .gb18030, "gb-18030": .gb18030, "csgb18030": .gb18030,
             "iso-8859-5": .latinCyrillic, "iso8859-5": .latinCyrillic, "cyrillic": .latinCyrillic,
             "iso-8859-9": .latin5, "iso8859-9": .latin5, "latin5": .latin5, "latin-5": .latin5, "l5": .latin5,
             "iso-8859-15": .latin9, "iso8859-15": .latin9, "latin9": .latin9, "latin-9": .latin9, "l9": .latin9,
diff --git a/Sources/Parsing/GB18030Decoder.swift b/Sources/Parsing/GB18030Decoder.swift
new file mode 100644
index 0000000..30b8703
--- /dev/null
+++ b/Sources/Parsing/GB18030Decoder.swift
@@ -0,0 +1,94 @@
+extension PureXML.Parsing.ByteDecoder {
+    /// The GB18030 decoder (the WHATWG `gb18030` algorithm). It extends GBK with
+    /// the four-byte sequences: a lead in `0x81`-`0xFE` followed by a digit
+    /// (`0x30`-`0x39`) opens a four-byte form whose pointer is mapped through the
+    /// linear ranges, reaching the astral planes. Two-byte pairs resolve through
+    /// the shared GBK index.
+    enum GB18030 {
+        static func decode(_ bytes: ArraySlice<UInt8>) -> String {
+            var scalars = String.UnicodeScalarView()
+            let arr = Array(bytes)
+            var index = 0
+            while index < arr.count {
+                let byte = arr[index]
+                if byte <= 0x7F {
+                    scalars.append(Unicode.Scalar(byte))
+                    index += 1
+                } else if byte == 0x80 {
+                    scalars.append("\u{20AC}") // euro
+                    index += 1
+                } else if (0x81 ... 0xFE).contains(byte) {
+                    index += step(arr, index, byte, into: &scalars)
+                } else {
+                    scalars.append("\u{FFFD}")
+                    index += 1
+                }
+            }
+            return String(scalars)
+        }
+
+        /// Consumes one sequence starting at a lead byte and returns how many bytes
+        /// it advanced. A digit second byte opens the four-byte form; anything else
+        /// is a two-byte pair resolved through the GBK index.
+        private static func step(
+            _ arr: [UInt8],
+            _ index: Int,
+            _ lead: UInt8,
+            into scalars: inout String.UnicodeScalarView,
+        ) -> Int {
+            guard index + 1 < arr.count else {
+                scalars.append("\u{FFFD}")
+                return 1
+            }
+            let second = arr[index + 1]
+            if (0x30 ... 0x39).contains(second) {
+                guard index + 3 < arr.count,
+                      (0x81 ... 0xFE).contains(arr[index + 2]),
+                      (0x30 ... 0x39).contains(arr[index + 3])
+                else {
+                    scalars.append("\u{FFFD}")
+                    return 1
+                }
+                let pointer = fourBytePointer(lead, second, arr[index + 2], arr[index + 3])
+                scalars.append(rangeScalar(pointer) ?? "\u{FFFD}")
+                return 4
+            }
+            if let scalar = GBK.twoByteScalar(lead: lead, trail: second) {
+                scalars.append(scalar)
+                return 2
+            }
+            scalars.append("\u{FFFD}")
+            return second <= 0x7F ? 1 : 2
+        }
+
+        /// The four-byte pointer from a `(b1, b2, b3, b4)` sequence.
+        static func fourBytePointer(_ byte1: UInt8, _ byte2: UInt8, _ byte3: UInt8, _ byte4: UInt8) -> Int {
+            (Int(byte1) - 0x81) * 12600
+                + (Int(byte2) - 0x30) * 1260
+                + (Int(byte3) - 0x81) * 10
+                + (Int(byte4) - 0x30)
+        }
+
+        /// The scalar for a four-byte pointer through the linear ranges, or nil for
+        /// the unmapped gap and the out-of-range tail.
+        static func rangeScalar(_ pointer: Int) -> Unicode.Scalar? {
+            if pointer > 39419, pointer < 189_000 { return nil }
+            if pointer > 1_237_575 { return nil }
+            let target = UInt32(pointer)
+            var low = 0
+            var high = gb18030RangePointers.count - 1
+            var found = 0
+            while low <= high {
+                let mid = (low + high) / 2
+                if gb18030RangePointers[mid] <= target {
+                    found = mid
+                    low = mid + 1
+                } else {
+                    high = mid - 1
+                }
+            }
+            let code = gb18030RangeCodePoints[found] + (target - gb18030RangePointers[found])
+            return Unicode.Scalar(code)
+        }
+    }
+}
diff --git a/Sources/Parsing/GB18030Ranges.swift b/Sources/Parsing/GB18030Ranges.swift
new file mode 100644
index 0000000..dbe02a6
--- /dev/null
+++ b/Sources/Parsing/GB18030Ranges.swift
@@ -0,0 +1,52 @@
+extension PureXML.Parsing.ByteDecoder {
+    /// The GB18030 four-byte linear ranges, vendored from the WHATWG
+    /// `index-gb18030-ranges.txt`. `gb18030RangePointers[i]` is the first
+    /// pointer of a run that maps to `gb18030RangeCodePoints[i]`. Reproduce
+    /// from that file.
+    static let gb18030RangePointers: [UInt32] = [
+        0x0, 0x24, 0x26, 0x2D, 0x32, 0x51, 0x59, 0x5F, 0x60, 0x64,
+        0x67, 0x68, 0x69, 0x6D, 0x7E, 0x85, 0x94, 0xAC, 0xAF, 0xB3,
+        0xD0, 0x132, 0x133, 0x134, 0x135, 0x136, 0x137, 0x138, 0x139, 0x155,
+        0x1AC, 0x1BB, 0x220, 0x221, 0x22E, 0x2E5, 0x2E6, 0x2ED, 0x2EE, 0x325,
+        0x333, 0x334, 0x1EF2, 0x1EF4, 0x1EF5, 0x1EF7, 0x1EFE, 0x1F07, 0x1F08, 0x1F09,
+        0x1F0E, 0x1F7E, 0x1FD4, 0x1FD5, 0x1FD8, 0x1FE4, 0x1FEE, 0x202C, 0x2030, 0x2046,
+        0x2048, 0x20B6, 0x20BC, 0x20BD, 0x20C0, 0x20C4, 0x20C6, 0x20C8, 0x20C9, 0x20CA,
+        0x20CC, 0x20D1, 0x20D6, 0x20E0, 0x20E3, 0x20E8, 0x20F5, 0x20F7, 0x20FD, 0x2122,
+        0x2125, 0x2130, 0x2149, 0x219B, 0x22E8, 0x22F2, 0x2356, 0x235A, 0x2367, 0x236A,
+        0x2374, 0x2384, 0x238C, 0x2394, 0x2397, 0x2399, 0x23AB, 0x23CA, 0x23CC, 0x2402,
+        0x2403, 0x2C41, 0x2C43, 0x2C46, 0x2C48, 0x2C52, 0x2C61, 0x2C63, 0x2C66, 0x2C6A,
+        0x2C6C, 0x2C6F, 0x2C7D, 0x2DA2, 0x2DA6, 0x2DA7, 0x2DAC, 0x2DAE, 0x2DC2, 0x2DC4,
+        0x2DCB, 0x2DCD, 0x2DD2, 0x2DD8, 0x2ECE, 0x2ED5, 0x2F46, 0x3030, 0x303C, 0x303E,
+        0x3060, 0x3069, 0x306B, 0x306D, 0x30DE, 0x3109, 0x3233, 0x32A2, 0x32AD, 0x35AA,
+        0x35FF, 0x365F, 0x366D, 0x3700, 0x37DA, 0x38F9, 0x396A, 0x3CDF, 0x3DE7, 0x3FBE,
+        0x4032, 0x4036, 0x4061, 0x4159, 0x42CE, 0x42E2, 0x43A3, 0x43A8, 0x43FA, 0x440A,
+        0x45C3, 0x45F5, 0x45F7, 0x45FB, 0x45FC, 0x4610, 0x4613, 0x4629, 0x48E8, 0x490F,
+        0x497E, 0x4A12, 0x4A63, 0x82BD, 0x82BE, 0x82BF, 0x82CC, 0x82CD, 0x82D2, 0x82D9,
+        0x82DD, 0x82E1, 0x82E9, 0x82F0, 0x8300, 0x830E, 0x93D5, 0x9421, 0x943C, 0x948D,
+        0x9496, 0x94B0, 0x94B1, 0x94B2, 0x94B5, 0x94BB, 0x94BC, 0x94BE, 0x98C4, 0x98C5,
+        0x98C9, 0x98CA, 0x98CB, 0x98CC, 0x9961, 0x99E2, 0x2E248,
+    ]
+    static let gb18030RangeCodePoints: [UInt32] = [
+        0x80, 0xA5, 0xA9, 0xB2, 0xB8, 0xD8, 0xE2, 0xEB, 0xEE, 0xF4,
+        0xF8, 0xFB, 0xFD, 0x102, 0x114, 0x11C, 0x12C, 0x145, 0x149, 0x14E,
+        0x16C, 0x1CF, 0x1D1, 0x1D3, 0x1D5, 0x1D7, 0x1D9, 0x1DB, 0x1DD, 0x1FA,
+        0x252, 0x262, 0x2C8, 0x2CC, 0x2DA, 0x3A2, 0x3AA, 0x3C2, 0x3CA, 0x402,
+        0x450, 0x452, 0x2011, 0x2017, 0x201A, 0x201E, 0x2027, 0x2031, 0x2034, 0x2036,
+        0x203C, 0x20AD, 0x2104, 0x2106, 0x210A, 0x2117, 0x2122, 0x216C, 0x217A, 0x2194,
+        0x219A, 0x2209, 0x2210, 0x2212, 0x2216, 0x221B, 0x2221, 0x2224, 0x2226, 0x222C,
+        0x222F, 0x2238, 0x223E, 0x2249, 0x224D, 0x2253, 0x2262, 0x2268, 0x2270, 0x2296,
+        0x229A, 0x22A6, 0x22C0, 0x2313, 0x246A, 0x249C, 0x254C, 0x2574, 0x2590, 0x2596,
+        0x25A2, 0x25B4, 0x25BE, 0x25C8, 0x25CC, 0x25D0, 0x25E6, 0x2607, 0x260A, 0x2641,
+        0x2643, 0x2E82, 0x2E85, 0x2E89, 0x2E8D, 0x2E98, 0x2EA8, 0x2EAB, 0x2EAF, 0x2EB4,
+        0x2EB8, 0x2EBC, 0x2ECB, 0x2FFC, 0x3004, 0x3018, 0x301F, 0x302A, 0x303F, 0x3094,
+        0x309F, 0x30F7, 0x30FF, 0x312A, 0x322A, 0x3232, 0x32A4, 0x3390, 0x339F, 0x33A2,
+        0x33C5, 0x33CF, 0x33D3, 0x33D6, 0x3448, 0x3474, 0x359F, 0x360F, 0x361B, 0x3919,
+        0x396F, 0x39D1, 0x39E0, 0x3A74, 0x3B4F, 0x3C6F, 0x3CE1, 0x4057, 0x4160, 0x4338,
+        0x43AD, 0x43B2, 0x43DE, 0x44D7, 0x464D, 0x4662, 0x4724, 0x472A, 0x477D, 0x478E,
+        0x4948, 0x497B, 0x497E, 0x4984, 0x4987, 0x499C, 0x49A0, 0x49B8, 0x4C78, 0x4CA4,
+        0x4D1A, 0x4DAF, 0x9FA6, 0xE76C, 0xE7C8, 0xE7E7, 0xE815, 0xE819, 0xE81F, 0xE827,
+        0xE82D, 0xE833, 0xE83C, 0xE844, 0xE856, 0xE865, 0xF92D, 0xF97A, 0xF996, 0xF9E8,
+        0xF9F2, 0xFA10, 0xFA12, 0xFA15, 0xFA19, 0xFA22, 0xFA25, 0xFA2A, 0xFE32, 0xFE45,
+        0xFE53, 0xFE58, 0xFE67, 0xFE6C, 0xFF5F, 0xFFE6, 0x10000,
+    ]
+}
diff --git a/Sources/Parsing/InputEncoding.swift b/Sources/Parsing/InputEncoding.swift
index 638ca28..2b69bcf 100644
--- a/Sources/Parsing/InputEncoding.swift
+++ b/Sources/Parsing/InputEncoding.swift
@@ -35,6 +35,7 @@ public extension PureXML.Parsing {
         case eucJP
         case eucKR
         case gbk
+        case gb18030
         case latinCyrillic // ISO-8859-5
         case latin5 // ISO-8859-9 (Turkish)
         case latin9 // ISO-8859-15
diff --git a/Sources/Parsing/StreamingDecoder.swift b/Sources/Parsing/StreamingDecoder.swift
index 65efe53..5ccf43c 100644
--- a/Sources/Parsing/StreamingDecoder.swift
+++ b/Sources/Parsing/StreamingDecoder.swift
@@ -46,6 +46,7 @@ extension PureXML.Parsing {
             case .eucJP: nextEUCJP()
             case .eucKR: nextEUCKR()
             case .gbk: nextGBK()
+            case .gb18030: nextGB18030()
             default: nextUTF8()
             }
         }
@@ -65,6 +66,30 @@ extension PureXML.Parsing {
             return PureXML.Parsing.ByteDecoder.GBK.twoByteScalar(lead: lead, trail: trail).map(Character.init) ?? Self.replacement
         }
 
+        /// Streams one GB18030 character: ASCII, `0x80` (euro), a four-byte
+        /// sequence when the lead is followed by a digit, or a two-byte GBK pair.
+        private mutating func nextGB18030() -> Character? {
+            guard let lead = nextByte() else {
+                finished = true
+                return nil
+            }
+            if lead <= 0x7F { return Character(Unicode.Scalar(lead)) }
+            if lead == 0x80 { return "\u{20AC}" }
+            guard (0x81 ... 0xFE).contains(lead) else { return Self.replacement }
+            guard let second = nextByte() else { finished = true
+                return Self.replacement
+            }
+            if (0x30 ... 0x39).contains(second) {
+                guard let third = nextByte(), let fourth = nextByte() else { finished = true
+                    return Self.replacement
+                }
+                guard (0x81 ... 0xFE).contains(third), (0x30 ... 0x39).contains(fourth) else { return Self.replacement }
+                let pointer = PureXML.Parsing.ByteDecoder.GB18030.fourBytePointer(lead, second, third, fourth)
+                return PureXML.Parsing.ByteDecoder.GB18030.rangeScalar(pointer).map(Character.init) ?? Self.replacement
+            }
+            return PureXML.Parsing.ByteDecoder.GBK.twoByteScalar(lead: lead, trail: second).map(Character.init) ?? Self.replacement
+        }
+
         /// Streams one EUC-KR character: ASCII, or a lead+trail through CP949.
         private mutating func nextEUCKR() -> Character? {
             guard let lead = nextByte() else {
diff --git a/Tests/EncodingTests.swift b/Tests/EncodingTests.swift
index 304ff80..3415986 100644
--- a/Tests/EncodingTests.swift
+++ b/Tests/EncodingTests.swift
@@ -170,6 +170,18 @@ struct EncodingTests {
         try #expect(decoded("gb2312", [0xD6, 0xD0]) == "\u{4E2D}") // GB2312 alias
     }
 
+    @Test("Decodes GB18030: two-byte pairs plus the four-byte and astral ranges")
+    func test_gb18030() throws {
+        // 中 (0xD6D0) shares the GBK two-byte range; 0x80 = euro.
+        try #expect(decoded("GB18030", [0xD6, 0xD0, 0x80]) == "\u{4E2D}\u{20AC}")
+        // Four-byte sequences: ptr 36 = ¥ (U+00A5), ptr 0 = U+0080.
+        try #expect(decoded("gb18030", [0x81, 0x30, 0x84, 0x36]) == "\u{00A5}")
+        try #expect(decoded("gb-18030", [0x81, 0x30, 0x81, 0x30]) == "\u{0080}")
+        // The astral ranges: ptr 189000 -> U+10000, ptr 254536 -> U+20000.
+        try #expect(decoded("GB18030", [0x90, 0x30, 0x81, 0x30]) == "\u{10000}")
+        try #expect(decoded("GB18030", [0x95, 0x32, 0x82, 0x36]) == "\u{20000}")
+    }
+
     @Test("Decodes ISO-8859-5: the Cyrillic block")
     func test_iso8859_5() throws {
         try #expect(decoded("ISO-8859-5", [0xB0]) == "\u{0410}") // А