feat(parsing): full GB18030 four-byte sequences
Extend the GBK two-byte subset to full GB18030: a lead byte followed by a digit opens a four-byte form whose pointer is mapped through the WHATWG index-gb18030-ranges linear ranges, reaching the supplementary planes. The 207 ranges are vendored verbatim. Wired into the whole-buffer and streaming decoders and the encoding-name table (gb18030/gb-18030). Closes #102
This commit is contained in:
parent
e6b93737ed
commit
2b2c1fb692
@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Added
|
||||
|
||||
- Full GB18030 (#102): the byte decoder gains the GB18030 four-byte sequences on top of the GBK two-byte subset, for both the whole-buffer and streaming decoders, selected by the `gb18030` declaration name. A lead in `0x81`-`0xFE` followed by a digit opens a four-byte form whose pointer is mapped through the WHATWG `index-gb18030-ranges` linear ranges, reaching the supplementary planes (the `189000` range maps to `U+10000` and beyond). The 207 ranges are vendored verbatim from the WHATWG `index-gb18030-ranges.txt`. GB18030 is China's mandatory standard, supported in libxml2 through iconv.
|
||||
- CJK multi-byte encodings (#99, in progress): the byte decoder gains a multi-byte path and the WHATWG **Shift-JIS** and **EUC-JP** algorithms, for both the whole-buffer and streaming decoders, selected by the XML declaration's `encoding` name. Shift-JIS covers ASCII, half-width katakana, and the JIS X 0208 plane; EUC-JP adds the `0x8E` half-width katakana and `0x8F` JIS X 0212 leads. EUC-KR (CP949/UHC) decodes Hangul and Hanja through the WHATWG `index-euc-kr`, and GBK (which also covers GB2312) decodes Chinese through the two-byte range of `index-gb18030` with `0x80` as the euro sign. The JIS X 0208/0212, CP949, and GBK indexes are all vendored verbatim from the corresponding WHATWG `index-*.txt` files. This completes the CJK multi-byte encodings.
|
||||
- HTML named character references: the complete WHATWG named character reference set (#84), all 2,125 semicolon-form references including the astral mathematical alphanumerics and the 93 two-codepoint references, vendored verbatim from the authoritative `entities.json` (replacing the previous 16-entry stopgap). Decoding stays longest-match with an optional trailing semicolon, on top of the existing numeric-reference validation.
|
||||
- HTML5 select content model (#83, stage 10): a `<select>` now follows the in-select rules. `<optgroup>` closes an open `<option>`, a nested `<select>` closes the open one rather than nesting, and an `<input>`/`<keygen>`/`<textarea>` closes the select and is placed after it. In the "in select in table" case, a table-structural tag (`caption`/`table`/`tbody`/`tfoot`/`thead`/`tr`/`td`/`th`) closes a select that is inside a table before being processed.
|
||||
|
||||
@ -88,7 +88,7 @@ flowchart TB
|
||||
|
||||
PureXML is a working, dependency-free XML library today: parse, emit, validate,
|
||||
query, and stream documents on macOS, Linux, Windows, and WASM. The test suite
|
||||
currently runs **763 tests in 117 suites** (`swift test`).
|
||||
currently runs **764 tests in 117 suites** (`swift test`).
|
||||
|
||||
### Shipped (libxml2-aligned surface)
|
||||
|
||||
@ -163,7 +163,7 @@ under epic **#71** (and its parser-breadth sub-epic **#74**):
|
||||
| XSLT 1.0 breadth | #81 #82 | Missing top-level elements and functions, html output |
|
||||
| HTML5 | #83 #84 | Full tree construction, tokenizer states, entity table |
|
||||
| C14N | #85 | Node-subset canonicalization, `xml:*` inheritance, 2.0 prefix rewrite |
|
||||
| Legacy encodings | #97 #99 | Single-byte and CJK multi-byte to-Unicode tables |
|
||||
| Legacy encodings | #97 #99 #102 | Single-byte, CJK multi-byte, and full GB18030 to-Unicode tables |
|
||||
| Validator decomposition | #101 | Inner XSD/DTD validators as composable rules |
|
||||
| Partial follow-ups | #79 #80 #86 #87 #88 #90 #91 | Remaining bullets on already-advanced areas |
|
||||
|
||||
|
||||
@ -35,6 +35,7 @@ extension PureXML.Parsing {
|
||||
case .eucJP: EUCJP.decode(body)
|
||||
case .eucKR: EUCKR.decode(body)
|
||||
case .gbk: GBK.decode(body)
|
||||
case .gb18030: GB18030.decode(body)
|
||||
default: nil
|
||||
}
|
||||
}
|
||||
@ -160,6 +161,7 @@ extension PureXML.Parsing {
|
||||
"euc-jp": .eucJP, "eucjp": .eucJP, "x-euc-jp": .eucJP, "cseucpkdfmtjapanese": .eucJP,
|
||||
"euc-kr": .eucKR, "euckr": .eucKR, "cp949": .eucKR, "uhc": .eucKR, "ks_c_5601-1987": .eucKR, "korean": .eucKR,
|
||||
"gbk": .gbk, "gb2312": .gbk, "gb_2312-80": .gbk, "csgb2312": .gbk, "x-gbk": .gbk, "chinese": .gbk,
|
||||
"gb18030": .gb18030, "gb-18030": .gb18030, "csgb18030": .gb18030,
|
||||
"iso-8859-5": .latinCyrillic, "iso8859-5": .latinCyrillic, "cyrillic": .latinCyrillic,
|
||||
"iso-8859-9": .latin5, "iso8859-9": .latin5, "latin5": .latin5, "latin-5": .latin5, "l5": .latin5,
|
||||
"iso-8859-15": .latin9, "iso8859-15": .latin9, "latin9": .latin9, "latin-9": .latin9, "l9": .latin9,
|
||||
|
||||
94
Sources/Parsing/GB18030Decoder.swift
Normal file
94
Sources/Parsing/GB18030Decoder.swift
Normal file
@ -0,0 +1,94 @@
|
||||
extension PureXML.Parsing.ByteDecoder {
|
||||
/// The GB18030 decoder (the WHATWG `gb18030` algorithm). It extends GBK with
|
||||
/// the four-byte sequences: a lead in `0x81`-`0xFE` followed by a digit
|
||||
/// (`0x30`-`0x39`) opens a four-byte form whose pointer is mapped through the
|
||||
/// linear ranges, reaching the astral planes. Two-byte pairs resolve through
|
||||
/// the shared GBK index.
|
||||
enum GB18030 {
|
||||
static func decode(_ bytes: ArraySlice<UInt8>) -> String {
|
||||
var scalars = String.UnicodeScalarView()
|
||||
let arr = Array(bytes)
|
||||
var index = 0
|
||||
while index < arr.count {
|
||||
let byte = arr[index]
|
||||
if byte <= 0x7F {
|
||||
scalars.append(Unicode.Scalar(byte))
|
||||
index += 1
|
||||
} else if byte == 0x80 {
|
||||
scalars.append("\u{20AC}") // euro
|
||||
index += 1
|
||||
} else if (0x81 ... 0xFE).contains(byte) {
|
||||
index += step(arr, index, byte, into: &scalars)
|
||||
} else {
|
||||
scalars.append("\u{FFFD}")
|
||||
index += 1
|
||||
}
|
||||
}
|
||||
return String(scalars)
|
||||
}
|
||||
|
||||
/// Consumes one sequence starting at a lead byte and returns how many bytes
|
||||
/// it advanced. A digit second byte opens the four-byte form; anything else
|
||||
/// is a two-byte pair resolved through the GBK index.
|
||||
private static func step(
|
||||
_ arr: [UInt8],
|
||||
_ index: Int,
|
||||
_ lead: UInt8,
|
||||
into scalars: inout String.UnicodeScalarView,
|
||||
) -> Int {
|
||||
guard index + 1 < arr.count else {
|
||||
scalars.append("\u{FFFD}")
|
||||
return 1
|
||||
}
|
||||
let second = arr[index + 1]
|
||||
if (0x30 ... 0x39).contains(second) {
|
||||
guard index + 3 < arr.count,
|
||||
(0x81 ... 0xFE).contains(arr[index + 2]),
|
||||
(0x30 ... 0x39).contains(arr[index + 3])
|
||||
else {
|
||||
scalars.append("\u{FFFD}")
|
||||
return 1
|
||||
}
|
||||
let pointer = fourBytePointer(lead, second, arr[index + 2], arr[index + 3])
|
||||
scalars.append(rangeScalar(pointer) ?? "\u{FFFD}")
|
||||
return 4
|
||||
}
|
||||
if let scalar = GBK.twoByteScalar(lead: lead, trail: second) {
|
||||
scalars.append(scalar)
|
||||
return 2
|
||||
}
|
||||
scalars.append("\u{FFFD}")
|
||||
return second <= 0x7F ? 1 : 2
|
||||
}
|
||||
|
||||
/// The four-byte pointer from a `(b1, b2, b3, b4)` sequence.
|
||||
static func fourBytePointer(_ byte1: UInt8, _ byte2: UInt8, _ byte3: UInt8, _ byte4: UInt8) -> Int {
|
||||
(Int(byte1) - 0x81) * 12600
|
||||
+ (Int(byte2) - 0x30) * 1260
|
||||
+ (Int(byte3) - 0x81) * 10
|
||||
+ (Int(byte4) - 0x30)
|
||||
}
|
||||
|
||||
/// The scalar for a four-byte pointer through the linear ranges, or nil for
|
||||
/// the unmapped gap and the out-of-range tail.
|
||||
static func rangeScalar(_ pointer: Int) -> Unicode.Scalar? {
|
||||
if pointer > 39419, pointer < 189_000 { return nil }
|
||||
if pointer > 1_237_575 { return nil }
|
||||
let target = UInt32(pointer)
|
||||
var low = 0
|
||||
var high = gb18030RangePointers.count - 1
|
||||
var found = 0
|
||||
while low <= high {
|
||||
let mid = (low + high) / 2
|
||||
if gb18030RangePointers[mid] <= target {
|
||||
found = mid
|
||||
low = mid + 1
|
||||
} else {
|
||||
high = mid - 1
|
||||
}
|
||||
}
|
||||
let code = gb18030RangeCodePoints[found] + (target - gb18030RangePointers[found])
|
||||
return Unicode.Scalar(code)
|
||||
}
|
||||
}
|
||||
}
|
||||
52
Sources/Parsing/GB18030Ranges.swift
Normal file
52
Sources/Parsing/GB18030Ranges.swift
Normal file
@ -0,0 +1,52 @@
|
||||
extension PureXML.Parsing.ByteDecoder {
|
||||
/// The GB18030 four-byte linear ranges, vendored from the WHATWG
|
||||
/// `index-gb18030-ranges.txt`. `gb18030RangePointers[i]` is the first
|
||||
/// pointer of a run that maps to `gb18030RangeCodePoints[i]`. Reproduce
|
||||
/// from that file.
|
||||
static let gb18030RangePointers: [UInt32] = [
|
||||
0x0, 0x24, 0x26, 0x2D, 0x32, 0x51, 0x59, 0x5F, 0x60, 0x64,
|
||||
0x67, 0x68, 0x69, 0x6D, 0x7E, 0x85, 0x94, 0xAC, 0xAF, 0xB3,
|
||||
0xD0, 0x132, 0x133, 0x134, 0x135, 0x136, 0x137, 0x138, 0x139, 0x155,
|
||||
0x1AC, 0x1BB, 0x220, 0x221, 0x22E, 0x2E5, 0x2E6, 0x2ED, 0x2EE, 0x325,
|
||||
0x333, 0x334, 0x1EF2, 0x1EF4, 0x1EF5, 0x1EF7, 0x1EFE, 0x1F07, 0x1F08, 0x1F09,
|
||||
0x1F0E, 0x1F7E, 0x1FD4, 0x1FD5, 0x1FD8, 0x1FE4, 0x1FEE, 0x202C, 0x2030, 0x2046,
|
||||
0x2048, 0x20B6, 0x20BC, 0x20BD, 0x20C0, 0x20C4, 0x20C6, 0x20C8, 0x20C9, 0x20CA,
|
||||
0x20CC, 0x20D1, 0x20D6, 0x20E0, 0x20E3, 0x20E8, 0x20F5, 0x20F7, 0x20FD, 0x2122,
|
||||
0x2125, 0x2130, 0x2149, 0x219B, 0x22E8, 0x22F2, 0x2356, 0x235A, 0x2367, 0x236A,
|
||||
0x2374, 0x2384, 0x238C, 0x2394, 0x2397, 0x2399, 0x23AB, 0x23CA, 0x23CC, 0x2402,
|
||||
0x2403, 0x2C41, 0x2C43, 0x2C46, 0x2C48, 0x2C52, 0x2C61, 0x2C63, 0x2C66, 0x2C6A,
|
||||
0x2C6C, 0x2C6F, 0x2C7D, 0x2DA2, 0x2DA6, 0x2DA7, 0x2DAC, 0x2DAE, 0x2DC2, 0x2DC4,
|
||||
0x2DCB, 0x2DCD, 0x2DD2, 0x2DD8, 0x2ECE, 0x2ED5, 0x2F46, 0x3030, 0x303C, 0x303E,
|
||||
0x3060, 0x3069, 0x306B, 0x306D, 0x30DE, 0x3109, 0x3233, 0x32A2, 0x32AD, 0x35AA,
|
||||
0x35FF, 0x365F, 0x366D, 0x3700, 0x37DA, 0x38F9, 0x396A, 0x3CDF, 0x3DE7, 0x3FBE,
|
||||
0x4032, 0x4036, 0x4061, 0x4159, 0x42CE, 0x42E2, 0x43A3, 0x43A8, 0x43FA, 0x440A,
|
||||
0x45C3, 0x45F5, 0x45F7, 0x45FB, 0x45FC, 0x4610, 0x4613, 0x4629, 0x48E8, 0x490F,
|
||||
0x497E, 0x4A12, 0x4A63, 0x82BD, 0x82BE, 0x82BF, 0x82CC, 0x82CD, 0x82D2, 0x82D9,
|
||||
0x82DD, 0x82E1, 0x82E9, 0x82F0, 0x8300, 0x830E, 0x93D5, 0x9421, 0x943C, 0x948D,
|
||||
0x9496, 0x94B0, 0x94B1, 0x94B2, 0x94B5, 0x94BB, 0x94BC, 0x94BE, 0x98C4, 0x98C5,
|
||||
0x98C9, 0x98CA, 0x98CB, 0x98CC, 0x9961, 0x99E2, 0x2E248,
|
||||
]
|
||||
static let gb18030RangeCodePoints: [UInt32] = [
|
||||
0x80, 0xA5, 0xA9, 0xB2, 0xB8, 0xD8, 0xE2, 0xEB, 0xEE, 0xF4,
|
||||
0xF8, 0xFB, 0xFD, 0x102, 0x114, 0x11C, 0x12C, 0x145, 0x149, 0x14E,
|
||||
0x16C, 0x1CF, 0x1D1, 0x1D3, 0x1D5, 0x1D7, 0x1D9, 0x1DB, 0x1DD, 0x1FA,
|
||||
0x252, 0x262, 0x2C8, 0x2CC, 0x2DA, 0x3A2, 0x3AA, 0x3C2, 0x3CA, 0x402,
|
||||
0x450, 0x452, 0x2011, 0x2017, 0x201A, 0x201E, 0x2027, 0x2031, 0x2034, 0x2036,
|
||||
0x203C, 0x20AD, 0x2104, 0x2106, 0x210A, 0x2117, 0x2122, 0x216C, 0x217A, 0x2194,
|
||||
0x219A, 0x2209, 0x2210, 0x2212, 0x2216, 0x221B, 0x2221, 0x2224, 0x2226, 0x222C,
|
||||
0x222F, 0x2238, 0x223E, 0x2249, 0x224D, 0x2253, 0x2262, 0x2268, 0x2270, 0x2296,
|
||||
0x229A, 0x22A6, 0x22C0, 0x2313, 0x246A, 0x249C, 0x254C, 0x2574, 0x2590, 0x2596,
|
||||
0x25A2, 0x25B4, 0x25BE, 0x25C8, 0x25CC, 0x25D0, 0x25E6, 0x2607, 0x260A, 0x2641,
|
||||
0x2643, 0x2E82, 0x2E85, 0x2E89, 0x2E8D, 0x2E98, 0x2EA8, 0x2EAB, 0x2EAF, 0x2EB4,
|
||||
0x2EB8, 0x2EBC, 0x2ECB, 0x2FFC, 0x3004, 0x3018, 0x301F, 0x302A, 0x303F, 0x3094,
|
||||
0x309F, 0x30F7, 0x30FF, 0x312A, 0x322A, 0x3232, 0x32A4, 0x3390, 0x339F, 0x33A2,
|
||||
0x33C5, 0x33CF, 0x33D3, 0x33D6, 0x3448, 0x3474, 0x359F, 0x360F, 0x361B, 0x3919,
|
||||
0x396F, 0x39D1, 0x39E0, 0x3A74, 0x3B4F, 0x3C6F, 0x3CE1, 0x4057, 0x4160, 0x4338,
|
||||
0x43AD, 0x43B2, 0x43DE, 0x44D7, 0x464D, 0x4662, 0x4724, 0x472A, 0x477D, 0x478E,
|
||||
0x4948, 0x497B, 0x497E, 0x4984, 0x4987, 0x499C, 0x49A0, 0x49B8, 0x4C78, 0x4CA4,
|
||||
0x4D1A, 0x4DAF, 0x9FA6, 0xE76C, 0xE7C8, 0xE7E7, 0xE815, 0xE819, 0xE81F, 0xE827,
|
||||
0xE82D, 0xE833, 0xE83C, 0xE844, 0xE856, 0xE865, 0xF92D, 0xF97A, 0xF996, 0xF9E8,
|
||||
0xF9F2, 0xFA10, 0xFA12, 0xFA15, 0xFA19, 0xFA22, 0xFA25, 0xFA2A, 0xFE32, 0xFE45,
|
||||
0xFE53, 0xFE58, 0xFE67, 0xFE6C, 0xFF5F, 0xFFE6, 0x10000,
|
||||
]
|
||||
}
|
||||
@ -35,6 +35,7 @@ public extension PureXML.Parsing {
|
||||
case eucJP
|
||||
case eucKR
|
||||
case gbk
|
||||
case gb18030
|
||||
case latinCyrillic // ISO-8859-5
|
||||
case latin5 // ISO-8859-9 (Turkish)
|
||||
case latin9 // ISO-8859-15
|
||||
|
||||
@ -46,6 +46,7 @@ extension PureXML.Parsing {
|
||||
case .eucJP: nextEUCJP()
|
||||
case .eucKR: nextEUCKR()
|
||||
case .gbk: nextGBK()
|
||||
case .gb18030: nextGB18030()
|
||||
default: nextUTF8()
|
||||
}
|
||||
}
|
||||
@ -65,6 +66,30 @@ extension PureXML.Parsing {
|
||||
return PureXML.Parsing.ByteDecoder.GBK.twoByteScalar(lead: lead, trail: trail).map(Character.init) ?? Self.replacement
|
||||
}
|
||||
|
||||
/// Streams one GB18030 character: ASCII, `0x80` (euro), a four-byte
|
||||
/// sequence when the lead is followed by a digit, or a two-byte GBK pair.
|
||||
private mutating func nextGB18030() -> Character? {
|
||||
guard let lead = nextByte() else {
|
||||
finished = true
|
||||
return nil
|
||||
}
|
||||
if lead <= 0x7F { return Character(Unicode.Scalar(lead)) }
|
||||
if lead == 0x80 { return "\u{20AC}" }
|
||||
guard (0x81 ... 0xFE).contains(lead) else { return Self.replacement }
|
||||
guard let second = nextByte() else { finished = true
|
||||
return Self.replacement
|
||||
}
|
||||
if (0x30 ... 0x39).contains(second) {
|
||||
guard let third = nextByte(), let fourth = nextByte() else { finished = true
|
||||
return Self.replacement
|
||||
}
|
||||
guard (0x81 ... 0xFE).contains(third), (0x30 ... 0x39).contains(fourth) else { return Self.replacement }
|
||||
let pointer = PureXML.Parsing.ByteDecoder.GB18030.fourBytePointer(lead, second, third, fourth)
|
||||
return PureXML.Parsing.ByteDecoder.GB18030.rangeScalar(pointer).map(Character.init) ?? Self.replacement
|
||||
}
|
||||
return PureXML.Parsing.ByteDecoder.GBK.twoByteScalar(lead: lead, trail: second).map(Character.init) ?? Self.replacement
|
||||
}
|
||||
|
||||
/// Streams one EUC-KR character: ASCII, or a lead+trail through CP949.
|
||||
private mutating func nextEUCKR() -> Character? {
|
||||
guard let lead = nextByte() else {
|
||||
|
||||
@ -170,6 +170,18 @@ struct EncodingTests {
|
||||
try #expect(decoded("gb2312", [0xD6, 0xD0]) == "\u{4E2D}") // GB2312 alias
|
||||
}
|
||||
|
||||
@Test("Decodes GB18030: two-byte pairs plus the four-byte and astral ranges")
|
||||
func test_gb18030() throws {
|
||||
// 中 (0xD6D0) shares the GBK two-byte range; 0x80 = euro.
|
||||
try #expect(decoded("GB18030", [0xD6, 0xD0, 0x80]) == "\u{4E2D}\u{20AC}")
|
||||
// Four-byte sequences: ptr 36 = ¥ (U+00A5), ptr 0 = U+0080.
|
||||
try #expect(decoded("gb18030", [0x81, 0x30, 0x84, 0x36]) == "\u{00A5}")
|
||||
try #expect(decoded("gb-18030", [0x81, 0x30, 0x81, 0x30]) == "\u{0080}")
|
||||
// The astral ranges: ptr 189000 -> U+10000, ptr 254536 -> U+20000.
|
||||
try #expect(decoded("GB18030", [0x90, 0x30, 0x81, 0x30]) == "\u{10000}")
|
||||
try #expect(decoded("GB18030", [0x95, 0x32, 0x82, 0x36]) == "\u{20000}")
|
||||
}
|
||||
|
||||
@Test("Decodes ISO-8859-5: the Cyrillic block")
|
||||
func test_iso8859_5() throws {
|
||||
try #expect(decoded("ISO-8859-5", [0xB0]) == "\u{0410}") // А
|
||||
|
||||
Loading…
Reference in New Issue
Block a user