NodetteHQ/PureXML
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

test(parsing): cover streaming ISO-2022-JP and EUC-TW decode

Browse Source 
Exercise the byte-at-a-time ISO-2022-JP mode escapes (ASCII, JIS-Roman,
half-width katakana, JIS X 0208) and EUC-TW plane-1 plus the 0x8E four-byte
selector. Raises streaming ISO-2022-JP coverage from 54% to 92% and EUC-TW
from 52% to 76%. Output is compared by Unicode scalar to avoid a grapheme
cluster swallowing the trailing sound mark.
This commit is contained in:
Mihaela Mihaljevic 2026-06-22 16:59:42 +02:00
parent d15faed122
commit 0ff46f6b78
1 changed files with 76 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
Tests/StreamingDecoderJPTWTests.swift Normal file
View File

@ -0,0 +1,76 @@
import Testing
@testable import PureXML
/// Streaming (byte-at-a-time) coverage for the two CJK decoders the existing
/// StreamingDecoderTests did not exercise: ISO-2022-JP (mode escapes: ASCII,
/// JIS-Roman, half-width katakana, JIS X 0208) and EUC-TW (two-byte plane 1 and
/// the 0x8E four-byte plane selector). Computed scalars (ASCII, yen, overline,
/// katakana) are derived from the decoder's arithmetic; table-mapped scalars are
/// pinned from the decoder itself, matching how StreamingDecoderTests was built.
///
/// Output is compared as Unicode scalars, not Characters: a half-width katakana
/// sound mark (U+FF9F) would grapheme-cluster with the preceding declaration byte,
/// which a Character-count `dropFirst` would then swallow. The scalars are intact.
@Suite("StreamingDecoder ISO-2022-JP / EUC-TW")
struct StreamingDecoderJPTWTests {
/// Decodes `bytes` pulled one at a time, so every multi-byte / multi-mode
/// sequence is split across pull boundaries.
private func decode(_ bytes: [UInt8]) -> String {
var index = 0
var decoder = PureXML.Parsing.StreamingDecoder(pullingBytes: {
defer { index += 1 }
return index < bytes.count ? bytes[index] : nil
})
var out = ""
while let character = decoder.next() {
out.append(character)
}
return out
}
/// The decoded content scalars after a declaration in `encoding`. The prefix is
/// pure ASCII (one scalar per byte), so dropping `prefix.count` scalars leaves
/// exactly the content's scalars.
private func scalars(_ encoding: String, _ content: [UInt8]) -> [UInt32] {
let prefix = Array("<?xml version='1.0' encoding='\(encoding)'?>".utf8)
let full = decode(prefix + content)
return Array(full.unicodeScalars.dropFirst(prefix.count)).map(\.value)
}
@Test("ISO-2022-JP escapes between ASCII, JIS-Roman and half-width katakana")
func test_iso2022jpComputedModes() {
// Default mode is ASCII.
#expect(scalars("ISO-2022-JP", [0x41]) == [0x41])
// ESC ( J -> JIS-Roman: 0x5C is yen, 0x7E is overline.
#expect(scalars("ISO-2022-JP", [0x1B, 0x28, 0x4A, 0x5C]) == [0x00A5])
#expect(scalars("ISO-2022-JP", [0x1B, 0x28, 0x4A, 0x7E]) == [0x203E])
// ESC ( I -> half-width katakana: byte 0x21 maps to U+FF61, 0x5F to U+FF9F.
#expect(scalars("ISO-2022-JP", [0x1B, 0x28, 0x49, 0x21]) == [0xFF61])
#expect(scalars("ISO-2022-JP", [0x1B, 0x28, 0x49, 0x5F]) == [0xFF9F])
// An unknown escape yields the replacement character, not a crash.
#expect(scalars("ISO-2022-JP", [0x1B, 0x28, 0x5A]) == [0xFFFD])
}
@Test("ISO-2022-JP JIS X 0208 two-byte pair decodes across pulls")
func test_iso2022jpJIS0208() {
// ESC $ B -> JIS X 0208; ku=4,ten=2 (bytes 0x24,0x22) is あ (U+3042). An ESC ( B
// returns to ASCII so a trailing ASCII byte decodes normally.
#expect(scalars("ISO-2022-JP", [0x1B, 0x24, 0x42, 0x24, 0x22]) == [0x3042])
#expect(scalars("ISO-2022-JP", [0x1B, 0x24, 0x42, 0x24, 0x22, 0x1B, 0x28, 0x42, 0x41]) == [0x3042, 0x41])
// An invalid lead (0x20) is one replacement; its trail (0x21) is then re-read
// as a fresh lead with no following byte, a second replacement.
#expect(scalars("ISO-2022-JP", [0x1B, 0x24, 0x42, 0x20, 0x21]) == [0xFFFD, 0xFFFD])
}
@Test("EUC-TW ASCII, two-byte plane 1, and the 0x8E four-byte selector")
func test_eucTW() {
#expect(scalars("EUC-TW", [0x41]) == [0x41])
// Plane-1 lead 0xA1 trail 0xA1 is the ideographic space U+3000.
#expect(scalars("EUC-TW", [0xA1, 0xA1]) == [0x3000])
// The 0x8E selector picks plane 1 explicitly: 0x8E 0xA1 0xA1 0xA1 is also U+3000.
#expect(scalars("EUC-TW", [0x8E, 0xA1, 0xA1, 0xA1]) == [0x3000])
// A lead outside 0xA1...0xFE (and not 0x8E) is the replacement character.
#expect(scalars("EUC-TW", [0x80]) == [0xFFFD])
#expect(scalars("EUC-TW", [0xFF]) == [0xFFFD])
}
}