feat(parsing): ISO-8859-6/8/10/14/16 vendored from Unicode mappings (#97)
This commit is contained in:
parent
43c7ecc5e3
commit
3f293ec1e7
@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Added
|
||||
|
||||
- Single-byte legacy encodings (#97, partial): the byte decoder gains a table-based single-byte path and the following legacy encodings, selected by the XML declaration's `encoding` name for both the whole-buffer and streaming decoders. ISO-8859-2/3/4/7/13 are vendored verbatim from the authoritative Unicode mapping files; ISO-8859-5/9/15 are derived exactly; Windows-1254 composes from Windows-1252. So far: ISO-8859-2 (Latin-2), ISO-8859-3 (Latin-3), ISO-8859-4 (Latin-4), ISO-8859-5 (Cyrillic), ISO-8859-7 (Greek), ISO-8859-9 (Turkish), ISO-8859-13 (Baltic), ISO-8859-15 (Latin-9), and Windows-1254. The remaining encodings are data drop-ins into the same path.
|
||||
- Single-byte legacy encodings (#97, partial): the byte decoder gains a table-based single-byte path and the full ISO-8859 family (ISO-8859-2 through 16, excluding the Thai part 11) plus Windows-1254, selected by the XML declaration's `encoding` name for both the whole-buffer and streaming decoders. The table-based encodings are vendored verbatim from the authoritative `unicode.org` mapping files; ISO-8859-5/9/15 are derived exactly; Windows-1254 composes from Windows-1252. The remaining encodings (Windows code pages, KOI8) are data drop-ins into the same path.
|
||||
|
||||
### Fixed
|
||||
|
||||
|
||||
@ -40,6 +40,11 @@ extension PureXML.Parsing {
|
||||
.latin4: SingleByte.iso8859_4,
|
||||
.greek: SingleByte.iso8859_7,
|
||||
.latin7: SingleByte.iso8859_13,
|
||||
.arabic: SingleByte.iso8859_6,
|
||||
.hebrew: SingleByte.iso8859_8,
|
||||
.latin6: SingleByte.iso8859_10,
|
||||
.latin8: SingleByte.iso8859_14,
|
||||
.latin10: SingleByte.iso8859_16,
|
||||
.latinCyrillic: SingleByte.iso8859_5,
|
||||
.latin5: SingleByte.iso8859_9,
|
||||
.latin9: SingleByte.iso8859_15,
|
||||
@ -111,6 +116,11 @@ extension PureXML.Parsing {
|
||||
"iso-8859-4": .latin4, "iso8859-4": .latin4, "latin4": .latin4, "latin-4": .latin4, "l4": .latin4,
|
||||
"iso-8859-7": .greek, "iso8859-7": .greek, "greek": .greek,
|
||||
"iso-8859-13": .latin7, "iso8859-13": .latin7, "latin7": .latin7, "latin-7": .latin7, "l7": .latin7,
|
||||
"iso-8859-6": .arabic, "iso8859-6": .arabic, "arabic": .arabic,
|
||||
"iso-8859-8": .hebrew, "iso8859-8": .hebrew, "hebrew": .hebrew,
|
||||
"iso-8859-10": .latin6, "iso8859-10": .latin6, "latin6": .latin6, "latin-6": .latin6, "l6": .latin6,
|
||||
"iso-8859-14": .latin8, "iso8859-14": .latin8, "latin8": .latin8, "latin-8": .latin8, "l8": .latin8,
|
||||
"iso-8859-16": .latin10, "iso8859-16": .latin10, "latin10": .latin10, "latin-10": .latin10, "l10": .latin10,
|
||||
"iso-8859-5": .latinCyrillic, "iso8859-5": .latinCyrillic, "cyrillic": .latinCyrillic,
|
||||
"iso-8859-9": .latin5, "iso8859-9": .latin5, "latin5": .latin5, "latin-5": .latin5, "l5": .latin5,
|
||||
"iso-8859-15": .latin9, "iso8859-15": .latin9, "latin9": .latin9, "latin-9": .latin9, "l9": .latin9,
|
||||
|
||||
@ -16,6 +16,11 @@ public extension PureXML.Parsing {
|
||||
case latin4 // ISO-8859-4
|
||||
case greek // ISO-8859-7
|
||||
case latin7 // ISO-8859-13
|
||||
case arabic // ISO-8859-6
|
||||
case hebrew // ISO-8859-8
|
||||
case latin6 // ISO-8859-10
|
||||
case latin8 // ISO-8859-14
|
||||
case latin10 // ISO-8859-16
|
||||
case latinCyrillic // ISO-8859-5
|
||||
case latin5 // ISO-8859-9 (Turkish)
|
||||
case latin9 // ISO-8859-15
|
||||
|
||||
@ -85,6 +85,26 @@ extension PureXML.Parsing.ByteDecoder {
|
||||
upperHalf(byte, latin7Upper)
|
||||
}
|
||||
|
||||
static func iso8859_6(_ byte: UInt8) -> Unicode.Scalar {
|
||||
upperHalf(byte, arabicUpper)
|
||||
}
|
||||
|
||||
static func iso8859_8(_ byte: UInt8) -> Unicode.Scalar {
|
||||
upperHalf(byte, hebrewUpper)
|
||||
}
|
||||
|
||||
static func iso8859_10(_ byte: UInt8) -> Unicode.Scalar {
|
||||
upperHalf(byte, latin6Upper)
|
||||
}
|
||||
|
||||
static func iso8859_14(_ byte: UInt8) -> Unicode.Scalar {
|
||||
upperHalf(byte, latin8Upper)
|
||||
}
|
||||
|
||||
static func iso8859_16(_ byte: UInt8) -> Unicode.Scalar {
|
||||
upperHalf(byte, latin10Upper)
|
||||
}
|
||||
|
||||
/// Maps a byte through a 96-entry upper-half table (`0xA0`-`0xFF`); bytes
|
||||
/// below `0xA0` are ASCII or C1 controls (identity). Used by the encodings
|
||||
/// whose upper half is vendored from the Unicode mapping files.
|
||||
|
||||
@ -79,4 +79,84 @@ extension PureXML.Parsing.ByteDecoder.SingleByte {
|
||||
0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7,
|
||||
0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x2019,
|
||||
]
|
||||
|
||||
/// ISO-8859-6 (Arabic) upper half, from `8859-6.TXT` (undefined bytes -> FFFD).
|
||||
static let arabicUpper: [UInt16] = [
|
||||
0x00A0, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A4, 0xFFFD, 0xFFFD, 0xFFFD,
|
||||
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x060C, 0x00AD, 0xFFFD, 0xFFFD,
|
||||
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
|
||||
0xFFFD, 0xFFFD, 0xFFFD, 0x061B, 0xFFFD, 0xFFFD, 0xFFFD, 0x061F,
|
||||
0xFFFD, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627,
|
||||
0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F,
|
||||
0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637,
|
||||
0x0638, 0x0639, 0x063A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
|
||||
0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647,
|
||||
0x0648, 0x0649, 0x064A, 0x064B, 0x064C, 0x064D, 0x064E, 0x064F,
|
||||
0x0650, 0x0651, 0x0652, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
|
||||
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
|
||||
]
|
||||
|
||||
/// ISO-8859-8 (Hebrew) upper half, from `8859-8.TXT` (undefined bytes -> FFFD).
|
||||
static let hebrewUpper: [UInt16] = [
|
||||
0x00A0, 0xFFFD, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
|
||||
0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
|
||||
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
|
||||
0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0xFFFD,
|
||||
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
|
||||
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
|
||||
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
|
||||
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2017,
|
||||
0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7,
|
||||
0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF,
|
||||
0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7,
|
||||
0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0x200E, 0x200F, 0xFFFD,
|
||||
]
|
||||
|
||||
/// ISO-8859-10 (Latin-6, Nordic) upper half, from `8859-10.TXT`.
|
||||
static let latin6Upper: [UInt16] = [
|
||||
0x00A0, 0x0104, 0x0112, 0x0122, 0x012A, 0x0128, 0x0136, 0x00A7,
|
||||
0x013B, 0x0110, 0x0160, 0x0166, 0x017D, 0x00AD, 0x016A, 0x014A,
|
||||
0x00B0, 0x0105, 0x0113, 0x0123, 0x012B, 0x0129, 0x0137, 0x00B7,
|
||||
0x013C, 0x0111, 0x0161, 0x0167, 0x017E, 0x2015, 0x016B, 0x014B,
|
||||
0x0100, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x012E,
|
||||
0x010C, 0x00C9, 0x0118, 0x00CB, 0x0116, 0x00CD, 0x00CE, 0x00CF,
|
||||
0x00D0, 0x0145, 0x014C, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x0168,
|
||||
0x00D8, 0x0172, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF,
|
||||
0x0101, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x012F,
|
||||
0x010D, 0x00E9, 0x0119, 0x00EB, 0x0117, 0x00ED, 0x00EE, 0x00EF,
|
||||
0x00F0, 0x0146, 0x014D, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x0169,
|
||||
0x00F8, 0x0173, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x0138,
|
||||
]
|
||||
|
||||
/// ISO-8859-14 (Latin-8, Celtic) upper half, from `8859-14.TXT`.
|
||||
static let latin8Upper: [UInt16] = [
|
||||
0x00A0, 0x1E02, 0x1E03, 0x00A3, 0x010A, 0x010B, 0x1E0A, 0x00A7,
|
||||
0x1E80, 0x00A9, 0x1E82, 0x1E0B, 0x1EF2, 0x00AD, 0x00AE, 0x0178,
|
||||
0x1E1E, 0x1E1F, 0x0120, 0x0121, 0x1E40, 0x1E41, 0x00B6, 0x1E56,
|
||||
0x1E81, 0x1E57, 0x1E83, 0x1E60, 0x1EF3, 0x1E84, 0x1E85, 0x1E61,
|
||||
0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
|
||||
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
|
||||
0x0174, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x1E6A,
|
||||
0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x0176, 0x00DF,
|
||||
0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
|
||||
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
|
||||
0x0175, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x1E6B,
|
||||
0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x0177, 0x00FF,
|
||||
]
|
||||
|
||||
/// ISO-8859-16 (Latin-10, South-Eastern European) upper half, from `8859-16.TXT`.
|
||||
static let latin10Upper: [UInt16] = [
|
||||
0x00A0, 0x0104, 0x0105, 0x0141, 0x20AC, 0x201E, 0x0160, 0x00A7,
|
||||
0x0161, 0x00A9, 0x0218, 0x00AB, 0x0179, 0x00AD, 0x017A, 0x017B,
|
||||
0x00B0, 0x00B1, 0x010C, 0x0142, 0x017D, 0x201D, 0x00B6, 0x00B7,
|
||||
0x017E, 0x010D, 0x0219, 0x00BB, 0x0152, 0x0153, 0x0178, 0x017C,
|
||||
0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0106, 0x00C6, 0x00C7,
|
||||
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
|
||||
0x0110, 0x0143, 0x00D2, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x015A,
|
||||
0x0170, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0118, 0x021A, 0x00DF,
|
||||
0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x0107, 0x00E6, 0x00E7,
|
||||
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
|
||||
0x0111, 0x0144, 0x00F2, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x015B,
|
||||
0x0171, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0119, 0x021B, 0x00FF,
|
||||
]
|
||||
}
|
||||
|
||||
@ -119,6 +119,11 @@ struct EncodingTests {
|
||||
try #expect(decoded("ISO-8859-7", [0xA4]) == "\u{20AC}") // €
|
||||
try #expect(decoded("ISO-8859-13", [0xA8]) == "\u{00D8}") // Ø
|
||||
try #expect(decoded("ISO-8859-13", [0xC0]) == "\u{0104}") // Ą
|
||||
try #expect(decoded("ISO-8859-6", [0xC7]) == "\u{0627}") // ا (Arabic alef)
|
||||
try #expect(decoded("ISO-8859-8", [0xE0]) == "\u{05D0}") // א (Hebrew alef)
|
||||
try #expect(decoded("ISO-8859-10", [0xFF]) == "\u{0138}") // ĸ
|
||||
try #expect(decoded("ISO-8859-14", [0xA1]) == "\u{1E02}") // Ḃ
|
||||
try #expect(decoded("ISO-8859-16", [0xAA]) == "\u{0218}") // Ș
|
||||
}
|
||||
|
||||
@Test("Decodes ISO-8859-5: the Cyrillic block")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user