diff --git a/CHANGELOG.md b/CHANGELOG.md index a3349cb..80b73c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Single-byte legacy encodings (#97, partial): the byte decoder gains a table-based single-byte path and the following legacy encodings, selected by the XML declaration's `encoding` name for both the whole-buffer and streaming decoders. ISO-8859-2/3/4/7/13 are vendored verbatim from the authoritative Unicode mapping files; ISO-8859-5/9/15 are derived exactly; Windows-1254 composes from Windows-1252. So far: ISO-8859-2 (Latin-2), ISO-8859-3 (Latin-3), ISO-8859-4 (Latin-4), ISO-8859-5 (Cyrillic), ISO-8859-7 (Greek), ISO-8859-9 (Turkish), ISO-8859-13 (Baltic), ISO-8859-15 (Latin-9), and Windows-1254. The remaining encodings are data drop-ins into the same path. +- Single-byte legacy encodings (#97, partial): the byte decoder gains a table-based single-byte path and the full ISO-8859 family (ISO-8859-2 through 16, excluding the Thai part 11) plus Windows-1254, selected by the XML declaration's `encoding` name for both the whole-buffer and streaming decoders. The table-based encodings are vendored verbatim from the authoritative `unicode.org` mapping files; ISO-8859-5/9/15 are derived exactly; Windows-1254 composes from Windows-1252. The remaining encodings (Windows code pages, KOI8) are data drop-ins into the same path. ### Fixed diff --git a/Sources/Parsing/ByteDecoder.swift b/Sources/Parsing/ByteDecoder.swift index 537c0f8..a7a2d4c 100644 --- a/Sources/Parsing/ByteDecoder.swift +++ b/Sources/Parsing/ByteDecoder.swift @@ -40,6 +40,11 @@ extension PureXML.Parsing { .latin4: SingleByte.iso8859_4, .greek: SingleByte.iso8859_7, .latin7: SingleByte.iso8859_13, + .arabic: SingleByte.iso8859_6, + .hebrew: SingleByte.iso8859_8, + .latin6: SingleByte.iso8859_10, + .latin8: SingleByte.iso8859_14, + .latin10: SingleByte.iso8859_16, .latinCyrillic: SingleByte.iso8859_5, .latin5: SingleByte.iso8859_9, .latin9: SingleByte.iso8859_15, @@ -111,6 +116,11 @@ extension PureXML.Parsing { "iso-8859-4": .latin4, "iso8859-4": .latin4, "latin4": .latin4, "latin-4": .latin4, "l4": .latin4, "iso-8859-7": .greek, "iso8859-7": .greek, "greek": .greek, "iso-8859-13": .latin7, "iso8859-13": .latin7, "latin7": .latin7, "latin-7": .latin7, "l7": .latin7, + "iso-8859-6": .arabic, "iso8859-6": .arabic, "arabic": .arabic, + "iso-8859-8": .hebrew, "iso8859-8": .hebrew, "hebrew": .hebrew, + "iso-8859-10": .latin6, "iso8859-10": .latin6, "latin6": .latin6, "latin-6": .latin6, "l6": .latin6, + "iso-8859-14": .latin8, "iso8859-14": .latin8, "latin8": .latin8, "latin-8": .latin8, "l8": .latin8, + "iso-8859-16": .latin10, "iso8859-16": .latin10, "latin10": .latin10, "latin-10": .latin10, "l10": .latin10, "iso-8859-5": .latinCyrillic, "iso8859-5": .latinCyrillic, "cyrillic": .latinCyrillic, "iso-8859-9": .latin5, "iso8859-9": .latin5, "latin5": .latin5, "latin-5": .latin5, "l5": .latin5, "iso-8859-15": .latin9, "iso8859-15": .latin9, "latin9": .latin9, "latin-9": .latin9, "l9": .latin9, diff --git a/Sources/Parsing/InputEncoding.swift b/Sources/Parsing/InputEncoding.swift index f6e2426..1d593b6 100644 --- a/Sources/Parsing/InputEncoding.swift +++ b/Sources/Parsing/InputEncoding.swift @@ -16,6 +16,11 @@ public extension PureXML.Parsing { case latin4 // ISO-8859-4 case greek // ISO-8859-7 case latin7 // ISO-8859-13 + case arabic // ISO-8859-6 + case hebrew // ISO-8859-8 + case latin6 // ISO-8859-10 + case latin8 // ISO-8859-14 + case latin10 // ISO-8859-16 case latinCyrillic // ISO-8859-5 case latin5 // ISO-8859-9 (Turkish) case latin9 // ISO-8859-15 diff --git a/Sources/Parsing/SingleByteEncodings.swift b/Sources/Parsing/SingleByteEncodings.swift index 54a58e4..0ec810b 100644 --- a/Sources/Parsing/SingleByteEncodings.swift +++ b/Sources/Parsing/SingleByteEncodings.swift @@ -85,6 +85,26 @@ extension PureXML.Parsing.ByteDecoder { upperHalf(byte, latin7Upper) } + static func iso8859_6(_ byte: UInt8) -> Unicode.Scalar { + upperHalf(byte, arabicUpper) + } + + static func iso8859_8(_ byte: UInt8) -> Unicode.Scalar { + upperHalf(byte, hebrewUpper) + } + + static func iso8859_10(_ byte: UInt8) -> Unicode.Scalar { + upperHalf(byte, latin6Upper) + } + + static func iso8859_14(_ byte: UInt8) -> Unicode.Scalar { + upperHalf(byte, latin8Upper) + } + + static func iso8859_16(_ byte: UInt8) -> Unicode.Scalar { + upperHalf(byte, latin10Upper) + } + /// Maps a byte through a 96-entry upper-half table (`0xA0`-`0xFF`); bytes /// below `0xA0` are ASCII or C1 controls (identity). Used by the encodings /// whose upper half is vendored from the Unicode mapping files. diff --git a/Sources/Parsing/SingleByteTables.swift b/Sources/Parsing/SingleByteTables.swift index ed64d51..2ae5d69 100644 --- a/Sources/Parsing/SingleByteTables.swift +++ b/Sources/Parsing/SingleByteTables.swift @@ -79,4 +79,84 @@ extension PureXML.Parsing.ByteDecoder.SingleByte { 0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7, 0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x2019, ] + + /// ISO-8859-6 (Arabic) upper half, from `8859-6.TXT` (undefined bytes -> FFFD). + static let arabicUpper: [UInt16] = [ + 0x00A0, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A4, 0xFFFD, 0xFFFD, 0xFFFD, + 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x060C, 0x00AD, 0xFFFD, 0xFFFD, + 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, + 0xFFFD, 0xFFFD, 0xFFFD, 0x061B, 0xFFFD, 0xFFFD, 0xFFFD, 0x061F, + 0xFFFD, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, + 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, + 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, + 0x0638, 0x0639, 0x063A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, + 0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, + 0x0648, 0x0649, 0x064A, 0x064B, 0x064C, 0x064D, 0x064E, 0x064F, + 0x0650, 0x0651, 0x0652, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, + 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, + ] + + /// ISO-8859-8 (Hebrew) upper half, from `8859-8.TXT` (undefined bytes -> FFFD). + static let hebrewUpper: [UInt16] = [ + 0x00A0, 0xFFFD, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, + 0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, + 0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0xFFFD, + 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, + 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, + 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, + 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2017, + 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, + 0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF, + 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, + 0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0x200E, 0x200F, 0xFFFD, + ] + + /// ISO-8859-10 (Latin-6, Nordic) upper half, from `8859-10.TXT`. + static let latin6Upper: [UInt16] = [ + 0x00A0, 0x0104, 0x0112, 0x0122, 0x012A, 0x0128, 0x0136, 0x00A7, + 0x013B, 0x0110, 0x0160, 0x0166, 0x017D, 0x00AD, 0x016A, 0x014A, + 0x00B0, 0x0105, 0x0113, 0x0123, 0x012B, 0x0129, 0x0137, 0x00B7, + 0x013C, 0x0111, 0x0161, 0x0167, 0x017E, 0x2015, 0x016B, 0x014B, + 0x0100, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x012E, + 0x010C, 0x00C9, 0x0118, 0x00CB, 0x0116, 0x00CD, 0x00CE, 0x00CF, + 0x00D0, 0x0145, 0x014C, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x0168, + 0x00D8, 0x0172, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF, + 0x0101, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x012F, + 0x010D, 0x00E9, 0x0119, 0x00EB, 0x0117, 0x00ED, 0x00EE, 0x00EF, + 0x00F0, 0x0146, 0x014D, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x0169, + 0x00F8, 0x0173, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x0138, + ] + + /// ISO-8859-14 (Latin-8, Celtic) upper half, from `8859-14.TXT`. + static let latin8Upper: [UInt16] = [ + 0x00A0, 0x1E02, 0x1E03, 0x00A3, 0x010A, 0x010B, 0x1E0A, 0x00A7, + 0x1E80, 0x00A9, 0x1E82, 0x1E0B, 0x1EF2, 0x00AD, 0x00AE, 0x0178, + 0x1E1E, 0x1E1F, 0x0120, 0x0121, 0x1E40, 0x1E41, 0x00B6, 0x1E56, + 0x1E81, 0x1E57, 0x1E83, 0x1E60, 0x1EF3, 0x1E84, 0x1E85, 0x1E61, + 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, + 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, + 0x0174, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x1E6A, + 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x0176, 0x00DF, + 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, + 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, + 0x0175, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x1E6B, + 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x0177, 0x00FF, + ] + + /// ISO-8859-16 (Latin-10, South-Eastern European) upper half, from `8859-16.TXT`. + static let latin10Upper: [UInt16] = [ + 0x00A0, 0x0104, 0x0105, 0x0141, 0x20AC, 0x201E, 0x0160, 0x00A7, + 0x0161, 0x00A9, 0x0218, 0x00AB, 0x0179, 0x00AD, 0x017A, 0x017B, + 0x00B0, 0x00B1, 0x010C, 0x0142, 0x017D, 0x201D, 0x00B6, 0x00B7, + 0x017E, 0x010D, 0x0219, 0x00BB, 0x0152, 0x0153, 0x0178, 0x017C, + 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0106, 0x00C6, 0x00C7, + 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, + 0x0110, 0x0143, 0x00D2, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x015A, + 0x0170, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0118, 0x021A, 0x00DF, + 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x0107, 0x00E6, 0x00E7, + 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, + 0x0111, 0x0144, 0x00F2, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x015B, + 0x0171, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0119, 0x021B, 0x00FF, + ] } diff --git a/Tests/EncodingTests.swift b/Tests/EncodingTests.swift index d2d6109..cfffeb3 100644 --- a/Tests/EncodingTests.swift +++ b/Tests/EncodingTests.swift @@ -119,6 +119,11 @@ struct EncodingTests { try #expect(decoded("ISO-8859-7", [0xA4]) == "\u{20AC}") // € try #expect(decoded("ISO-8859-13", [0xA8]) == "\u{00D8}") // Ø try #expect(decoded("ISO-8859-13", [0xC0]) == "\u{0104}") // Ą + try #expect(decoded("ISO-8859-6", [0xC7]) == "\u{0627}") // ا (Arabic alef) + try #expect(decoded("ISO-8859-8", [0xE0]) == "\u{05D0}") // א (Hebrew alef) + try #expect(decoded("ISO-8859-10", [0xFF]) == "\u{0138}") // ĸ + try #expect(decoded("ISO-8859-14", [0xA1]) == "\u{1E02}") // Ḃ + try #expect(decoded("ISO-8859-16", [0xAA]) == "\u{0218}") // Ș } @Test("Decodes ISO-8859-5: the Cyrillic block")