From 43dc7b00c41393d977d3836dab6fe8851302f7fa Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Tue, 9 Jun 2026 07:15:35 +0200 Subject: [PATCH] feat(parsing): ISO-8859-3/4/7/13 vendored from Unicode mappings (#97) --- CHANGELOG.md | 2 +- README.md | 2 +- Sources/Parsing/ByteDecoder.swift | 29 ++++++---- Sources/Parsing/InputEncoding.swift | 4 ++ Sources/Parsing/SingleByteEncodings.swift | 19 ++++++- Sources/Parsing/SingleByteTables.swift | 64 +++++++++++++++++++++++ Tests/EncodingTests.swift | 13 +++++ 7 files changed, 119 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c04bb21..a3349cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Single-byte legacy encodings (#97, partial): the byte decoder gains a table-based single-byte path and five legacy encodings: ISO-8859-15 (Latin-9, euro), ISO-8859-9 (Latin-5, Turkish), ISO-8859-5 (Latin/Cyrillic), Windows-1254 (Turkish, composed from Windows-1252), and ISO-8859-2 (Latin-2, vendored verbatim from the authoritative Unicode mapping file). They are selected by the XML declaration's `encoding` name, for both the whole-buffer and streaming decoders. The remaining legacy encodings are data drop-ins into the same path. +- Single-byte legacy encodings (#97, partial): the byte decoder gains a table-based single-byte path and the following legacy encodings, selected by the XML declaration's `encoding` name for both the whole-buffer and streaming decoders. ISO-8859-2/3/4/7/13 are vendored verbatim from the authoritative Unicode mapping files; ISO-8859-5/9/15 are derived exactly; Windows-1254 composes from Windows-1252. So far: ISO-8859-2 (Latin-2), ISO-8859-3 (Latin-3), ISO-8859-4 (Latin-4), ISO-8859-5 (Cyrillic), ISO-8859-7 (Greek), ISO-8859-9 (Turkish), ISO-8859-13 (Baltic), ISO-8859-15 (Latin-9), and Windows-1254. The remaining encodings are data drop-ins into the same path. ### Fixed diff --git a/README.md b/README.md index bd7e9be..c477e4d 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ flowchart TB PureXML is a working, dependency-free XML library today: parse, emit, validate, query, and stream documents on macOS, Linux, Windows, and WASM. The test suite -currently runs **755 tests in 117 suites** (`swift test`). +currently runs **756 tests in 117 suites** (`swift test`). ### Shipped (libxml2-aligned surface) diff --git a/Sources/Parsing/ByteDecoder.swift b/Sources/Parsing/ByteDecoder.swift index ce695f3..537c0f8 100644 --- a/Sources/Parsing/ByteDecoder.swift +++ b/Sources/Parsing/ByteDecoder.swift @@ -28,18 +28,23 @@ extension PureXML.Parsing { /// The byte-to-scalar mapping for a single-byte encoding, or nil for the /// Unicode transformation formats. static func singleByteMap(_ encoding: InputEncoding) -> ((UInt8) -> Unicode.Scalar)? { - switch encoding { - case .latin1: { Unicode.Scalar($0) } - case .windows1252: windows1252Scalar - case .windows1254: SingleByte.windows1254 - case .latin2: SingleByte.iso8859_2 - case .latinCyrillic: SingleByte.iso8859_5 - case .latin5: SingleByte.iso8859_9 - case .latin9: SingleByte.iso8859_15 - default: nil - } + singleByteMaps[encoding] } + private static let singleByteMaps: [InputEncoding: @Sendable (UInt8) -> Unicode.Scalar] = [ + .latin1: { Unicode.Scalar($0) }, + .windows1252: windows1252Scalar, + .windows1254: SingleByte.windows1254, + .latin2: SingleByte.iso8859_2, + .latin3: SingleByte.iso8859_3, + .latin4: SingleByte.iso8859_4, + .greek: SingleByte.iso8859_7, + .latin7: SingleByte.iso8859_13, + .latinCyrillic: SingleByte.iso8859_5, + .latin5: SingleByte.iso8859_9, + .latin9: SingleByte.iso8859_15, + ] + private static func decodeUTF16(_ bytes: ArraySlice, bigEndian: Bool) throws -> String { guard bytes.count.isMultiple(of: 2) else { throw PureXML.Parsing.ParseError.malformedEncoding @@ -102,6 +107,10 @@ extension PureXML.Parsing { "windows-1252": .windows1252, "cp1252": .windows1252, "cp-1252": .windows1252, "windows-1254": .windows1254, "cp1254": .windows1254, "iso-8859-2": .latin2, "iso8859-2": .latin2, "latin2": .latin2, "latin-2": .latin2, "l2": .latin2, + "iso-8859-3": .latin3, "iso8859-3": .latin3, "latin3": .latin3, "latin-3": .latin3, "l3": .latin3, + "iso-8859-4": .latin4, "iso8859-4": .latin4, "latin4": .latin4, "latin-4": .latin4, "l4": .latin4, + "iso-8859-7": .greek, "iso8859-7": .greek, "greek": .greek, + "iso-8859-13": .latin7, "iso8859-13": .latin7, "latin7": .latin7, "latin-7": .latin7, "l7": .latin7, "iso-8859-5": .latinCyrillic, "iso8859-5": .latinCyrillic, "cyrillic": .latinCyrillic, "iso-8859-9": .latin5, "iso8859-9": .latin5, "latin5": .latin5, "latin-5": .latin5, "l5": .latin5, "iso-8859-15": .latin9, "iso8859-15": .latin9, "latin9": .latin9, "latin-9": .latin9, "l9": .latin9, diff --git a/Sources/Parsing/InputEncoding.swift b/Sources/Parsing/InputEncoding.swift index 7516dd8..f6e2426 100644 --- a/Sources/Parsing/InputEncoding.swift +++ b/Sources/Parsing/InputEncoding.swift @@ -12,6 +12,10 @@ public extension PureXML.Parsing { case windows1252 case windows1254 // Turkish case latin2 // ISO-8859-2 + case latin3 // ISO-8859-3 + case latin4 // ISO-8859-4 + case greek // ISO-8859-7 + case latin7 // ISO-8859-13 case latinCyrillic // ISO-8859-5 case latin5 // ISO-8859-9 (Turkish) case latin9 // ISO-8859-15 diff --git a/Sources/Parsing/SingleByteEncodings.swift b/Sources/Parsing/SingleByteEncodings.swift index 6427252..54a58e4 100644 --- a/Sources/Parsing/SingleByteEncodings.swift +++ b/Sources/Parsing/SingleByteEncodings.swift @@ -64,12 +64,27 @@ extension PureXML.Parsing.ByteDecoder { } } - /// ISO-8859-2 (Latin-2, Central European). Vendored from the authoritative - /// Unicode mapping file. + /// The table-driven encodings, vendored from the Unicode mapping files. static func iso8859_2(_ byte: UInt8) -> Unicode.Scalar { upperHalf(byte, latin2Upper) } + static func iso8859_3(_ byte: UInt8) -> Unicode.Scalar { + upperHalf(byte, latin3Upper) + } + + static func iso8859_4(_ byte: UInt8) -> Unicode.Scalar { + upperHalf(byte, latin4Upper) + } + + static func iso8859_7(_ byte: UInt8) -> Unicode.Scalar { + upperHalf(byte, greekUpper) + } + + static func iso8859_13(_ byte: UInt8) -> Unicode.Scalar { + upperHalf(byte, latin7Upper) + } + /// Maps a byte through a 96-entry upper-half table (`0xA0`-`0xFF`); bytes /// below `0xA0` are ASCII or C1 controls (identity). Used by the encodings /// whose upper half is vendored from the Unicode mapping files. diff --git a/Sources/Parsing/SingleByteTables.swift b/Sources/Parsing/SingleByteTables.swift index f4b4d91..ed64d51 100644 --- a/Sources/Parsing/SingleByteTables.swift +++ b/Sources/Parsing/SingleByteTables.swift @@ -15,4 +15,68 @@ extension PureXML.Parsing.ByteDecoder.SingleByte { 0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7, 0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, 0x0163, 0x02D9, ] + + /// ISO-8859-3 (Latin-3) upper half, from `8859-3.TXT` (undefined bytes -> FFFD). + static let latin3Upper: [UInt16] = [ + 0x00A0, 0x0126, 0x02D8, 0x00A3, 0x00A4, 0xFFFD, 0x0124, 0x00A7, + 0x00A8, 0x0130, 0x015E, 0x011E, 0x0134, 0x00AD, 0xFFFD, 0x017B, + 0x00B0, 0x0127, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x0125, 0x00B7, + 0x00B8, 0x0131, 0x015F, 0x011F, 0x0135, 0x00BD, 0xFFFD, 0x017C, + 0x00C0, 0x00C1, 0x00C2, 0xFFFD, 0x00C4, 0x010A, 0x0108, 0x00C7, + 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, + 0xFFFD, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x0120, 0x00D6, 0x00D7, + 0x011C, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x016C, 0x015C, 0x00DF, + 0x00E0, 0x00E1, 0x00E2, 0xFFFD, 0x00E4, 0x010B, 0x0109, 0x00E7, + 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, + 0xFFFD, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x0121, 0x00F6, 0x00F7, + 0x011D, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x016D, 0x015D, 0x02D9, + ] + + /// ISO-8859-4 (Latin-4) upper half, from `8859-4.TXT`. + static let latin4Upper: [UInt16] = [ + 0x00A0, 0x0104, 0x0138, 0x0156, 0x00A4, 0x0128, 0x013B, 0x00A7, + 0x00A8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00AD, 0x017D, 0x00AF, + 0x00B0, 0x0105, 0x02DB, 0x0157, 0x00B4, 0x0129, 0x013C, 0x02C7, + 0x00B8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014A, 0x017E, 0x014B, + 0x0100, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x012E, + 0x010C, 0x00C9, 0x0118, 0x00CB, 0x0116, 0x00CD, 0x00CE, 0x012A, + 0x0110, 0x0145, 0x014C, 0x0136, 0x00D4, 0x00D5, 0x00D6, 0x00D7, + 0x00D8, 0x0172, 0x00DA, 0x00DB, 0x00DC, 0x0168, 0x016A, 0x00DF, + 0x0101, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x012F, + 0x010D, 0x00E9, 0x0119, 0x00EB, 0x0117, 0x00ED, 0x00EE, 0x012B, + 0x0111, 0x0146, 0x014D, 0x0137, 0x00F4, 0x00F5, 0x00F6, 0x00F7, + 0x00F8, 0x0173, 0x00FA, 0x00FB, 0x00FC, 0x0169, 0x016B, 0x02D9, + ] + + /// ISO-8859-7 (Greek) upper half, from `8859-7.TXT` (undefined bytes -> FFFD). + static let greekUpper: [UInt16] = [ + 0x00A0, 0x2018, 0x2019, 0x00A3, 0x20AC, 0x20AF, 0x00A6, 0x00A7, + 0x00A8, 0x00A9, 0x037A, 0x00AB, 0x00AC, 0x00AD, 0xFFFD, 0x2015, + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x0385, 0x0386, 0x00B7, + 0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F, + 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, + 0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, + 0x03A8, 0x03A9, 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF, + 0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, + 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF, + 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, + 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, 0x03CE, 0xFFFD, + ] + + /// ISO-8859-13 (Latin-7, Baltic) upper half, from `8859-13.TXT`. + static let latin7Upper: [UInt16] = [ + 0x00A0, 0x201D, 0x00A2, 0x00A3, 0x00A4, 0x201E, 0x00A6, 0x00A7, + 0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6, + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x201C, 0x00B5, 0x00B6, 0x00B7, + 0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6, + 0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112, + 0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B, + 0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7, + 0x0172, 0x0141, 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF, + 0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113, + 0x010D, 0x00E9, 0x017A, 0x0117, 0x0123, 0x0137, 0x012B, 0x013C, + 0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7, + 0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x2019, + ] } diff --git a/Tests/EncodingTests.swift b/Tests/EncodingTests.swift index ec13a7a..d2d6109 100644 --- a/Tests/EncodingTests.swift +++ b/Tests/EncodingTests.swift @@ -108,6 +108,19 @@ struct EncodingTests { try #expect(decoded("ISO-8859-2", [0xFF]) == "\u{02D9}") // ˙ } + @Test("Decodes the vendored ISO-8859 Latin and Greek tables") + func test_vendoredTables() throws { + try #expect(decoded("ISO-8859-3", [0xA1]) == "\u{0126}") // Ħ + try #expect(decoded("ISO-8859-3", [0xC6]) == "\u{0108}") // Ĉ + try #expect(decoded("ISO-8859-4", [0xA1]) == "\u{0104}") // Ą + try #expect(decoded("ISO-8859-4", [0xC0]) == "\u{0100}") // Ā + try #expect(decoded("ISO-8859-7", [0xC1]) == "\u{0391}") // Α + try #expect(decoded("ISO-8859-7", [0xE1]) == "\u{03B1}") // α + try #expect(decoded("ISO-8859-7", [0xA4]) == "\u{20AC}") // € + try #expect(decoded("ISO-8859-13", [0xA8]) == "\u{00D8}") // Ø + try #expect(decoded("ISO-8859-13", [0xC0]) == "\u{0104}") // Ą + } + @Test("Decodes ISO-8859-5: the Cyrillic block") func test_iso8859_5() throws { try #expect(decoded("ISO-8859-5", [0xB0]) == "\u{0410}") // А