Merge #97: ISO-8859-3/4/7/13

This commit is contained in:
Mihaela Mihaljevic 2026-06-09 07:15:36 +02:00
commit 43c7ecc5e3
7 changed files with 119 additions and 14 deletions

View File

@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Single-byte legacy encodings (#97, partial): the byte decoder gains a table-based single-byte path and five legacy encodings: ISO-8859-15 (Latin-9, euro), ISO-8859-9 (Latin-5, Turkish), ISO-8859-5 (Latin/Cyrillic), Windows-1254 (Turkish, composed from Windows-1252), and ISO-8859-2 (Latin-2, vendored verbatim from the authoritative Unicode mapping file). They are selected by the XML declaration's `encoding` name, for both the whole-buffer and streaming decoders. The remaining legacy encodings are data drop-ins into the same path.
- Single-byte legacy encodings (#97, partial): the byte decoder gains a table-based single-byte path and the following legacy encodings, selected by the XML declaration's `encoding` name for both the whole-buffer and streaming decoders. ISO-8859-2/3/4/7/13 are vendored verbatim from the authoritative Unicode mapping files; ISO-8859-5/9/15 are derived exactly; Windows-1254 composes from Windows-1252. So far: ISO-8859-2 (Latin-2), ISO-8859-3 (Latin-3), ISO-8859-4 (Latin-4), ISO-8859-5 (Cyrillic), ISO-8859-7 (Greek), ISO-8859-9 (Turkish), ISO-8859-13 (Baltic), ISO-8859-15 (Latin-9), and Windows-1254. The remaining encodings are data drop-ins into the same path.
### Fixed

View File

@ -89,7 +89,7 @@ flowchart TB
PureXML is a working, dependency-free XML library today: parse, emit, validate,
query, and stream documents on macOS, Linux, Windows, and WASM. The test suite
currently runs **755 tests in 117 suites** (`swift test`).
currently runs **756 tests in 117 suites** (`swift test`).
### Shipped (libxml2-aligned surface)

View File

@ -28,18 +28,23 @@ extension PureXML.Parsing {
/// The byte-to-scalar mapping for a single-byte encoding, or nil for the
/// Unicode transformation formats.
static func singleByteMap(_ encoding: InputEncoding) -> ((UInt8) -> Unicode.Scalar)? {
switch encoding {
case .latin1: { Unicode.Scalar($0) }
case .windows1252: windows1252Scalar
case .windows1254: SingleByte.windows1254
case .latin2: SingleByte.iso8859_2
case .latinCyrillic: SingleByte.iso8859_5
case .latin5: SingleByte.iso8859_9
case .latin9: SingleByte.iso8859_15
default: nil
}
singleByteMaps[encoding]
}
private static let singleByteMaps: [InputEncoding: @Sendable (UInt8) -> Unicode.Scalar] = [
.latin1: { Unicode.Scalar($0) },
.windows1252: windows1252Scalar,
.windows1254: SingleByte.windows1254,
.latin2: SingleByte.iso8859_2,
.latin3: SingleByte.iso8859_3,
.latin4: SingleByte.iso8859_4,
.greek: SingleByte.iso8859_7,
.latin7: SingleByte.iso8859_13,
.latinCyrillic: SingleByte.iso8859_5,
.latin5: SingleByte.iso8859_9,
.latin9: SingleByte.iso8859_15,
]
private static func decodeUTF16(_ bytes: ArraySlice<UInt8>, bigEndian: Bool) throws -> String {
guard bytes.count.isMultiple(of: 2) else {
throw PureXML.Parsing.ParseError.malformedEncoding
@ -102,6 +107,10 @@ extension PureXML.Parsing {
"windows-1252": .windows1252, "cp1252": .windows1252, "cp-1252": .windows1252,
"windows-1254": .windows1254, "cp1254": .windows1254,
"iso-8859-2": .latin2, "iso8859-2": .latin2, "latin2": .latin2, "latin-2": .latin2, "l2": .latin2,
"iso-8859-3": .latin3, "iso8859-3": .latin3, "latin3": .latin3, "latin-3": .latin3, "l3": .latin3,
"iso-8859-4": .latin4, "iso8859-4": .latin4, "latin4": .latin4, "latin-4": .latin4, "l4": .latin4,
"iso-8859-7": .greek, "iso8859-7": .greek, "greek": .greek,
"iso-8859-13": .latin7, "iso8859-13": .latin7, "latin7": .latin7, "latin-7": .latin7, "l7": .latin7,
"iso-8859-5": .latinCyrillic, "iso8859-5": .latinCyrillic, "cyrillic": .latinCyrillic,
"iso-8859-9": .latin5, "iso8859-9": .latin5, "latin5": .latin5, "latin-5": .latin5, "l5": .latin5,
"iso-8859-15": .latin9, "iso8859-15": .latin9, "latin9": .latin9, "latin-9": .latin9, "l9": .latin9,

View File

@ -12,6 +12,10 @@ public extension PureXML.Parsing {
case windows1252
case windows1254 // Turkish
case latin2 // ISO-8859-2
case latin3 // ISO-8859-3
case latin4 // ISO-8859-4
case greek // ISO-8859-7
case latin7 // ISO-8859-13
case latinCyrillic // ISO-8859-5
case latin5 // ISO-8859-9 (Turkish)
case latin9 // ISO-8859-15

View File

@ -64,12 +64,27 @@ extension PureXML.Parsing.ByteDecoder {
}
}
/// ISO-8859-2 (Latin-2, Central European). Vendored from the authoritative
/// Unicode mapping file.
/// The table-driven encodings, vendored from the Unicode mapping files.
static func iso8859_2(_ byte: UInt8) -> Unicode.Scalar {
upperHalf(byte, latin2Upper)
}
static func iso8859_3(_ byte: UInt8) -> Unicode.Scalar {
upperHalf(byte, latin3Upper)
}
static func iso8859_4(_ byte: UInt8) -> Unicode.Scalar {
upperHalf(byte, latin4Upper)
}
static func iso8859_7(_ byte: UInt8) -> Unicode.Scalar {
upperHalf(byte, greekUpper)
}
static func iso8859_13(_ byte: UInt8) -> Unicode.Scalar {
upperHalf(byte, latin7Upper)
}
/// Maps a byte through a 96-entry upper-half table (`0xA0`-`0xFF`); bytes
/// below `0xA0` are ASCII or C1 controls (identity). Used by the encodings
/// whose upper half is vendored from the Unicode mapping files.

View File

@ -15,4 +15,68 @@ extension PureXML.Parsing.ByteDecoder.SingleByte {
0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7,
0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, 0x0163, 0x02D9,
]
/// ISO-8859-3 (Latin-3) upper half, from `8859-3.TXT` (undefined bytes -> FFFD).
static let latin3Upper: [UInt16] = [
0x00A0, 0x0126, 0x02D8, 0x00A3, 0x00A4, 0xFFFD, 0x0124, 0x00A7,
0x00A8, 0x0130, 0x015E, 0x011E, 0x0134, 0x00AD, 0xFFFD, 0x017B,
0x00B0, 0x0127, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x0125, 0x00B7,
0x00B8, 0x0131, 0x015F, 0x011F, 0x0135, 0x00BD, 0xFFFD, 0x017C,
0x00C0, 0x00C1, 0x00C2, 0xFFFD, 0x00C4, 0x010A, 0x0108, 0x00C7,
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
0xFFFD, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x0120, 0x00D6, 0x00D7,
0x011C, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x016C, 0x015C, 0x00DF,
0x00E0, 0x00E1, 0x00E2, 0xFFFD, 0x00E4, 0x010B, 0x0109, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
0xFFFD, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x0121, 0x00F6, 0x00F7,
0x011D, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x016D, 0x015D, 0x02D9,
]
/// ISO-8859-4 (Latin-4) upper half, from `8859-4.TXT`.
static let latin4Upper: [UInt16] = [
0x00A0, 0x0104, 0x0138, 0x0156, 0x00A4, 0x0128, 0x013B, 0x00A7,
0x00A8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00AD, 0x017D, 0x00AF,
0x00B0, 0x0105, 0x02DB, 0x0157, 0x00B4, 0x0129, 0x013C, 0x02C7,
0x00B8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014A, 0x017E, 0x014B,
0x0100, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x012E,
0x010C, 0x00C9, 0x0118, 0x00CB, 0x0116, 0x00CD, 0x00CE, 0x012A,
0x0110, 0x0145, 0x014C, 0x0136, 0x00D4, 0x00D5, 0x00D6, 0x00D7,
0x00D8, 0x0172, 0x00DA, 0x00DB, 0x00DC, 0x0168, 0x016A, 0x00DF,
0x0101, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x012F,
0x010D, 0x00E9, 0x0119, 0x00EB, 0x0117, 0x00ED, 0x00EE, 0x012B,
0x0111, 0x0146, 0x014D, 0x0137, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
0x00F8, 0x0173, 0x00FA, 0x00FB, 0x00FC, 0x0169, 0x016B, 0x02D9,
]
/// ISO-8859-7 (Greek) upper half, from `8859-7.TXT` (undefined bytes -> FFFD).
static let greekUpper: [UInt16] = [
0x00A0, 0x2018, 0x2019, 0x00A3, 0x20AC, 0x20AF, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0x037A, 0x00AB, 0x00AC, 0x00AD, 0xFFFD, 0x2015,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x0385, 0x0386, 0x00B7,
0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F,
0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397,
0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F,
0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7,
0x03A8, 0x03A9, 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF,
0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7,
0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF,
0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7,
0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, 0x03CE, 0xFFFD,
]
/// ISO-8859-13 (Latin-7, Baltic) upper half, from `8859-13.TXT`.
static let latin7Upper: [UInt16] = [
0x00A0, 0x201D, 0x00A2, 0x00A3, 0x00A4, 0x201E, 0x00A6, 0x00A7,
0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x201C, 0x00B5, 0x00B6, 0x00B7,
0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6,
0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112,
0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B,
0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7,
0x0172, 0x0141, 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF,
0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113,
0x010D, 0x00E9, 0x017A, 0x0117, 0x0123, 0x0137, 0x012B, 0x013C,
0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7,
0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x2019,
]
}

View File

@ -108,6 +108,19 @@ struct EncodingTests {
try #expect(decoded("ISO-8859-2", [0xFF]) == "\u{02D9}") // ˙
}
@Test("Decodes the vendored ISO-8859 Latin and Greek tables")
func test_vendoredTables() throws {
try #expect(decoded("ISO-8859-3", [0xA1]) == "\u{0126}") // Ħ
try #expect(decoded("ISO-8859-3", [0xC6]) == "\u{0108}") // Ĉ
try #expect(decoded("ISO-8859-4", [0xA1]) == "\u{0104}") // Ą
try #expect(decoded("ISO-8859-4", [0xC0]) == "\u{0100}") // Ā
try #expect(decoded("ISO-8859-7", [0xC1]) == "\u{0391}") // Α
try #expect(decoded("ISO-8859-7", [0xE1]) == "\u{03B1}") // α
try #expect(decoded("ISO-8859-7", [0xA4]) == "\u{20AC}") // €
try #expect(decoded("ISO-8859-13", [0xA8]) == "\u{00D8}") // Ø
try #expect(decoded("ISO-8859-13", [0xC0]) == "\u{0104}") // Ą
}
@Test("Decodes ISO-8859-5: the Cyrillic block")
func test_iso8859_5() throws {
try #expect(decoded("ISO-8859-5", [0xB0]) == "\u{0410}") // А