Merge #97: ISO-8859-3/4/7/13
This commit is contained in:
commit
43c7ecc5e3
@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Added
|
||||
|
||||
- Single-byte legacy encodings (#97, partial): the byte decoder gains a table-based single-byte path and five legacy encodings: ISO-8859-15 (Latin-9, euro), ISO-8859-9 (Latin-5, Turkish), ISO-8859-5 (Latin/Cyrillic), Windows-1254 (Turkish, composed from Windows-1252), and ISO-8859-2 (Latin-2, vendored verbatim from the authoritative Unicode mapping file). They are selected by the XML declaration's `encoding` name, for both the whole-buffer and streaming decoders. The remaining legacy encodings are data drop-ins into the same path.
|
||||
- Single-byte legacy encodings (#97, partial): the byte decoder gains a table-based single-byte path and the following legacy encodings, selected by the XML declaration's `encoding` name for both the whole-buffer and streaming decoders. ISO-8859-2/3/4/7/13 are vendored verbatim from the authoritative Unicode mapping files; ISO-8859-5/9/15 are derived exactly; Windows-1254 composes from Windows-1252. So far: ISO-8859-2 (Latin-2), ISO-8859-3 (Latin-3), ISO-8859-4 (Latin-4), ISO-8859-5 (Cyrillic), ISO-8859-7 (Greek), ISO-8859-9 (Turkish), ISO-8859-13 (Baltic), ISO-8859-15 (Latin-9), and Windows-1254. The remaining encodings are data drop-ins into the same path.
|
||||
|
||||
### Fixed
|
||||
|
||||
|
||||
@ -89,7 +89,7 @@ flowchart TB
|
||||
|
||||
PureXML is a working, dependency-free XML library today: parse, emit, validate,
|
||||
query, and stream documents on macOS, Linux, Windows, and WASM. The test suite
|
||||
currently runs **755 tests in 117 suites** (`swift test`).
|
||||
currently runs **756 tests in 117 suites** (`swift test`).
|
||||
|
||||
### Shipped (libxml2-aligned surface)
|
||||
|
||||
|
||||
@ -28,18 +28,23 @@ extension PureXML.Parsing {
|
||||
/// The byte-to-scalar mapping for a single-byte encoding, or nil for the
|
||||
/// Unicode transformation formats.
|
||||
static func singleByteMap(_ encoding: InputEncoding) -> ((UInt8) -> Unicode.Scalar)? {
|
||||
switch encoding {
|
||||
case .latin1: { Unicode.Scalar($0) }
|
||||
case .windows1252: windows1252Scalar
|
||||
case .windows1254: SingleByte.windows1254
|
||||
case .latin2: SingleByte.iso8859_2
|
||||
case .latinCyrillic: SingleByte.iso8859_5
|
||||
case .latin5: SingleByte.iso8859_9
|
||||
case .latin9: SingleByte.iso8859_15
|
||||
default: nil
|
||||
}
|
||||
singleByteMaps[encoding]
|
||||
}
|
||||
|
||||
private static let singleByteMaps: [InputEncoding: @Sendable (UInt8) -> Unicode.Scalar] = [
|
||||
.latin1: { Unicode.Scalar($0) },
|
||||
.windows1252: windows1252Scalar,
|
||||
.windows1254: SingleByte.windows1254,
|
||||
.latin2: SingleByte.iso8859_2,
|
||||
.latin3: SingleByte.iso8859_3,
|
||||
.latin4: SingleByte.iso8859_4,
|
||||
.greek: SingleByte.iso8859_7,
|
||||
.latin7: SingleByte.iso8859_13,
|
||||
.latinCyrillic: SingleByte.iso8859_5,
|
||||
.latin5: SingleByte.iso8859_9,
|
||||
.latin9: SingleByte.iso8859_15,
|
||||
]
|
||||
|
||||
private static func decodeUTF16(_ bytes: ArraySlice<UInt8>, bigEndian: Bool) throws -> String {
|
||||
guard bytes.count.isMultiple(of: 2) else {
|
||||
throw PureXML.Parsing.ParseError.malformedEncoding
|
||||
@ -102,6 +107,10 @@ extension PureXML.Parsing {
|
||||
"windows-1252": .windows1252, "cp1252": .windows1252, "cp-1252": .windows1252,
|
||||
"windows-1254": .windows1254, "cp1254": .windows1254,
|
||||
"iso-8859-2": .latin2, "iso8859-2": .latin2, "latin2": .latin2, "latin-2": .latin2, "l2": .latin2,
|
||||
"iso-8859-3": .latin3, "iso8859-3": .latin3, "latin3": .latin3, "latin-3": .latin3, "l3": .latin3,
|
||||
"iso-8859-4": .latin4, "iso8859-4": .latin4, "latin4": .latin4, "latin-4": .latin4, "l4": .latin4,
|
||||
"iso-8859-7": .greek, "iso8859-7": .greek, "greek": .greek,
|
||||
"iso-8859-13": .latin7, "iso8859-13": .latin7, "latin7": .latin7, "latin-7": .latin7, "l7": .latin7,
|
||||
"iso-8859-5": .latinCyrillic, "iso8859-5": .latinCyrillic, "cyrillic": .latinCyrillic,
|
||||
"iso-8859-9": .latin5, "iso8859-9": .latin5, "latin5": .latin5, "latin-5": .latin5, "l5": .latin5,
|
||||
"iso-8859-15": .latin9, "iso8859-15": .latin9, "latin9": .latin9, "latin-9": .latin9, "l9": .latin9,
|
||||
|
||||
@ -12,6 +12,10 @@ public extension PureXML.Parsing {
|
||||
case windows1252
|
||||
case windows1254 // Turkish
|
||||
case latin2 // ISO-8859-2
|
||||
case latin3 // ISO-8859-3
|
||||
case latin4 // ISO-8859-4
|
||||
case greek // ISO-8859-7
|
||||
case latin7 // ISO-8859-13
|
||||
case latinCyrillic // ISO-8859-5
|
||||
case latin5 // ISO-8859-9 (Turkish)
|
||||
case latin9 // ISO-8859-15
|
||||
|
||||
@ -64,12 +64,27 @@ extension PureXML.Parsing.ByteDecoder {
|
||||
}
|
||||
}
|
||||
|
||||
/// ISO-8859-2 (Latin-2, Central European). Vendored from the authoritative
|
||||
/// Unicode mapping file.
|
||||
/// The table-driven encodings, vendored from the Unicode mapping files.
|
||||
static func iso8859_2(_ byte: UInt8) -> Unicode.Scalar {
|
||||
upperHalf(byte, latin2Upper)
|
||||
}
|
||||
|
||||
static func iso8859_3(_ byte: UInt8) -> Unicode.Scalar {
|
||||
upperHalf(byte, latin3Upper)
|
||||
}
|
||||
|
||||
static func iso8859_4(_ byte: UInt8) -> Unicode.Scalar {
|
||||
upperHalf(byte, latin4Upper)
|
||||
}
|
||||
|
||||
static func iso8859_7(_ byte: UInt8) -> Unicode.Scalar {
|
||||
upperHalf(byte, greekUpper)
|
||||
}
|
||||
|
||||
static func iso8859_13(_ byte: UInt8) -> Unicode.Scalar {
|
||||
upperHalf(byte, latin7Upper)
|
||||
}
|
||||
|
||||
/// Maps a byte through a 96-entry upper-half table (`0xA0`-`0xFF`); bytes
|
||||
/// below `0xA0` are ASCII or C1 controls (identity). Used by the encodings
|
||||
/// whose upper half is vendored from the Unicode mapping files.
|
||||
|
||||
@ -15,4 +15,68 @@ extension PureXML.Parsing.ByteDecoder.SingleByte {
|
||||
0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7,
|
||||
0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, 0x0163, 0x02D9,
|
||||
]
|
||||
|
||||
/// ISO-8859-3 (Latin-3) upper half, from `8859-3.TXT` (undefined bytes -> FFFD).
|
||||
static let latin3Upper: [UInt16] = [
|
||||
0x00A0, 0x0126, 0x02D8, 0x00A3, 0x00A4, 0xFFFD, 0x0124, 0x00A7,
|
||||
0x00A8, 0x0130, 0x015E, 0x011E, 0x0134, 0x00AD, 0xFFFD, 0x017B,
|
||||
0x00B0, 0x0127, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x0125, 0x00B7,
|
||||
0x00B8, 0x0131, 0x015F, 0x011F, 0x0135, 0x00BD, 0xFFFD, 0x017C,
|
||||
0x00C0, 0x00C1, 0x00C2, 0xFFFD, 0x00C4, 0x010A, 0x0108, 0x00C7,
|
||||
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
|
||||
0xFFFD, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x0120, 0x00D6, 0x00D7,
|
||||
0x011C, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x016C, 0x015C, 0x00DF,
|
||||
0x00E0, 0x00E1, 0x00E2, 0xFFFD, 0x00E4, 0x010B, 0x0109, 0x00E7,
|
||||
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
|
||||
0xFFFD, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x0121, 0x00F6, 0x00F7,
|
||||
0x011D, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x016D, 0x015D, 0x02D9,
|
||||
]
|
||||
|
||||
/// ISO-8859-4 (Latin-4) upper half, from `8859-4.TXT`.
|
||||
static let latin4Upper: [UInt16] = [
|
||||
0x00A0, 0x0104, 0x0138, 0x0156, 0x00A4, 0x0128, 0x013B, 0x00A7,
|
||||
0x00A8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00AD, 0x017D, 0x00AF,
|
||||
0x00B0, 0x0105, 0x02DB, 0x0157, 0x00B4, 0x0129, 0x013C, 0x02C7,
|
||||
0x00B8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014A, 0x017E, 0x014B,
|
||||
0x0100, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x012E,
|
||||
0x010C, 0x00C9, 0x0118, 0x00CB, 0x0116, 0x00CD, 0x00CE, 0x012A,
|
||||
0x0110, 0x0145, 0x014C, 0x0136, 0x00D4, 0x00D5, 0x00D6, 0x00D7,
|
||||
0x00D8, 0x0172, 0x00DA, 0x00DB, 0x00DC, 0x0168, 0x016A, 0x00DF,
|
||||
0x0101, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x012F,
|
||||
0x010D, 0x00E9, 0x0119, 0x00EB, 0x0117, 0x00ED, 0x00EE, 0x012B,
|
||||
0x0111, 0x0146, 0x014D, 0x0137, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
|
||||
0x00F8, 0x0173, 0x00FA, 0x00FB, 0x00FC, 0x0169, 0x016B, 0x02D9,
|
||||
]
|
||||
|
||||
/// ISO-8859-7 (Greek) upper half, from `8859-7.TXT` (undefined bytes -> FFFD).
|
||||
static let greekUpper: [UInt16] = [
|
||||
0x00A0, 0x2018, 0x2019, 0x00A3, 0x20AC, 0x20AF, 0x00A6, 0x00A7,
|
||||
0x00A8, 0x00A9, 0x037A, 0x00AB, 0x00AC, 0x00AD, 0xFFFD, 0x2015,
|
||||
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x0385, 0x0386, 0x00B7,
|
||||
0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F,
|
||||
0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397,
|
||||
0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F,
|
||||
0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7,
|
||||
0x03A8, 0x03A9, 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF,
|
||||
0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7,
|
||||
0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF,
|
||||
0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7,
|
||||
0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, 0x03CE, 0xFFFD,
|
||||
]
|
||||
|
||||
/// ISO-8859-13 (Latin-7, Baltic) upper half, from `8859-13.TXT`.
|
||||
static let latin7Upper: [UInt16] = [
|
||||
0x00A0, 0x201D, 0x00A2, 0x00A3, 0x00A4, 0x201E, 0x00A6, 0x00A7,
|
||||
0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6,
|
||||
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x201C, 0x00B5, 0x00B6, 0x00B7,
|
||||
0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6,
|
||||
0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112,
|
||||
0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B,
|
||||
0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7,
|
||||
0x0172, 0x0141, 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF,
|
||||
0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113,
|
||||
0x010D, 0x00E9, 0x017A, 0x0117, 0x0123, 0x0137, 0x012B, 0x013C,
|
||||
0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7,
|
||||
0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x2019,
|
||||
]
|
||||
}
|
||||
|
||||
@ -108,6 +108,19 @@ struct EncodingTests {
|
||||
try #expect(decoded("ISO-8859-2", [0xFF]) == "\u{02D9}") // ˙
|
||||
}
|
||||
|
||||
@Test("Decodes the vendored ISO-8859 Latin and Greek tables")
|
||||
func test_vendoredTables() throws {
|
||||
try #expect(decoded("ISO-8859-3", [0xA1]) == "\u{0126}") // Ħ
|
||||
try #expect(decoded("ISO-8859-3", [0xC6]) == "\u{0108}") // Ĉ
|
||||
try #expect(decoded("ISO-8859-4", [0xA1]) == "\u{0104}") // Ą
|
||||
try #expect(decoded("ISO-8859-4", [0xC0]) == "\u{0100}") // Ā
|
||||
try #expect(decoded("ISO-8859-7", [0xC1]) == "\u{0391}") // Α
|
||||
try #expect(decoded("ISO-8859-7", [0xE1]) == "\u{03B1}") // α
|
||||
try #expect(decoded("ISO-8859-7", [0xA4]) == "\u{20AC}") // €
|
||||
try #expect(decoded("ISO-8859-13", [0xA8]) == "\u{00D8}") // Ø
|
||||
try #expect(decoded("ISO-8859-13", [0xC0]) == "\u{0104}") // Ą
|
||||
}
|
||||
|
||||
@Test("Decodes ISO-8859-5: the Cyrillic block")
|
||||
func test_iso8859_5() throws {
|
||||
try #expect(decoded("ISO-8859-5", [0xB0]) == "\u{0410}") // А
|
||||
|
||||
Loading…
Reference in New Issue
Block a user