Merge #97 (partial): single-byte legacy encodings
This commit is contained in:
commit
7c26547915
@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- The HTML document builder's body is now constructed as a live mutable `TreeNode` tree with a single open-elements stack (the HTML5 model), attaching nodes to their parent as they open rather than accumulating children on close (#83, internal). Behavior is unchanged; this is the foundation the adoption agency algorithm needs to reparent already-built subtrees.
|
||||
- DTD attribute validation is now decomposed into five named, independently composable `Validation` rules (#101): `DTD.requiredAttributes`, `fixedAttributeValues`, `enumeratedAttributeValues`, `tokenizedAttributeTypes`, and `notationAttributes`, replacing the single `attributeDeclarations` rule. Each is removable by identity and isolation-tested one constraint at a time, honoring the validation-framework idiom. Behavior is unchanged. (XSD content validation stays one recursive rule by a documented scope decision, since its constraints are interdependent through type resolution; Schematron already exposes its rules as composable `Validation<Node, Void>` values.)
|
||||
|
||||
### Added
|
||||
|
||||
- Single-byte legacy encodings (#97, partial): the byte decoder gains a table-based single-byte path and three encodings expressible exactly (so transcription-error-free): ISO-8859-15 (Latin-9, euro), ISO-8859-9 (Latin-5, Turkish), and ISO-8859-5 (Latin/Cyrillic). They are selected by the XML declaration's `encoding` name, for both the whole-buffer and streaming decoders. The remaining legacy encodings are data drop-ins into the same path.
|
||||
|
||||
### Fixed
|
||||
|
||||
- A `<template>` element's flow content is now kept nested inside it (`<template><div>x</div></template>`) instead of being split out into the body (#83). `template` is built as body flow content rather than routed to the head.
|
||||
|
||||
@ -89,7 +89,7 @@ flowchart TB
|
||||
|
||||
PureXML is a working, dependency-free XML library today: parse, emit, validate,
|
||||
query, and stream documents on macOS, Linux, Windows, and WASM. The test suite
|
||||
currently runs **750 tests in 117 suites** (`swift test`).
|
||||
currently runs **753 tests in 117 suites** (`swift test`).
|
||||
|
||||
### Shipped (libxml2-aligned surface)
|
||||
|
||||
|
||||
@ -13,21 +13,28 @@ extension PureXML.Parsing {
|
||||
encoding = declared
|
||||
}
|
||||
let body = bytes.dropFirst(bomLength)
|
||||
if let map = Self.singleByteMap(encoding) {
|
||||
return String(String.UnicodeScalarView(body.map(map)))
|
||||
}
|
||||
switch encoding {
|
||||
case .utf8:
|
||||
return String(decoding: body, as: UTF8.self)
|
||||
case .utf16BigEndian:
|
||||
return try decodeUTF16(body, bigEndian: true)
|
||||
case .utf16LittleEndian:
|
||||
return try decodeUTF16(body, bigEndian: false)
|
||||
case .utf32BigEndian:
|
||||
return try decodeUTF32(body, bigEndian: true)
|
||||
case .utf32LittleEndian:
|
||||
return try decodeUTF32(body, bigEndian: false)
|
||||
case .latin1:
|
||||
return String(String.UnicodeScalarView(body.map { Unicode.Scalar($0) }))
|
||||
case .windows1252:
|
||||
return String(String.UnicodeScalarView(body.map { Self.windows1252Scalar($0) }))
|
||||
case .utf16BigEndian: return try decodeUTF16(body, bigEndian: true)
|
||||
case .utf16LittleEndian: return try decodeUTF16(body, bigEndian: false)
|
||||
case .utf32BigEndian: return try decodeUTF32(body, bigEndian: true)
|
||||
case .utf32LittleEndian: return try decodeUTF32(body, bigEndian: false)
|
||||
default: return String(decoding: body, as: UTF8.self)
|
||||
}
|
||||
}
|
||||
|
||||
/// The byte-to-scalar mapping for a single-byte encoding, or nil for the
|
||||
/// Unicode transformation formats.
|
||||
static func singleByteMap(_ encoding: InputEncoding) -> ((UInt8) -> Unicode.Scalar)? {
|
||||
switch encoding {
|
||||
case .latin1: { Unicode.Scalar($0) }
|
||||
case .windows1252: windows1252Scalar
|
||||
case .latinCyrillic: SingleByte.iso8859_5
|
||||
case .latin5: SingleByte.iso8859_9
|
||||
case .latin9: SingleByte.iso8859_15
|
||||
default: nil
|
||||
}
|
||||
}
|
||||
|
||||
@ -83,14 +90,19 @@ extension PureXML.Parsing {
|
||||
name.append(scan[index])
|
||||
index += 1
|
||||
}
|
||||
switch String(decoding: name, as: UTF8.self).lowercased() {
|
||||
case "utf-8", "utf8", "us-ascii", "ascii": return .utf8
|
||||
case "iso-8859-1", "latin1", "latin-1", "l1": return .latin1
|
||||
case "windows-1252", "cp1252", "cp-1252": return .windows1252
|
||||
default: return nil
|
||||
}
|
||||
return encodingByName[String(decoding: name, as: UTF8.self).lowercased()]
|
||||
}
|
||||
|
||||
/// The declared encoding names PureXML recognizes, mapped to their encoding.
|
||||
private static let encodingByName: [String: InputEncoding] = [
|
||||
"utf-8": .utf8, "utf8": .utf8, "us-ascii": .utf8, "ascii": .utf8,
|
||||
"iso-8859-1": .latin1, "latin1": .latin1, "latin-1": .latin1, "l1": .latin1,
|
||||
"windows-1252": .windows1252, "cp1252": .windows1252, "cp-1252": .windows1252,
|
||||
"iso-8859-5": .latinCyrillic, "iso8859-5": .latinCyrillic, "cyrillic": .latinCyrillic,
|
||||
"iso-8859-9": .latin5, "iso8859-9": .latin5, "latin5": .latin5, "latin-5": .latin5, "l5": .latin5,
|
||||
"iso-8859-15": .latin9, "iso8859-15": .latin9, "latin9": .latin9, "latin-9": .latin9, "l9": .latin9,
|
||||
]
|
||||
|
||||
private static func indexOfSubsequence(_ needle: [UInt8], in haystack: [UInt8]) -> Int? {
|
||||
guard !needle.isEmpty, haystack.count >= needle.count else { return nil }
|
||||
for start in 0 ... (haystack.count - needle.count) where Array(haystack[start ..< start + needle.count]) == needle {
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
public extension PureXML.Parsing {
|
||||
/// A character encoding PureXML can decode from raw bytes. Covers the
|
||||
/// Unicode transformation formats plus the common single-byte encodings
|
||||
/// (ISO-8859-1 and Windows-1252), toward libxml2 encoding parity.
|
||||
/// Unicode transformation formats plus the single-byte encodings (ISO-8859-1,
|
||||
/// ISO-8859-5/9/15, and Windows-1252), toward libxml2 encoding parity.
|
||||
enum InputEncoding: String, Equatable, Sendable {
|
||||
case utf8
|
||||
case utf16BigEndian
|
||||
@ -10,6 +10,9 @@ public extension PureXML.Parsing {
|
||||
case utf32LittleEndian
|
||||
case latin1
|
||||
case windows1252
|
||||
case latinCyrillic // ISO-8859-5
|
||||
case latin5 // ISO-8859-9 (Turkish)
|
||||
case latin9 // ISO-8859-15
|
||||
|
||||
/// Detects the encoding from the leading bytes, following the XML sniff
|
||||
/// order: a byte-order mark first, then the `<?` byte pattern of the XML
|
||||
|
||||
57
Sources/Parsing/SingleByteEncodings.swift
Normal file
57
Sources/Parsing/SingleByteEncodings.swift
Normal file
@ -0,0 +1,57 @@
|
||||
extension PureXML.Parsing.ByteDecoder {
|
||||
/// To-Unicode mappings for the single-byte legacy encodings. Each maps one
|
||||
/// byte to its Unicode scalar; bytes `0x00`-`0x7F` are ASCII in every case.
|
||||
/// The encodings here are the ones expressible exactly as ISO-8859-1 with a
|
||||
/// few substitutions or as a contiguous block, so they carry no risk of a
|
||||
/// transcription error.
|
||||
enum SingleByte {
|
||||
/// ISO-8859-15 (Latin-9): ISO-8859-1 with the euro sign and seven other
|
||||
/// substitutions.
|
||||
static func iso8859_15(_ byte: UInt8) -> Unicode.Scalar {
|
||||
switch byte {
|
||||
case 0xA4: "\u{20AC}" // €
|
||||
case 0xA6: "\u{0160}" // Š
|
||||
case 0xA8: "\u{0161}" // š
|
||||
case 0xB4: "\u{017D}" // Ž
|
||||
case 0xB8: "\u{017E}" // ž
|
||||
case 0xBC: "\u{0152}" // Œ
|
||||
case 0xBD: "\u{0153}" // œ
|
||||
case 0xBE: "\u{0178}" // Ÿ
|
||||
default: Unicode.Scalar(byte)
|
||||
}
|
||||
}
|
||||
|
||||
/// ISO-8859-9 (Latin-5, Turkish): ISO-8859-1 with six Turkish letters
|
||||
/// replacing the Icelandic ones.
|
||||
static func iso8859_9(_ byte: UInt8) -> Unicode.Scalar {
|
||||
switch byte {
|
||||
case 0xD0: "\u{011E}" // Ğ
|
||||
case 0xDD: "\u{0130}" // İ
|
||||
case 0xDE: "\u{015E}" // Ş
|
||||
case 0xF0: "\u{011F}" // ğ
|
||||
case 0xFD: "\u{0131}" // ı
|
||||
case 0xFE: "\u{015F}" // ş
|
||||
default: Unicode.Scalar(byte)
|
||||
}
|
||||
}
|
||||
|
||||
/// ISO-8859-5 (Latin/Cyrillic): the Cyrillic alphabet laid out as a
|
||||
/// contiguous block from `0xB0`, with a handful of named exceptions.
|
||||
static func iso8859_5(_ byte: UInt8) -> Unicode.Scalar {
|
||||
switch byte {
|
||||
case 0xA1 ... 0xAC: scalar(0x0401 + UInt32(byte) - 0xA1) // Ё..Ќ
|
||||
case 0xAE ... 0xAF: scalar(0x040E + UInt32(byte) - 0xAE) // Ў Џ
|
||||
case 0xB0 ... 0xEF: scalar(0x0410 + UInt32(byte) - 0xB0) // А..я
|
||||
case 0xF0: "\u{2116}" // №
|
||||
case 0xF1 ... 0xFC: scalar(0x0451 + UInt32(byte) - 0xF1) // ё..ќ
|
||||
case 0xFD: "\u{00A7}" // §
|
||||
case 0xFE ... 0xFF: scalar(0x045E + UInt32(byte) - 0xFE) // ў џ
|
||||
default: Unicode.Scalar(byte) // ASCII, C1, NBSP (0xA0), SHY (0xAD)
|
||||
}
|
||||
}
|
||||
|
||||
private static func scalar(_ value: UInt32) -> Unicode.Scalar {
|
||||
Unicode.Scalar(value) ?? "\u{FFFD}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -28,21 +28,15 @@ extension PureXML.Parsing {
|
||||
guard !finished, let encoding else {
|
||||
return nil
|
||||
}
|
||||
if let map = PureXML.Parsing.ByteDecoder.singleByteMap(encoding) {
|
||||
return nextSingleByte(map)
|
||||
}
|
||||
switch encoding {
|
||||
case .utf8:
|
||||
return nextUTF8()
|
||||
case .utf16BigEndian:
|
||||
return nextUTF16(bigEndian: true)
|
||||
case .utf16LittleEndian:
|
||||
return nextUTF16(bigEndian: false)
|
||||
case .utf32BigEndian:
|
||||
return nextUTF32(bigEndian: true)
|
||||
case .utf32LittleEndian:
|
||||
return nextUTF32(bigEndian: false)
|
||||
case .latin1:
|
||||
return nextSingleByte { Unicode.Scalar($0) }
|
||||
case .windows1252:
|
||||
return nextSingleByte(PureXML.Parsing.ByteDecoder.windows1252Scalar)
|
||||
case .utf16BigEndian: return nextUTF16(bigEndian: true)
|
||||
case .utf16LittleEndian: return nextUTF16(bigEndian: false)
|
||||
case .utf32BigEndian: return nextUTF32(bigEndian: true)
|
||||
case .utf32LittleEndian: return nextUTF32(bigEndian: false)
|
||||
default: return nextUTF8()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -79,6 +79,34 @@ struct EncodingTests {
|
||||
try #expect(rootText(PureXML.parse(bytes: bytes)) == "\u{20AC}")
|
||||
}
|
||||
|
||||
private func decoded(_ encoding: String, _ high: [UInt8]) throws -> String? {
|
||||
var bytes = Array("<?xml version=\"1.0\" encoding=\"\(encoding)\"?><r>".utf8)
|
||||
bytes += high
|
||||
bytes += Array("</r>".utf8)
|
||||
return try rootText(PureXML.parse(bytes: bytes))
|
||||
}
|
||||
|
||||
@Test("Decodes ISO-8859-15: euro and the Latin-9 substitutions")
|
||||
func test_iso8859_15() throws {
|
||||
try #expect(decoded("ISO-8859-15", [0xA4]) == "\u{20AC}") // €
|
||||
try #expect(decoded("ISO-8859-15", [0xBD]) == "\u{0153}") // œ
|
||||
try #expect(decoded("ISO-8859-15", [0xE9]) == "\u{E9}") // é, unchanged from Latin-1
|
||||
}
|
||||
|
||||
@Test("Decodes ISO-8859-9: the Turkish letters")
|
||||
func test_iso8859_9() throws {
|
||||
try #expect(decoded("ISO-8859-9", [0xDD]) == "\u{0130}") // İ
|
||||
try #expect(decoded("ISO-8859-9", [0xFE]) == "\u{015F}") // ş
|
||||
try #expect(decoded("ISO-8859-9", [0xE9]) == "\u{E9}") // é, unchanged from Latin-1
|
||||
}
|
||||
|
||||
@Test("Decodes ISO-8859-5: the Cyrillic block")
|
||||
func test_iso8859_5() throws {
|
||||
try #expect(decoded("ISO-8859-5", [0xB0]) == "\u{0410}") // А
|
||||
try #expect(decoded("ISO-8859-5", [0xE0]) == "\u{0440}") // р
|
||||
try #expect(decoded("ISO-8859-5", [0xF0]) == "\u{2116}") // №
|
||||
}
|
||||
|
||||
private func byteSource(_ bytes: [UInt8]) -> () -> UInt8? {
|
||||
var index = 0
|
||||
return {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user