diff --git a/CHANGELOG.md b/CHANGELOG.md index 783f3fd..8d76f62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Validity-layer conformance, first Sun burn-down (#121, #124): tokenized attribute values are normalized per 3.3.3 before validity checks (whitespace runs collapse, outer whitespace strips, CDATA untouched), so ` nonce ` satisfies an enumeration and an entity-expanded IDREF resolves; an external parsed entity's leading text declaration is stripped per 4.3.1 (and validated: version optional, encoding required, standalone forbidden) instead of being included in the replacement text; character references in parameter-entity literals are expanded at declaration time per 4.4.5, making the spec's Appendix D `%zz;` example parse; and a PI target inside the DTD subset must be separated from its data (production 16). Sun baselines tighten to not-wf 55/56, valid 26/28, invalid 43 reporting; the two remaining valid failures share one cause, filed separately: general-entity replacement text containing markup is included as character data rather than reparsed into elements. - W3C xmlconf Sun section runner (#121, #124): the first validity-layer runner. Sun's manifests classify cases three ways and the runner checks all three: not-wf rejected by the strict parser (54/56), valid documents parse AND DTD-validate cleanly (19/28), invalid documents parse but must report at least one validity error (43/74). The exact deviation baselines are carried in the test with their causes: tokenized attribute-value normalization (3.3.3), external-entity text declarations (4.3.1), declaration-time character-reference expansion in PE literals (4.4.5), the standalone VCs, ID/IDREF corners, #REQUIRED enforcement, per-type attribute VCs, and the root/DOCTYPE name match, the work list this issue burns down. - W3C xmlconf OASIS/NIST section, fully clean (#121, #123): a second opt-in runner (XMLCONF_ROOT-gated, suite never vendored) drives the oasis section from its own manifest, parsed with PureXML itself; all 247 not-wf cases are rejected and all 100 well-formed (valid + invalid) cases parse, with zero deviations (the one excluded case is flagged NAMESPACE='no' by the manifest itself: pre-namespace colons in names, which a namespace-aware parser correctly refuses). Closing the section's last 24 gaps made the DTD subset grammar strict: unknown, lowercase, or space-separated markup-declaration keywords are rejected; a parameter-entity reference must be '%' Name ';' exactly; entity, parameter-entity, and notation names must start with a NameStartChar; NDATA requires whitespace and a strict name; a notation declaration requires its identifier and a clean tail; entity value literals admit no bare '%' (production 9); conditional-section keywords are case-sensitive INCLUDE/IGNORE and sections must balance to their ']]>'; junk between declarations is not well-formed; a grammar violation in the external subset now rejects the document (previously swallowed); the external subset may open with a text declaration (version optional, encoding required, standalone forbidden, production 77); and a processing-instruction target must be separated from its data by whitespace (production 16). The new strictness also tightened the xmltest baseline from 180/186 to 181/186: a CDATA section in the internal subset (case 107) is now correctly rejected by the conditional-keyword check, leaving five documented deviations. diff --git a/README.md b/README.md index 66df82d..b1e46b3 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ flowchart TB PureXML is a working, dependency-free XML library today: parse, emit, validate, query, and stream documents on macOS, Linux, Windows, and WASM. The test suite -currently runs **866 tests in 145 suites** (`swift test`). +currently runs **869 tests in 145 suites** (`swift test`). ### Shipped (libxml2-aligned surface) diff --git a/Sources/Parsing/DoctypeScanner.swift b/Sources/Parsing/DoctypeScanner.swift index 2928bb1..049259e 100644 --- a/Sources/Parsing/DoctypeScanner.swift +++ b/Sources/Parsing/DoctypeScanner.swift @@ -75,7 +75,9 @@ struct DTDScanner { private mutating func resolveExternalEntities() { for (name, id) in doctype.externalEntities where doctype.entities[name] == nil { if let text = resolver.resolveEntity(name, id) { - doctype.entities[name] = text + // An external parsed entity may begin with a text declaration, + // which is not part of its replacement text (4.3.1). + doctype.entities[name] = strippingTextDeclaration(text) } } } @@ -137,13 +139,17 @@ struct DTDScanner { } else if reader.matches("") } else if reader.matches("") { + throw ParseError.unexpectedCharacter(next, reader.mark) + } skip(&reader, until: "?>") } else { // Any other construct here (an unknown or lowercase declaration @@ -188,7 +194,7 @@ struct DTDScanner { } } else if isParameter { if doctype.parameterEntities[name] == nil, let text = resolver.resolveExternalSubset(id) { - doctype.parameterEntities[name] = expandParameterReferences(text) + doctype.parameterEntities[name] = expandParameterReferences(strippingTextDeclaration(text)) } } else if doctype.externalEntities[name] == nil { doctype.externalEntities[name] = id diff --git a/Sources/Parsing/DoctypeScannerStrict.swift b/Sources/Parsing/DoctypeScannerStrict.swift index e042b22..ebde043 100644 --- a/Sources/Parsing/DoctypeScannerStrict.swift +++ b/Sources/Parsing/DoctypeScannerStrict.swift @@ -37,10 +37,21 @@ extension DTDScanner { throw ParseError.invalidEntityDeclaration(mark) } skip(&reader, until: ">") - // The replacement-text well-formedness constraint binds at reference - // time (an unreferenced entity may carry a bad value), so the reparse - // happens in EntityDecoder when the entity is first included. - storeEntity(name: name, value: expandParameterReferences(value), isParameter: isParameter) + // Character references in a parameter-entity literal are expanded when + // the declaration is parsed (4.4.5; the Appendix D `%zz;` case), so + // a stored PE value can itself carry a usable `%name;`. General-entity + // values keep their character references; the reference-time decoder + // expands them. The replacement-text well-formedness constraint binds + // at reference time (an unreferenced entity may carry a bad value), so + // the reparse happens in EntityDecoder when the entity is included. + var replacement = value + if isParameter { + guard let expanded = PureXML.Parsing.EntityReplacementGrammar.expandCharacterReferences(value) else { + throw ParseError.invalidEntityDeclaration(mark) + } + replacement = expanded + } + storeEntity(name: name, value: expandParameterReferences(replacement), isParameter: isParameter) } /// The tail of an external entity: the identifier, an optional `S NDATA Name` @@ -111,6 +122,28 @@ extension DTDScanner { return scanName(&reader) } + /// Removes a leading text declaration from an external entity's resolved + /// text (4.3.1): the declaration is not part of the replacement text. Text + /// without one passes through unchanged, including text whose ` String { + guard text.hasPrefix("'. + var close = text.index(text.startIndex, offsetBy: 5) + while close < text.endIndex { + if text[close] == "?", text.index(after: close) < text.endIndex, text[text.index(after: close)] == ">" { + break + } + close = text.index(after: close) + } + guard close < text.endIndex else { return text } + let declaration = String(text[text.index(text.startIndex, offsetBy: 5) ..< close]) + guard declaration.first?.isWhitespace == true, + PureXML.Parsing.XMLDeclaration.parseTextDeclaration(declaration) != nil + else { return text } + return String(text[text.index(close, offsetBy: 2)...]) + } + /// Consumes the external subset's optional text declaration /// `` (production 77): unlike the /// document's XML declaration, the version is optional, the encoding is diff --git a/Sources/Parsing/EntityReplacementGrammar.swift b/Sources/Parsing/EntityReplacementGrammar.swift index 8ddddb9..2fcea2c 100644 --- a/Sources/Parsing/EntityReplacementGrammar.swift +++ b/Sources/Parsing/EntityReplacementGrammar.swift @@ -20,7 +20,9 @@ extension PureXML.Parsing { /// One declaration-time pass over the entity value: `&#...;` becomes its /// character; everything else (including `&name;`) is kept verbatim. - private static func expandCharacterReferences(_ value: String) -> String? { + /// Also used by the DTD scanner to expand parameter-entity literals at + /// declaration time (4.4.5). + static func expandCharacterReferences(_ value: String) -> String? { var result = "" let characters = Array(value) var index = 0 diff --git a/Sources/Validation/DTDValidationHelpers.swift b/Sources/Validation/DTDValidationHelpers.swift index 666f909..6cab6f4 100644 --- a/Sources/Validation/DTDValidationHelpers.swift +++ b/Sources/Validation/DTDValidationHelpers.swift @@ -75,12 +75,22 @@ extension PureXML.Validation.DTD { // MARK: Attributes - /// The value supplied for a declaration's attribute on an element, or nil when - /// it is absent. + /// The value supplied for a declaration's attribute on an element, or nil + /// when it is absent, normalized per its declared type: for any non-CDATA + /// type, whitespace runs collapse to a single space and leading/trailing + /// whitespace is stripped (3.3.3), so ` nonce ` satisfies an enumeration + /// listing `nonce` and an IDREF resolves regardless of surrounding space. static func attributeValue(of declaration: PureXML.Validation.AttributeDeclaration, on element: DTDElement) -> String? { - element.attributes.first { + let raw = element.attributes.first { $0.name.description == declaration.name || $0.name.localName == declaration.name }?.value + return raw.map { normalize($0, for: declaration.type) } + } + + /// The 3.3.3 tokenized normalization. CDATA values pass through untouched. + static func normalize(_ value: String, for type: PureXML.Validation.AttributeType) -> String { + if case .cdata = type { return value } + return value.split(whereSeparator: \.isWhitespace).joined(separator: " ") } /// The error when a `#REQUIRED` attribute is absent. @@ -208,9 +218,7 @@ extension PureXML.Validation.DTD { let name = element.name.description guard let declarations = schema.attributes[name] else { return } for declaration in declarations { - guard let value = element.attributes.first(where: { - $0.name.description == declaration.name || $0.name.localName == declaration.name - })?.value else { continue } + guard let value = attributeValue(of: declaration, on: element) else { continue } switch declaration.type { case .id: counts[value, default: 0] += 1 diff --git a/Tests/DTDSubsetStrictnessTests.swift b/Tests/DTDSubsetStrictnessTests.swift index 365460a..6af4b68 100644 --- a/Tests/DTDSubsetStrictnessTests.swift +++ b/Tests/DTDSubsetStrictnessTests.swift @@ -110,4 +110,44 @@ struct DTDSubsetStrictnessTests { #expect(!parses(" junk ]>\n")) #expect(!parses(externalHost, externalSubset: " ]]>")) } + + @Test("A PI target inside the subset must be separated from its data") + func test_subsetPISeparation() { + #expect(!parses("]>\n")) + #expect(parses("]>\n")) + } + + @Test("Character references in a PE literal are expanded at declaration (Appendix D)") + func test_parameterEntityCharacterReferences() { + // The spec's tricky example: %xx; expands to %zz; which declares tricky. + let xml = """ + + + ' > + %xx; + ]> + This sample shows a &tricky; method. + """ + let node = try? PureXML.parse(xml, limits: limits()) + #expect(node != nil) + } + + @Test("An external entity's text declaration is not part of its replacement") + func test_externalEntityTextDeclaration() { + let xml = "]>\n&outside;" + let resolver = PureXML.Parsing.EntityResolver( + resolveEntity: { _, _ in "ok" }, + resolveExternalSubset: { _ in nil }, + ) + let node = try? PureXML.parse(xml, limits: limits(), resolver: resolver) + #expect(node != nil) + // The declaration must be stripped, not included as text. + guard let node, case let .document(children) = node, + case let .element(root)? = children.first(where: { $0.element != nil }) + else { return } + #expect(root.children.allSatisfy { child in + if case let .text(value) = child { value == "ok" } else { false } + }) + } } diff --git a/Tests/W3CSunSuiteTests.swift b/Tests/W3CSunSuiteTests.swift index 1d55184..8aae992 100644 --- a/Tests/W3CSunSuiteTests.swift +++ b/Tests/W3CSunSuiteTests.swift @@ -20,21 +20,16 @@ struct W3CSunSuiteTests { /// not-wf cases this implementation knowingly accepts, the current /// frontier (exact, so any regression is caught): not-sa03 needs the - /// standalone='yes' entity-declared WFC; pi needs target/data separation - /// inside the DTD subset. + /// standalone='yes' entity-declared WFC. private let knownNotWFAccepted: Set = [ - "not-wf/not-sa03.xml", "not-wf/pi.xml", + "not-wf/not-sa03.xml", ] - /// valid cases this implementation knowingly fails: the sa/not-sa cases - /// need tokenized attribute-value normalization (3.3.3) before validity - /// checks; ext01/ext02 need external-entity text declarations (4.3.1); - /// pe02 needs character-reference expansion in parameter-entity literals - /// at declaration time (4.4.5, Appendix D); pe03 is under investigation. + /// valid cases this implementation knowingly fails: both need general- + /// entity replacement text containing markup to be reparsed into elements + /// rather than included as character data (tracked as its own issue). private let knownValidFailures: Set = [ - "valid/sa02.xml", "valid/sa04.xml", "valid/not-sa02.xml", - "valid/not-sa03.xml", "valid/not-sa04.xml", "valid/ext01.xml", - "valid/ext02.xml", "valid/pe02.xml", "valid/pe03.xml", + "valid/pe03.xml", "valid/ext02.xml", ] /// invalid cases where this implementation knowingly reports no validity @@ -46,8 +41,10 @@ struct W3CSunSuiteTests { "invalid/dtd01.xml", "invalid/dtd02.xml", "invalid/el01.xml", "invalid/el04.xml", "invalid/el05.xml", "invalid/id01.xml", "invalid/id03.xml", "invalid/id04.xml", "invalid/id05.xml", - "invalid/not-sa01.xml", "invalid/not-sa04.xml", "invalid/not-sa08.xml", - "invalid/not-sa09.xml", "invalid/not-sa11.xml", "invalid/not-sa13.xml", + "invalid/not-sa01.xml", "invalid/not-sa02.xml", "invalid/not-sa04.xml", + "invalid/not-sa05.xml", "invalid/not-sa06.xml", "invalid/not-sa07.xml", + "invalid/not-sa08.xml", "invalid/not-sa09.xml", "invalid/not-sa10.xml", + "invalid/not-sa11.xml", "invalid/not-sa12.xml", "invalid/not-sa13.xml", "invalid/not-sa14.xml", "invalid/required01.xml", "invalid/required02.xml", "invalid/root.xml", "invalid/attr04.xml", "invalid/attr09.xml", "invalid/attr10.xml", "invalid/attr11.xml", "invalid/attr12.xml", @@ -83,15 +80,18 @@ struct W3CSunSuiteTests { PureXML.Parsing.Limits(allowDoctype: true) } - /// Resolves relative SYSTEM identifiers against the case's own directory. + /// Resolves relative SYSTEM identifiers against the case's own directory, + /// byte-decoding through PureXML's own decoder so UTF-16 entity files work. private func resolver(directory: String) -> PureXML.Parsing.EntityResolver { - PureXML.Parsing.EntityResolver( - resolveEntity: { _, id in - (try? String(contentsOfFile: directory + "/" + id.systemID, encoding: .utf8)) - }, - resolveExternalSubset: { id in - (try? String(contentsOfFile: directory + "/" + id.systemID, encoding: .utf8)) - }, + @Sendable func load(_ id: PureXML.Parsing.ExternalID) -> String? { + guard let data = try? Data(contentsOf: URL(fileURLWithPath: directory + "/" + id.systemID)) else { + return nil + } + return try? PureXML.Parsing.ByteDecoder.decode([UInt8](data)) + } + return PureXML.Parsing.EntityResolver( + resolveEntity: { _, id in load(id) }, + resolveExternalSubset: { id in load(id) }, ) }