From add101507e19ca4c10dec79584e6eae50292393f Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Wed, 10 Jun 2026 06:35:24 +0200 Subject: [PATCH] test(conformance): sun section runner with exact validity-layer baselines The first runner to exercise the validity layer: not-wf must be rejected (54/56), valid must parse and DTD-validate clean (19/28), invalid must parse but report at least one validity error (43/74). Deviation baselines are exact and annotated with their causes, the burn-down list for #124: tokenized attribute-value normalization, external-entity text declarations, declaration-time char-ref expansion in PE literals, standalone VCs, ID/IDREF corners, REQUIRED enforcement, per-type attribute VCs, and the root/DOCTYPE name match. --- CHANGELOG.md | 1 + README.md | 4 +- Tests/W3CSunSuiteTests.swift | 180 +++++++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 Tests/W3CSunSuiteTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 4969b32..783f3fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- W3C xmlconf Sun section runner (#121, #124): the first validity-layer runner. Sun's manifests classify cases three ways and the runner checks all three: not-wf rejected by the strict parser (54/56), valid documents parse AND DTD-validate cleanly (19/28), invalid documents parse but must report at least one validity error (43/74). The exact deviation baselines are carried in the test with their causes: tokenized attribute-value normalization (3.3.3), external-entity text declarations (4.3.1), declaration-time character-reference expansion in PE literals (4.4.5), the standalone VCs, ID/IDREF corners, #REQUIRED enforcement, per-type attribute VCs, and the root/DOCTYPE name match, the work list this issue burns down. - W3C xmlconf OASIS/NIST section, fully clean (#121, #123): a second opt-in runner (XMLCONF_ROOT-gated, suite never vendored) drives the oasis section from its own manifest, parsed with PureXML itself; all 247 not-wf cases are rejected and all 100 well-formed (valid + invalid) cases parse, with zero deviations (the one excluded case is flagged NAMESPACE='no' by the manifest itself: pre-namespace colons in names, which a namespace-aware parser correctly refuses). Closing the section's last 24 gaps made the DTD subset grammar strict: unknown, lowercase, or space-separated markup-declaration keywords are rejected; a parameter-entity reference must be '%' Name ';' exactly; entity, parameter-entity, and notation names must start with a NameStartChar; NDATA requires whitespace and a strict name; a notation declaration requires its identifier and a clean tail; entity value literals admit no bare '%' (production 9); conditional-section keywords are case-sensitive INCLUDE/IGNORE and sections must balance to their ']]>'; junk between declarations is not well-formed; a grammar violation in the external subset now rejects the document (previously swallowed); the external subset may open with a text declaration (version optional, encoding required, standalone forbidden, production 77); and a processing-instruction target must be separated from its data by whitespace (production 16). The new strictness also tightened the xmltest baseline from 180/186 to 181/186: a CDATA section in the internal subset (case 107) is now correctly rejected by the conditional-keyword check, leaving five documented deviations. ### Changed diff --git a/README.md b/README.md index a077bdb..66df82d 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ flowchart TB Epic105["#105 parity frontier: complete (#106 #107 #108 output encodings, streaming validation, last encodings; findings #109-#119; first W3C pass #120)"]:::epic Epic121["#121 xmlconf full archive"]:::epic Oasis["#123 OASIS/NIST section: complete, zero deviations"]:::done - Sun["#124 Sun section: DTD-validity baselines"]:::todo + Sun["#124 Sun section: runner + exact baselines in, burning down"]:::review IBM["#125 IBM section: per-production validity depth"]:::todo Japanese["#126 japanese section: encoding declarations"]:::todo Eduni["#127 eduni: errata + namespaces"]:::todo @@ -117,7 +117,7 @@ flowchart TB PureXML is a working, dependency-free XML library today: parse, emit, validate, query, and stream documents on macOS, Linux, Windows, and WASM. The test suite -currently runs **863 tests in 144 suites** (`swift test`). +currently runs **866 tests in 145 suites** (`swift test`). ### Shipped (libxml2-aligned surface) diff --git a/Tests/W3CSunSuiteTests.swift b/Tests/W3CSunSuiteTests.swift new file mode 100644 index 0000000..1d55184 --- /dev/null +++ b/Tests/W3CSunSuiteTests.swift @@ -0,0 +1,180 @@ +import Foundation +@testable import PureXML +import Testing + +/// Opt-in runner for the W3C XML Conformance Test Suite's Sun section, the +/// first section to exercise the validity layer: `valid` documents must parse +/// AND validate cleanly against their DTDs, `invalid` documents are +/// well-formed but must produce at least one validity error, and `not-wf` +/// documents must be rejected by the strict parser. Point `XMLCONF_ROOT` at +/// the extracted `xmlconf` directory to run (the suite is never vendored; its +/// license permits redistribution only as the unmodified archive). The +/// section's manifests are rootless fragments designed for entity inclusion, +/// so they are wrapped in a synthetic root before being parsed with PureXML. +/// Foundation is used here for file access only. +@Suite("W3C sun conformance (opt-in via XMLCONF_ROOT)") +struct W3CSunSuiteTests { + private var root: String? { + ProcessInfo.processInfo.environment["XMLCONF_ROOT"] + } + + /// not-wf cases this implementation knowingly accepts, the current + /// frontier (exact, so any regression is caught): not-sa03 needs the + /// standalone='yes' entity-declared WFC; pi needs target/data separation + /// inside the DTD subset. + private let knownNotWFAccepted: Set = [ + "not-wf/not-sa03.xml", "not-wf/pi.xml", + ] + + /// valid cases this implementation knowingly fails: the sa/not-sa cases + /// need tokenized attribute-value normalization (3.3.3) before validity + /// checks; ext01/ext02 need external-entity text declarations (4.3.1); + /// pe02 needs character-reference expansion in parameter-entity literals + /// at declaration time (4.4.5, Appendix D); pe03 is under investigation. + private let knownValidFailures: Set = [ + "valid/sa02.xml", "valid/sa04.xml", "valid/not-sa02.xml", + "valid/not-sa03.xml", "valid/not-sa04.xml", "valid/ext01.xml", + "valid/ext02.xml", "valid/pe02.xml", "valid/pe03.xml", + ] + + /// invalid cases where this implementation knowingly reports no validity + /// error yet, the validity-layer frontier this issue works down: the + /// standalone VCs (not-sa*), ID/IDREF corners, #REQUIRED enforcement, + /// per-type attribute VCs (attr*), the root-element/DOCTYPE name match, + /// EMPTY content, and UTF-16 cases. + private let knownInvalidAccepted: Set = [ + "invalid/dtd01.xml", "invalid/dtd02.xml", "invalid/el01.xml", + "invalid/el04.xml", "invalid/el05.xml", "invalid/id01.xml", + "invalid/id03.xml", "invalid/id04.xml", "invalid/id05.xml", + "invalid/not-sa01.xml", "invalid/not-sa04.xml", "invalid/not-sa08.xml", + "invalid/not-sa09.xml", "invalid/not-sa11.xml", "invalid/not-sa13.xml", + "invalid/not-sa14.xml", "invalid/required01.xml", "invalid/required02.xml", + "invalid/root.xml", "invalid/attr04.xml", "invalid/attr09.xml", + "invalid/attr10.xml", "invalid/attr11.xml", "invalid/attr12.xml", + "invalid/attr13.xml", "invalid/attr14.xml", "invalid/attr15.xml", + "invalid/attr16.xml", "invalid/utf16b.xml", "invalid/utf16l.xml", + "invalid/empty.xml", + ] + + private struct ManifestCase { + let id: String + let uri: String + } + + /// Reads one manifest fragment (`sun-valid.xml` and friends): the fragment + /// has no single root, so it is wrapped before parsing. + private func manifest(_ file: String, in directory: String) throws -> [ManifestCase] { + var source = try String(contentsOfFile: directory + "/" + file, encoding: .utf8) + if let declarationEnd = source.range(of: "?>") { + source = String(source[declarationEnd.upperBound...]) + } + let document = try PureXML.parse("\(source)") + return try PureXML.xpath("//TEST", over: document).compactMap { selection in + guard let element = selection.element else { return nil } + func attribute(_ name: String) -> String? { + element.attributes.first { $0.name.description == name }?.value + } + guard let id = attribute("ID"), let uri = attribute("URI") else { return nil } + return ManifestCase(id: id, uri: uri) + } + } + + private func limits() -> PureXML.Parsing.Limits { + PureXML.Parsing.Limits(allowDoctype: true) + } + + /// Resolves relative SYSTEM identifiers against the case's own directory. + private func resolver(directory: String) -> PureXML.Parsing.EntityResolver { + PureXML.Parsing.EntityResolver( + resolveEntity: { _, id in + (try? String(contentsOfFile: directory + "/" + id.systemID, encoding: .utf8)) + }, + resolveExternalSubset: { id in + (try? String(contentsOfFile: directory + "/" + id.systemID, encoding: .utf8)) + }, + ) + } + + private func caseDirectory(_ section: String, uri: String) -> String { + guard let slash = uri.lastIndex(of: "/") else { return section } + return section + "/" + String(uri.prefix(upTo: slash)) + } + + private func source(of uri: String, in section: String) throws -> String { + let data = try Data(contentsOf: URL(fileURLWithPath: section + "/" + uri)) + return try PureXML.Parsing.ByteDecoder.decode([UInt8](data)) + } + + @Test("Every sun not-wf case is rejected by the strict parser") + func test_notWellFormedRejected() throws { + guard let root else { + return // Opt-in: suite not present. + } + let section = root + "/sun" + var unexpectedlyAccepted: [String] = [] + for testCase in try manifest("sun-not-wf.xml", in: section) where !knownNotWFAccepted.contains(testCase.uri) { + let directory = caseDirectory(section, uri: testCase.uri) + let accepted = (try? PureXML.parse( + source(of: testCase.uri, in: section), + limits: limits(), + resolver: resolver(directory: directory), + )) != nil + if accepted { + unexpectedlyAccepted.append(testCase.uri) + } + } + #expect(unexpectedlyAccepted.isEmpty, "accepted \(unexpectedlyAccepted.count) not-wf cases: \(unexpectedlyAccepted)") + } + + @Test("Every sun valid case parses and validates cleanly against its DTD") + func test_validCleans() throws { + guard let root else { + return // Opt-in: suite not present. + } + let section = root + "/sun" + var failures: [String: String] = [:] + for testCase in try manifest("sun-valid.xml", in: section) where !knownValidFailures.contains(testCase.uri) { + let directory = caseDirectory(section, uri: testCase.uri) + do { + let errors = try PureXML.validateAgainstInternalDTD( + source(of: testCase.uri, in: section), + limits: limits(), + resolver: resolver(directory: directory), + ) + if let first = errors.first { + failures[testCase.uri] = String(describing: first) + } + } catch { + failures[testCase.uri] = String(describing: error) + } + } + #expect(failures.isEmpty, "\(failures.count) valid cases failed: \(failures)") + } + + @Test("Every sun invalid case parses but reports a validity error") + func test_invalidReports() throws { + guard let root else { + return // Opt-in: suite not present. + } + let section = root + "/sun" + var silentlyClean: [String] = [] + var parseFailures: [String: String] = [:] + for testCase in try manifest("sun-invalid.xml", in: section) where !knownInvalidAccepted.contains(testCase.uri) { + let directory = caseDirectory(section, uri: testCase.uri) + do { + let errors = try PureXML.validateAgainstInternalDTD( + source(of: testCase.uri, in: section), + limits: limits(), + resolver: resolver(directory: directory), + ) + if errors.isEmpty { + silentlyClean.append(testCase.uri) + } + } catch { + parseFailures[testCase.uri] = String(describing: error) + } + } + #expect(parseFailures.isEmpty, "\(parseFailures.count) invalid (well-formed) cases failed to parse: \(parseFailures)") + #expect(silentlyClean.isEmpty, "\(silentlyClean.count) invalid cases validated clean: \(silentlyClean)") + } +}