From 7f82627403bc962a0986f0f2d4f219cf4cbcc9e0 Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Sat, 13 Jun 2026 18:45:40 +0200 Subject: [PATCH] test(xsts): count only settled expectations, excluding queried/disputed entries The runner read but ignored , so it counted disagreements with W3C-disputed expectations as deviations. The 2006-11-06 archive marks 748 instanceTests (plus schemaTests) status="queried" with the expected validity contested in W3C Bugzilla (for example reJ11.i: \p{Lu}* over two uppercase letters, prior expectation valid, current invalid+queried, where PureXML's accept is defensible). Those are now skipped: neither agreement nor deviation, since the expectation is not authoritative. Recalibrated baseline (settled expectations only): valid schemas rejected 74 to 72, invalid schemas accepted 2467 to 2461, valid instances rejected 369 to 338, invalid instances accepted 210 to 179. About 70 apparent deviations were disagreements with contested expectations, never bugs. Refs #129, #122. --- Tests/XSTSSuiteTests.swift | 31 +++++++++++++++++++++---------- docs/xsts-deviations.md | 7 +++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/Tests/XSTSSuiteTests.swift b/Tests/XSTSSuiteTests.swift index c84bc74..6882afb 100644 --- a/Tests/XSTSSuiteTests.swift +++ b/Tests/XSTSSuiteTests.swift @@ -19,19 +19,19 @@ struct XSTSSuiteTests { } /// Exact-count baselines against the 2006-11-06 archive (first measured - /// 2026-06-13, over 14383 groups). These ratchet down as conformance work - /// closes deviations (#145-#148); a count rising is a regression. The - /// instance counts reflect the #146 list-facet fix (built-in list length - /// counts items), the #147 union-facet fix (pattern and enumeration - /// enforced on unions), and the #147 duration-ordering fix (minInclusive and - /// friends on xs:duration). The suite is opt-in, so CI and plain + /// 2026-06-13, over 14383 groups), counting only tests whose expected outcome + /// is settled: entries marked `` (disputed in W3C + /// Bugzilla, with the expected validity itself contested) are excluded, since + /// agreeing or disagreeing with a contested expectation is not meaningful. + /// These ratchet down as conformance work closes deviations (#145-#148); a + /// count rising is a regression. The suite is opt-in, so CI and plain /// `swift test` are unaffected; run it with `swift test -c release --filter /// XSTS` (debug is far slower and the corpus is large). Per-case deviations /// are written to /tmp/xsts-failures.txt for the burn-down. - private let knownSchemaValidRejected = 74 - private let knownSchemaInvalidAccepted = 2467 - private let knownInstanceValidRejected = 369 - private let knownInstanceInvalidAccepted = 210 + private let knownSchemaValidRejected = 72 + private let knownSchemaInvalidAccepted = 2461 + private let knownInstanceValidRejected = 338 + private let knownInstanceInvalidAccepted = 179 @Test("Every XSTS case behaves: compile, reject, validate, invalidate") func test_suite() throws { @@ -98,6 +98,7 @@ struct XSTSSuiteTests { var schema: PureXML.Schema.Document? var schemaExpectedValid = true for schemaTest in elements(named: "schemaTest", under: .element(group)) { + guard isAccepted(schemaTest) else { continue } guard let first = references(in: .element(schemaTest), named: "schemaDocument").first else { continue } schemaExpectedValid = expectedValidity(schemaTest) != "invalid" let schemaPath = resolve(first, against: directory) @@ -133,6 +134,7 @@ struct XSTSSuiteTests { into counters: inout Counters, ) { for instanceTest in elements(named: "instanceTest", under: .element(group)) { + guard isAccepted(instanceTest) else { continue } let expected = expectedValidity(instanceTest) guard expected == "valid" || expected == "invalid", let href = references(in: .element(instanceTest), named: "instanceDocument").first, @@ -185,6 +187,15 @@ struct XSTSSuiteTests { element.attributes.first { $0.name.localName == name }?.value } + /// Whether a schemaTest/instanceTest's expected outcome is settled. The suite + /// marks disputed entries `` (often with a W3C + /// Bugzilla reference) where the expected validity is itself contested; those + /// are not authoritative, so they are not counted as agreements or deviations. + private func isAccepted(_ test: PureXML.Model.Element) -> Bool { + guard let current = elements(named: "current", under: .element(test)).first else { return true } + return (attribute("status", of: current) ?? "accepted") == "accepted" + } + private func expectedValidity(_ element: PureXML.Model.Element) -> String { elements(named: "expected", under: .element(element)).first .flatMap { attribute("validity", of: $0) } ?? "notKnown" diff --git a/docs/xsts-deviations.md b/docs/xsts-deviations.md index 4c9105c..ac44c2d 100644 --- a/docs/xsts-deviations.md +++ b/docs/xsts-deviations.md @@ -22,6 +22,13 @@ The oracle is binary and coarse: "invalid schema accepted" conflates *we have a bug* with *we do not implement this validity constraint by design*. The point of this document is to separate those. +Only tests with a settled expectation are counted. The suite marks contested +entries `` (disputed in W3C Bugzilla, the expected +validity itself in question, e.g. `\p{Lu}*` over two uppercase letters one of +which is an astral math capital); agreeing or disagreeing with a contested +expectation is not meaningful, so the runner skips them. Excluding the queried +entries removed about 70 apparent deviations that were never bugs. + ## Baseline (post #146 list-facet fix) | Kind | Count |