From 7a126a9b0a80e54db4bc5a7be1a516f29dd162a2 Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Sat, 20 Jun 2026 19:34:04 +0200 Subject: [PATCH] test(schema): enforce streaming-vs-tree agreement as a standing gate The streaming XSD validator is a separate implementation of the same semantics as the tree validator (the conformance oracle), and three false positives in it were found and fixed this cycle (#186, #200, #202) only by adversarial hunting, because the XSTS gate exercises the tree path, not validate(streaming:). A snapshot is not an invariant. Add test_streamingMatchesTreeDifferential to the generative fuzz suite: 2000 deterministic schema/instance pairs (15 on WASI), asserting that streaming and the tree validator agree on whether each document is valid (so streaming never rejects a valid document and never accepts an invalid one the tree rejects), and that the tree's errors are a subset of streaming's (streaming may add errors on an already-invalid document, #206, but may never miss a real error or invent a rejection). A failure prints the seed, schema, instance, and both error sets, so it reproduces exactly. This runs in the standard suite on all five platforms, so a future streaming false positive fails CI rather than reaching users. --- CHANGELOG.md | 10 ++++++++++ Tests/SchemaFuzzTests.swift | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77ca068..6629f7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a standing streaming-vs-tree differential gate (`SchemaFuzzTests.test_streamingMatchesTreeDifferential`, + related to #194). The streaming XSD validator is a separate implementation of the same semantics as the + tree validator (the conformance oracle), so it must agree with it on every input. The gate fuzzes 2000 + deterministic schema/instance pairs and asserts that streaming and the tree validator agree on whether + each document is valid, and that every tree error appears in streaming's errors (streaming may add errors + on an already-invalid document, issue #206, but may never reject a valid document or miss a real error). + This runs in the standard test suite on all platforms, so a future change that reintroduces a streaming + false positive (three were found and fixed this cycle: #186, #200, #202) fails CI with a reproducible seed, + turning streaming-oracle agreement from a manually-hunted snapshot into an enforced invariant. + - Added an opt-in differential schema-validity harness against libxml2 (`XSTSDifferentialTests`, production-readiness stopper #3: characterise correctness against an independent reference, not just the labelled suite). Over 9618 corpus schemas PureXML and libxml2 agree on 4912 (3161 valid, 1751 invalid); the 87 where PureXML is stricter were triaged as PureXML being more correct (libxml2 omits facet consistency, Element Declarations Consistent, whiteSpace restriction, au-props-correct, and notation rules), independently corroborating that PureXML does not over-reject. The 117 where libxml2 is stricter are a false-negative triage queue (stopper #2), written with the rest to /tmp/xsts-differential.txt. xmllint and the corpus are used only in the test harness, never the shipped package. - Added the counted-content-automaton design for the interactive resource-bound diff --git a/Tests/SchemaFuzzTests.swift b/Tests/SchemaFuzzTests.swift index 3f54474..4022ff3 100644 --- a/Tests/SchemaFuzzTests.swift +++ b/Tests/SchemaFuzzTests.swift @@ -143,9 +143,11 @@ struct SchemaFuzzTests { #if os(WASI) private static let schemaIterations: UInt64 = 15 private static let parserIterations: UInt64 = 40 + private static let differentialIterations: UInt64 = 15 #else private static let schemaIterations: UInt64 = 250 private static let parserIterations: UInt64 = 600 + private static let differentialIterations: UInt64 = 2000 #endif // The `.timeLimit` trait (native hang detection) is not supported by the @@ -170,6 +172,37 @@ struct SchemaFuzzTests { } } + // Standing differential gate: the streaming validator is a separate + // implementation of the same semantics as the tree validator (the conformance + // oracle), so it must AGREE with it on every input. This enforces what manual + // hunting only sampled: the streaming path never rejects a document the tree + // accepts (no false positive) and never accepts one the tree rejects. Streaming + // may emit additional errors on an already-invalid document (issue #206), so the + // tree's errors must be a subset of streaming's; it may never miss one, and the + // two must agree on whether the document is valid. A failure prints the seed, + // schema, instance, and both error sets, so it reproduces exactly. + #if os(WASI) + @Test("Streaming validation agrees with the tree validator on every fuzzed document") + #else + @Test("Streaming validation agrees with the tree validator on every fuzzed document", .timeLimit(.minutes(1))) + #endif + func test_streamingMatchesTreeDifferential() { + for seed in UInt64(1) ... Self.differentialIterations { + var generator = SchemaGenerator(rng: SeededRNG(seed: seed)) + let xsd = generator.schema() + guard let document = try? PureXML.Schema.Document(xsd) else { continue } + let xml = generator.instance() + guard let tree = try? document.validate(xml), + let streamed = try? document.validate(streaming: xml) + else { continue } + let treeReasons = Set(tree.map(\.reason)) + let streamedReasons = Set(streamed.map(\.reason)) + let detail = "seed \(seed)\n tree: \(treeReasons.sorted())\n stream: \(streamedReasons.sorted())\n xsd: \(xsd)\n xml: \(xml)" + #expect(tree.isEmpty == streamed.isEmpty, "streaming/tree verdict disagreement (\(detail))") + #expect(treeReasons.isSubset(of: streamedReasons), "streaming missed a tree error (\(detail))") + } + } + #if os(WASI) @Test("The parser never crashes on arbitrary byte sequences") #else