From ae37f493f9e3a5c6e2602ed41cdd65d927ab0b31 Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Sun, 28 Jun 2026 21:14:12 +0200 Subject: [PATCH] test(schema): generated-instance validation differential vs libxml2 (#370) The schema-compile determinism check has a generated-schema differential; this adds the counterpart for the instance-validation path, exercising the per-instance content matcher (the graph-structured-stack ContentRTNMatcher now in production) over the open input space. For every generated, both-engines-compile schema it validates many generated instances with PureXML and with libxml2 and compares the verdicts, characterising the cardinal rule's instance half beyond the finite XSTS corpus. It asserts the cardinal rule (PureXML never rejects an instance libxml2 accepts) and reports false negatives for triage. Over 6568 generated validations (1357 both-valid, 5211 both-invalid) across sequences, choices, all groups, group references, occurrences, and wildcards it found zero false positives and zero false negatives: independent confirmation that the matcher agrees with libxml2. Opt-in via INSTANCE_DIFF and xmllint, a no-op in plain swift test and CI; the libxml2 subprocess is guarded for WASI. No behaviour change. --- CHANGELOG.md | 2 + .../InstanceValidationDifferentialTests.swift | 223 ++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 Tests/InstanceValidationDifferentialTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 46608c9..29fe81f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- A generated-instance validation differential against libxml2 (`InstanceValidationDifferentialTests`, opt-in via `INSTANCE_DIFF` and `xmllint`) exercises the per-instance content matcher (the graph-structured-stack `ContentRTNMatcher` now in production) over the open input space: for every generated, both-engines-compile schema it validates many generated `` instances with PureXML and with libxml2 and compares the verdicts. Where the schema-compile determinism check has its own differential, this characterises instance validation, the cardinal rule's instance half, beyond the finite XSTS corpus. It asserts the cardinal rule (PureXML never rejects an instance libxml2 accepts) and reports false negatives for triage. Over 6568 generated validations (1357 both-valid, 5211 both-invalid) across sequences, choices, `all` groups, group references, occurrences, and wildcards it found zero false positives and zero false negatives, independent confirmation that the matcher agrees with libxml2. Test-only; no shipped behaviour change. + - The graph-structured-stack content matcher's runtime resource use now has a proven, asserted bound: `ContentRTNMatcher` exposes `peakLiveConfigurations` (the largest live-configuration set in any closure) and `gssNodeCount` (the total graph-structured-stack nodes built), and `ContentRTNDifferentialTests` asserts they stay linear in a shared-group chain's depth (measured `gss = K + 2`, peak configurations `= 3K + 3`), where an explicit return stack would reach `O(distinct stacks)`. The bound is checked by count, not by a machine-dependent wall-clock timeout, so a regression to super-linear fails the test rather than only slowing down. Test-and-metadata only; no shipped behaviour change. - A generated-schema determinism differential against libxml2 (`DeterminismDifferentialTests`, opt-in via `DETERMINISM_DIFF` and `xmllint`) fuzzes content models, elements, group references, choices, sequences, `all` groups, occurrences, and wildcards, and compares PureXML's Unique Particle Attribution accept/reject verdict to the reference's. The XSTS corpus is finite and did not exercise the multiply-referenced-group overlap that was a real gap; this characterises the UPA check over the open shape space instead, asserting the cardinal rule (zero schemas PureXML rejects as ambiguous that libxml2 accepts) and reporting false negatives for triage. It found zero false positives and a genuine, named under-rejection: a repeating model whose optional trailing particle and leading particle share a name (`(c, b?, c?)*`), where the compositional check forms the internal gap before the enclosing repetition's loop edge, so the combined decision set is never checked. Test-only; no shipped behaviour change. diff --git a/Tests/InstanceValidationDifferentialTests.swift b/Tests/InstanceValidationDifferentialTests.swift new file mode 100644 index 0000000..fe543fb --- /dev/null +++ b/Tests/InstanceValidationDifferentialTests.swift @@ -0,0 +1,223 @@ +import Foundation +import Testing +@testable import PureXML + +/// A generated-instance differential for content-model VALIDATION against libxml2 +/// (`xmllint --schema`), the reference. The schema-compile determinism check has its own +/// differential; this one exercises the per-instance content matcher (the +/// graph-structured-stack `ContentRTNMatcher` now in production) over the open input +/// space: for each generated, both-compile schema it validates many generated `` +/// instances with PureXML and with libxml2 and flags every disagreement. +/// +/// Opt-in (needs `/usr/bin/xmllint`, set `INSTANCE_DIFF=1`); never runs in plain +/// `swift test` or CI. A PureXML-rejects / libxml2-accepts disagreement is a FALSE +/// POSITIVE (the cardinal rule: never reject a valid instance); a PureXML-accepts / +/// libxml2-rejects is a false negative to triage. Only schemas BOTH engines compile are +/// used, so a disagreement is about instance validation, not schema compilation. +@Suite("Instance-validation differential vs libxml2 (opt-in via INSTANCE_DIFF)") +struct InstanceValidationDifferentialTests { + private var enabled: Bool { + ProcessInfo.processInfo.environment["INSTANCE_DIFF"] != nil + } + + private var xmllint: String? { + let path = ProcessInfo.processInfo.environment["XMLLINT"].flatMap { $0.isEmpty ? nil : $0 } ?? "/usr/bin/xmllint" + return FileManager.default.isExecutableFile(atPath: path) ? path : nil + } + + /// libxml2's verdict for `instance` against `schemaFile`: the schema failed to + /// compile (skip), or the instance is valid / invalid. + private enum Outcome: Equatable { case schemaInvalid, valid, invalid } + + /// How PureXML's verdict relates to the reference for one instance. + private enum Comparison { case falsePositive, falseNegative, agreeValid, agreeInvalid } + + private func compare(pureValid: Bool, reference: Outcome) -> Comparison { + switch (pureValid, reference) { + case (false, .valid): .falsePositive // PureXML rejects a valid instance (cardinal-rule blocker) + case (true, .invalid): .falseNegative // PureXML accepts an invalid instance + case (true, _): .agreeValid + case (false, _): .agreeInvalid + } + } + + private func libxml2(_ schemaFile: String, _ instance: String, xmllint: String) -> Outcome? { + #if os(WASI) + return nil + #else + let instanceFile = NSTemporaryDirectory() + "inst-diff-\(abs(instance.hashValue)).xml" + guard (try? instance.write(toFile: instanceFile, atomically: true, encoding: .utf8)) != nil else { return nil } + defer { try? FileManager.default.removeItem(atPath: instanceFile) } + let process = Process() + process.executableURL = URL(fileURLWithPath: xmllint) + process.arguments = ["--noout", "--schema", schemaFile, instanceFile] + let pipe = Pipe() + process.standardError = pipe + process.standardOutput = pipe + guard (try? process.run()) != nil else { return nil } + let data = pipe.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + let output = String(decoding: data, as: UTF8.self) + if output.contains("failed to compile") || output.contains("Schemas parser error") { return .schemaInvalid } + return process.terminationStatus == 0 ? .valid : .invalid + #endif + } + + /// The running comparison totals; the cardinal-rule check is `falsePositives.isEmpty`. + private struct Tally { + var compared = 0 + var agreeValid = 0 + var agreeInvalid = 0 + var falsePositives: [String] = [] + var falseNegatives: [String] = [] + } + + /// A schema both engines compile: the PureXML document, its source, and the file + /// written for libxml2. + private struct CompiledSchema { + let document: PureXML.Schema.Document + let source: String + let file: String + } + + /// Validate several generated instances of one both-compile schema with both engines, + /// folding the outcomes into `tally`. + private func compareInstances( + _ schema: CompiledSchema, + xmllint: String, + generator: inout InstanceSchemaGenerator, + into tally: inout Tally, + ) { + for _ in 0 ..< 8 { + let instance = generator.instance() + let pureValid = ((try? schema.document.validate(instance)) ?? [PureXML.Validation.ValidationError]()).isEmpty + guard let reference = libxml2(schema.file, instance, xmllint: xmllint) else { continue } + if reference == .schemaInvalid { return } // libxml2 rejects the schema; skip its instances + tally.compared += 1 + let detail = "\(schema.source)\n instance: \(instance)" + switch compare(pureValid: pureValid, reference: reference) { + case .falsePositive: if tally.falsePositives.count < 20 { tally.falsePositives.append(detail) } + case .falseNegative: if tally.falseNegatives.count < 20 { tally.falseNegatives.append(detail) } + case .agreeValid: tally.agreeValid += 1 + case .agreeInvalid: tally.agreeInvalid += 1 + } + } + } + + @Test("PureXML's instance-validity verdict agrees with libxml2 over generated documents") + func test_instanceDifferential() { + guard enabled, let xmllint else { return } + var generator = InstanceSchemaGenerator(seed: 0x1A57_C0DE_5EED_BEEF) + var tally = Tally() + let schemaFile = NSTemporaryDirectory() + "inst-diff-schema.xsd" + for _ in 0 ..< 1500 { + let schema = generator.schema() + guard let document = try? PureXML.Schema.Document(schema) else { continue } // PureXML compiles + guard (try? schema.write(toFile: schemaFile, atomically: true, encoding: .utf8)) != nil else { continue } + defer { try? FileManager.default.removeItem(atPath: schemaFile) } + let compiled = CompiledSchema(document: document, source: schema, file: schemaFile) + compareInstances(compiled, xmllint: xmllint, generator: &generator, into: &tally) + } + let report = "compared \(tally.compared) instances (agree-valid \(tally.agreeValid), agree-invalid \(tally.agreeInvalid));" + + " false positives \(tally.falsePositives.count), false negatives \(tally.falseNegatives.count)" + FileHandle.standardError.write(Data("instance-diff: \(report)\n".utf8)) + try? (tally.falsePositives + ["--- false negatives ---"] + tally.falseNegatives).joined(separator: "\n\n") + .write(toFile: NSTemporaryDirectory() + "instance-diff.txt", atomically: true, encoding: .utf8) + // The cardinal rule: PureXML must never reject a valid instance that libxml2 accepts. + #expect( + tally.falsePositives.isEmpty, + "\(tally.falsePositives.count) instance-validation false positives (see \(NSTemporaryDirectory())instance-diff.txt): \(tally.falsePositives.first ?? "")", + ) + } +} + +/// Generates a `doc` element whose content model varies over the matcher's space +/// (sequences, choices, `all` groups, group references, occurrences, wildcards) using a +/// small element alphabet, and matching `` instances drawn from that alphabet so a +/// fair fraction are valid. Kept structurally well-formed so a rejection is a content +/// rejection, not a parse error. +private struct InstanceSchemaGenerator { + private var rng: InstanceRNG + private var groupNames: [String] = [] + private static let alphabet = ["a", "b", "c"] + + init(seed: UInt64) { + rng = InstanceRNG(seed: seed) + } + + private mutating func pick(_ options: [T]) -> T { + options[Int.random(in: 0 ..< options.count, using: &rng)] + } + + private mutating func chance(_ bound: Int) -> Bool { + Int.random(in: 0 ..< bound, using: &rng) == 0 + } + + private mutating func occurrence() -> String { + switch Int.random(in: 0 ..< 5, using: &rng) { + case 0: "" + case 1: " minOccurs=\"0\"" + case 2: " maxOccurs=\"unbounded\"" + case 3: " minOccurs=\"0\" maxOccurs=\"unbounded\"" + default: " maxOccurs=\"3\"" + } + } + + private mutating func particle(depth: Int) -> String { + if depth == 0 || chance(2) { + if chance(5) { return "" } + return "" + } + if !groupNames.isEmpty, chance(3) { + return "" + } + let compositor = pick(["sequence", "choice", "all"]) + let count = Int.random(in: 1 ... 3, using: &rng) + if compositor == "all" { + let members = (0 ..< count).map { _ in "" } + return "\(members.joined())" + } + let members = (0 ..< count).map { _ in particle(depth: depth - 1) }.joined() + return "\(members)" + } + + private func wrapModel(_ particle: String) -> String { + if particle.hasPrefix("\(particle)" + } + + mutating func schema() -> String { + groupNames = [] + var groups = "" + for index in 0 ..< Int.random(in: 0 ... 2, using: &rng) { + let name = "g\(index)" + groupNames.append(name) + groups += "\(wrapModel(particle(depth: 1)))" + } + let body = wrapModel(particle(depth: 3)) + return "\(groups)" + + "\(body)" + } + + mutating func instance() -> String { + let count = Int.random(in: 0 ... 6, using: &rng) + let children = (0 ..< count).map { _ in "<\(pick(Self.alphabet))/>" }.joined() + return "\(children)" + } +} + +private struct InstanceRNG: RandomNumberGenerator { + private var state: UInt64 + init(seed: UInt64) { + state = seed == 0 ? 0x9E37_79B9_7F4A_7C15 : seed + } + + mutating func next() -> UInt64 { + state ^= state >> 12 + state ^= state << 25 + state ^= state >> 27 + return state &* 0x2545_F491_4F6C_DD1D + } +}