From 41338c9e3f4afae4c859f3167025098bdcb2b417 Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Mon, 29 Jun 2026 12:32:08 +0200 Subject: [PATCH] test(schema): adjudicate the instance differential with an independent oracle, not libxml2 alone (#386) The instance-validation differential treated "PureXML rejects, libxml2 accepts" as a PureXML false positive. That is unsound: libxml2's own content validation is lenient and accepts some INVALID instances. A DIFF_SEED sweep surfaced exactly such a case (seed 13371337): the model ( (a*|c*){1,3}, b? ){1,3} and the instance . libxml2 accepts it, but it is genuinely INVALID (the three b's anchor three (choice,b?) segments, leaving the trailing 'ca' with no fourth segment), so PureXML's rejection is correct, not a false positive. Verified against an external regex engine and exhaustive enumeration. Add ContentModelOracle, an independent brute-force membership matcher over element names, implemented separately from PureXML's matcher. The generated instances are empty xs:string elements, so an instance is valid iff its element-name sequence is in the model's language, which this oracle decides exactly. The generator now emits the structured ContentModel alongside the schema string (group references inlined), and the differential classifies a false positive / false negative against the ORACLE as ground truth; libxml2 is kept as a secondary cross-check whose disagreements with the oracle are counted (its leniency) but never charged to PureXML. The oracle is unit-tested (ContentModelOracleTests), including the libxml2-leniency case it exists to catch. Over a sweep of seeds the differential now reports zero oracle-adjudicated false positives and zero false negatives, so PureXML's content matcher agrees with the independent oracle on every generated case; the default seed is unchanged. Test-only; no library behaviour change. --- Tests/ContentModelOracle.swift | 106 +++++++++++++ Tests/ContentModelOracleTests.swift | 60 +++++++ .../InstanceValidationDifferentialTests.swift | 149 +++++++++++------- 3 files changed, 261 insertions(+), 54 deletions(-) create mode 100644 Tests/ContentModelOracle.swift create mode 100644 Tests/ContentModelOracleTests.swift diff --git a/Tests/ContentModelOracle.swift b/Tests/ContentModelOracle.swift new file mode 100644 index 0000000..4f01ec7 --- /dev/null +++ b/Tests/ContentModelOracle.swift @@ -0,0 +1,106 @@ +@testable import PureXML + +/// An independent, brute-force content-model membership oracle over element names, +/// deliberately implemented separately from PureXML's matcher so the instance +/// differential can adjudicate a disagreement with libxml2: libxml2's content +/// validation has leniency bugs (it accepts some invalid nested/nullable/unbounded +/// models), so "PureXML rejects, libxml2 accepts" is not by itself a PureXML fault. +/// The generated instances are empty `xs:string` elements, so an instance is valid +/// iff its element-name sequence is in this language; this oracle is that decision, +/// verified against an external regex engine on the known cases. +indirect enum ContentModel: Sendable { + case element(String) + case any + case sequence([ContentModel]) + case choice([ContentModel]) + /// An `xs:all` group; each member is its own `occurs(element, min, 1)`. + case all([ContentModel]) + case occurs(ContentModel, Int, Int?) +} + +enum ContentModelOracle { + /// Whether the model can match the empty sequence. + static func nullable(_ model: ContentModel) -> Bool { + switch model { + case .element, .any: false + case let .sequence(parts): parts.allSatisfy(nullable) + case let .choice(parts): parts.contains(where: nullable) + case let .all(parts): parts.allSatisfy(nullable) + case let .occurs(sub, low, _): low == 0 || nullable(sub) + } + } + + /// Every position reachable after matching `model` starting at `start`. + static func ends(_ model: ContentModel, _ tokens: [String], _ start: Int) -> Set { + switch model { + case let .element(name): + return start < tokens.count && tokens[start] == name ? [start + 1] : [] + case .any: + return start < tokens.count ? [start + 1] : [] + case let .sequence(parts): + var positions: Set = [start] + for part in parts { + var next: Set = [] + for position in positions { + next.formUnion(ends(part, tokens, position)) + } + positions = next + if positions.isEmpty { break } + } + return positions + case let .choice(parts): + var out: Set = [] + for part in parts { + out.formUnion(ends(part, tokens, start)) + } + return out + case let .all(parts): + return allEnds(parts, Array(repeating: false, count: parts.count), tokens, start) + case let .occurs(sub, low, high): + return occursEnds(sub, low, high, tokens, start) + } + } + + /// Positions reachable by matching `sub` between `low` and `high` times. A + /// nullable `sub` is bounded by stopping once the reachable set stops growing. + private static func occursEnds(_ sub: ContentModel, _ low: Int, _ high: Int?, _ tokens: [String], _ start: Int) -> Set { + var byReps: [Set] = [[start]] + let cap = min(high ?? Int.max, tokens.count + low + 2) + var reps = 0 + while reps < cap { + var next: Set = [] + for position in byReps[reps] { + next.formUnion(ends(sub, tokens, position)) + } + byReps.append(next) + reps += 1 + let priorPositions = byReps[0 ..< reps].reduce(into: Set()) { $0.formUnion($1) } + if next.isSubset(of: priorPositions), reps >= low { break } + } + var out: Set = [] + if low == 0 { out.insert(start) } + for count in low ... cap where count < byReps.count { + out.formUnion(byReps[count]) + } + return out + } + + /// Positions reachable by matching the remaining (not-`done`) members of an + /// `xs:all` group in any order; a member that is nullable may be skipped. + private static func allEnds(_ members: [ContentModel], _ done: [Bool], _ tokens: [String], _ start: Int) -> Set { + var out: Set = [] + if members.indices.allSatisfy({ done[$0] || nullable(members[$0]) }) { out.insert(start) } + for index in members.indices where !done[index] { + for end in ends(members[index], tokens, start) where end > start { + var next = done + next[index] = true + out.formUnion(allEnds(members, next, tokens, end)) + } + } + return out + } + + static func matches(_ model: ContentModel, _ tokens: [String]) -> Bool { + ends(model, tokens, 0).contains(tokens.count) + } +} diff --git a/Tests/ContentModelOracleTests.swift b/Tests/ContentModelOracleTests.swift new file mode 100644 index 0000000..56847b5 --- /dev/null +++ b/Tests/ContentModelOracleTests.swift @@ -0,0 +1,60 @@ +import Testing +@testable import PureXML + +/// The independent content-model membership oracle that adjudicates instance- +/// differential disagreements. Pins the cases that matter, including the libxml2 +/// leniency case the oracle exists to catch: a model libxml2 wrongly accepts an +/// invalid instance for, which the oracle (and PureXML) correctly reject. +@Suite("Content-model membership oracle") +struct ContentModelOracleTests { + private func tokens(_ string: String) -> [String] { + string.map(String.init) + } + + /// `( (a*|c*){1,3} b? ){1,3}`, the model from the DIFF_SEED=13371337 case. + private var nestedNullable: ContentModel { + let choice = ContentModel.occurs( + .choice([.occurs(.element("a"), 0, nil), .occurs(.sequence([.occurs(.element("c"), 0, nil)]), 1, nil)]), + 1, 3, + ) + return .occurs(.sequence([choice, .occurs(.element("b"), 0, 1)]), 1, 3) + } + + @Test("the nested nullable-repetition model matches exactly its language") + func test_nestedNullable() { + // 'bbabca' is the instance libxml2 LENIENTLY accepts but is NOT in the + // language: three b's anchor three (choice,b?) segments, leaving the + // trailing 'ca' with no fourth segment. The oracle (like PureXML) rejects it. + #expect(!ContentModelOracle.matches(nestedNullable, tokens("bbabca"))) + #expect(!ContentModelOracle.matches(nestedNullable, tokens("bbabcab"))) + // Members, cross-checked against an external regex engine. + #expect(ContentModelOracle.matches(nestedNullable, tokens("bbab"))) + #expect(ContentModelOracle.matches(nestedNullable, tokens("aca"))) + #expect(ContentModelOracle.matches(nestedNullable, tokens("abcabc"))) + #expect(ContentModelOracle.matches(nestedNullable, tokens("b"))) + #expect(ContentModelOracle.matches(nestedNullable, tokens(""))) // one empty outer iteration + } + + @Test("sequence, choice, occurrence, and wildcard match as specified") + func test_basics() { + let seq = ContentModel.sequence([.element("a"), .occurs(.element("b"), 0, 1), .element("c")]) + #expect(ContentModelOracle.matches(seq, tokens("abc"))) + #expect(ContentModelOracle.matches(seq, tokens("ac"))) + #expect(!ContentModelOracle.matches(seq, tokens("abbc"))) + let choice = ContentModel.occurs(.choice([.element("a"), .element("c")]), 1, 3) + #expect(ContentModelOracle.matches(choice, tokens("aca"))) + #expect(!ContentModelOracle.matches(choice, tokens("acac"))) + let wildcard = ContentModel.occurs(.any, 0, nil) + #expect(ContentModelOracle.matches(wildcard, tokens("abc"))) + } + + @Test("an all group matches its members in any order, requiring the non-optional ones") + func test_allGroup() { + let all = ContentModel.all([.occurs(.element("a"), 1, 1), .occurs(.element("b"), 0, 1)]) + #expect(ContentModelOracle.matches(all, tokens("a"))) + #expect(ContentModelOracle.matches(all, tokens("ab"))) + #expect(ContentModelOracle.matches(all, tokens("ba"))) + #expect(!ContentModelOracle.matches(all, tokens("b"))) // required 'a' absent + #expect(!ContentModelOracle.matches(all, tokens("aa"))) // 'a' is max 1 + } +} diff --git a/Tests/InstanceValidationDifferentialTests.swift b/Tests/InstanceValidationDifferentialTests.swift index dee8c6f..2ab2ae5 100644 --- a/Tests/InstanceValidationDifferentialTests.swift +++ b/Tests/InstanceValidationDifferentialTests.swift @@ -29,18 +29,6 @@ struct InstanceValidationDifferentialTests { /// compile (skip), or the instance is valid / invalid. private enum Outcome: Equatable { case schemaInvalid, valid, invalid } - /// How PureXML's verdict relates to the reference for one instance. - private enum Comparison { case falsePositive, falseNegative, agreeValid, agreeInvalid } - - private func compare(pureValid: Bool, reference: Outcome) -> Comparison { - switch (pureValid, reference) { - case (false, .valid): .falsePositive // PureXML rejects a valid instance (cardinal-rule blocker) - case (true, .invalid): .falseNegative // PureXML accepts an invalid instance - case (true, _): .agreeValid - case (false, _): .agreeInvalid - } - } - private func libxml2(_ schemaFile: String, _ instance: String, xmllint: String) -> Outcome? { #if os(WASI) return nil @@ -70,18 +58,25 @@ struct InstanceValidationDifferentialTests { var agreeInvalid = 0 var falsePositives: [String] = [] var falseNegatives: [String] = [] + /// Instances where libxml2 disagrees with the independent oracle (libxml2's + /// own content-validation leniency), tracked but never blamed on PureXML. + var libxml2Disagreements = 0 } - /// A schema both engines compile: the PureXML document, its source, and the file - /// written for libxml2. + /// A schema both engines compile: the PureXML document, its independent content + /// model (the oracle's ground truth), its source, and the file written for libxml2. private struct CompiledSchema { let document: PureXML.Schema.Document + let model: ContentModel let source: String let file: String } - /// Validate several generated instances of one both-compile schema with both engines, - /// folding the outcomes into `tally`. + /// Validate several generated instances of one both-compile schema. Validity is + /// decided by the independent ``ContentModelOracle`` (the generated instances are + /// empty `xs:string` elements, so an instance is valid iff its element-name + /// sequence is in the model's language); libxml2 is a secondary cross-check whose + /// disagreements are counted but not attributed to PureXML. private func compareInstances( _ schema: CompiledSchema, xmllint: String, @@ -90,16 +85,18 @@ struct InstanceValidationDifferentialTests { ) { for _ in 0 ..< 8 { let instance = generator.instance() - let pureValid = ((try? schema.document.validate(instance)) ?? [PureXML.Validation.ValidationError]()).isEmpty - guard let reference = libxml2(schema.file, instance, xmllint: xmllint) else { continue } + let pureValid = ((try? schema.document.validate(instance.xml)) ?? [PureXML.Validation.ValidationError]()).isEmpty + guard let reference = libxml2(schema.file, instance.xml, xmllint: xmllint) else { continue } if reference == .schemaInvalid { return } // libxml2 rejects the schema; skip its instances + let truth = ContentModelOracle.matches(schema.model, instance.tokens) tally.compared += 1 - let detail = "\(schema.source)\n instance: \(instance)" - switch compare(pureValid: pureValid, reference: reference) { - case .falsePositive: if tally.falsePositives.count < 20 { tally.falsePositives.append(detail) } - case .falseNegative: if tally.falseNegatives.count < 20 { tally.falseNegatives.append(detail) } - case .agreeValid: tally.agreeValid += 1 - case .agreeInvalid: tally.agreeInvalid += 1 + if (reference == .valid) != truth { tally.libxml2Disagreements += 1 } + let detail = "\(schema.source)\n instance: \(instance.xml)\n oracle: \(truth ? "valid" : "invalid"), libxml2: \(reference)" + switch (pureValid, truth) { + case (false, true): if tally.falsePositives.count < 20 { tally.falsePositives.append(detail) } + case (true, false): if tally.falseNegatives.count < 20 { tally.falseNegatives.append(detail) } + case (true, true): tally.agreeValid += 1 + case (false, false): tally.agreeInvalid += 1 } } } @@ -112,19 +109,22 @@ struct InstanceValidationDifferentialTests { var tally = Tally() let schemaFile = NSTemporaryDirectory() + "inst-diff-schema.xsd" for _ in 0 ..< 1500 { - let schema = generator.schema() - guard let document = try? PureXML.Schema.Document(schema) else { continue } // PureXML compiles - guard (try? schema.write(toFile: schemaFile, atomically: true, encoding: .utf8)) != nil else { continue } + let generated = generator.schema() + guard let document = try? PureXML.Schema.Document(generated.xml) else { continue } // PureXML compiles + guard (try? generated.xml.write(toFile: schemaFile, atomically: true, encoding: .utf8)) != nil else { continue } defer { try? FileManager.default.removeItem(atPath: schemaFile) } - let compiled = CompiledSchema(document: document, source: schema, file: schemaFile) + let compiled = CompiledSchema(document: document, model: generated.model, source: generated.xml, file: schemaFile) compareInstances(compiled, xmllint: xmllint, generator: &generator, into: &tally) } let report = "compared \(tally.compared) instances (agree-valid \(tally.agreeValid), agree-invalid \(tally.agreeInvalid));" - + " false positives \(tally.falsePositives.count), false negatives \(tally.falseNegatives.count)" + + " false positives \(tally.falsePositives.count), false negatives \(tally.falseNegatives.count)," + + " libxml2 disagreements with the oracle \(tally.libxml2Disagreements)" FileHandle.standardError.write(Data("instance-diff: \(report)\n".utf8)) try? (tally.falsePositives + ["--- false negatives ---"] + tally.falseNegatives).joined(separator: "\n\n") .write(toFile: NSTemporaryDirectory() + "instance-diff.txt", atomically: true, encoding: .utf8) - // The cardinal rule: PureXML must never reject a valid instance that libxml2 accepts. + // The cardinal rule, adjudicated against the independent oracle (not libxml2, + // whose content validation is itself lenient): PureXML must never reject an + // instance that is genuinely valid. #expect( tally.falsePositives.isEmpty, "\(tally.falsePositives.count) instance-validation false positives (see \(NSTemporaryDirectory())instance-diff.txt): \(tally.falsePositives.first ?? "")", @@ -138,8 +138,23 @@ struct InstanceValidationDifferentialTests { /// fair fraction are valid. Kept structurally well-formed so a rejection is a content /// rejection, not a parse error. private struct InstanceSchemaGenerator { + /// A generated content model paired with its independent ``ContentModel``, so a + /// disagreement can be adjudicated against the oracle rather than libxml2 alone. + struct Particle { + let xml: String + let model: ContentModel + } + + /// A generated occurrence: the attribute text plus its parsed bounds. + private struct Occurrence { + let xml: String + let min: Int + let max: Int? + } + private var rng: InstanceRNG private var groupNames: [String] = [] + private var groupModels: [String: ContentModel] = [:] private static let alphabet = ["a", "b", "c"] init(seed: UInt64) { @@ -154,58 +169,84 @@ private struct InstanceSchemaGenerator { Int.random(in: 0 ..< bound, using: &rng) == 0 } - private mutating func occurrence() -> String { + private mutating func occurrence() -> Occurrence { switch Int.random(in: 0 ..< 5, using: &rng) { - case 0: "" - case 1: " minOccurs=\"0\"" - case 2: " maxOccurs=\"unbounded\"" - case 3: " minOccurs=\"0\" maxOccurs=\"unbounded\"" - default: " maxOccurs=\"3\"" + case 0: Occurrence(xml: "", min: 1, max: 1) + case 1: Occurrence(xml: " minOccurs=\"0\"", min: 0, max: 1) + case 2: Occurrence(xml: " maxOccurs=\"unbounded\"", min: 1, max: nil) + case 3: Occurrence(xml: " minOccurs=\"0\" maxOccurs=\"unbounded\"", min: 0, max: nil) + default: Occurrence(xml: " maxOccurs=\"3\"", min: 1, max: 3) } } - private mutating func particle(depth: Int) -> String { + /// Wraps a base model in an occurrence, leaving an exact-once occurrence bare. + private func occurring(_ base: ContentModel, _ occ: Occurrence) -> ContentModel { + occ.min == 1 && occ.max == 1 ? base : .occurs(base, occ.min, occ.max) + } + + private mutating func particle(depth: Int) -> Particle { if depth == 0 || chance(2) { - if chance(5) { return "" } - return "" + let occ = occurrence() + if chance(5) { + return Particle(xml: "", model: occurring(.any, occ)) + } + let name = pick(Self.alphabet) + return Particle(xml: "", model: occurring(.element(name), occ)) } if !groupNames.isEmpty, chance(3) { - return "" + let name = pick(groupNames) + let occ = occurrence() + return Particle(xml: "", model: occurring(groupModels[name] ?? .sequence([]), occ)) } let compositor = pick(["sequence", "choice", "all"]) let count = Int.random(in: 1 ... 3, using: &rng) if compositor == "all" { - let members = (0 ..< count).map { _ in "" } - return "\(members.joined())" + var xml = "" + var members: [ContentModel] = [] + for _ in 0 ..< count { + let name = pick(Self.alphabet) + let optional = chance(2) + xml += "" + members.append(.occurs(.element(name), optional ? 0 : 1, 1)) + } + return Particle(xml: "\(xml)", model: .all(members)) } - let members = (0 ..< count).map { _ in particle(depth: depth - 1) }.joined() - return "\(members)" + let occ = occurrence() + let parts = (0 ..< count).map { _ in particle(depth: depth - 1) } + let base: ContentModel = compositor == "sequence" ? .sequence(parts.map(\.model)) : .choice(parts.map(\.model)) + return Particle(xml: "\(parts.map(\.xml).joined())", model: occurring(base, occ)) } - private func wrapModel(_ particle: String) -> String { - if particle.hasPrefix(" Particle { + if particle.xml.hasPrefix("\(particle)" + return Particle(xml: "\(particle.xml)", model: .sequence([particle.model])) } - mutating func schema() -> String { + mutating func schema() -> (xml: String, model: ContentModel) { groupNames = [] + groupModels = [:] var groups = "" for index in 0 ..< Int.random(in: 0 ... 2, using: &rng) { let name = "g\(index)" + let body = wrapModel(particle(depth: 1)) groupNames.append(name) - groups += "\(wrapModel(particle(depth: 1)))" + groupModels[name] = body.model + groups += "\(body.xml)" } let body = wrapModel(particle(depth: 3)) - return "\(groups)" - + "\(body)" + let xml = "\(groups)" + + "\(body.xml)" + return (xml, body.model) } - mutating func instance() -> String { + mutating func instance() -> (xml: String, tokens: [String]) { let count = Int.random(in: 0 ... 6, using: &rng) - let children = (0 ..< count).map { _ in "<\(pick(Self.alphabet))/>" }.joined() - return "\(children)" + let tokens = (0 ..< count).map { _ in pick(Self.alphabet) } + return ("\(tokens.map { "<\($0)/>" }.joined())", tokens) } }