test(schema): adjudicate the instance differential with an independent oracle, not libxml2 alone (#386)

The instance-validation differential treated "PureXML rejects, libxml2 accepts"
as a PureXML false positive. That is unsound: libxml2's own content validation is
lenient and accepts some INVALID instances. A DIFF_SEED sweep surfaced exactly
such a case (seed 13371337): the model ( (a*|c*){1,3}, b? ){1,3} and the instance
<doc><b/><b/><a/><b/><c/><a/></doc>. libxml2 accepts it, but it is genuinely
INVALID (the three b's anchor three (choice,b?) segments, leaving the trailing
'ca' with no fourth segment), so PureXML's rejection is correct, not a false
positive. Verified against an external regex engine and exhaustive enumeration.

Add ContentModelOracle, an independent brute-force membership matcher over element
names, implemented separately from PureXML's matcher. The generated instances are
empty xs:string elements, so an instance is valid iff its element-name sequence is
in the model's language, which this oracle decides exactly. The generator now
emits the structured ContentModel alongside the schema string (group references
inlined), and the differential classifies a false positive / false negative
against the ORACLE as ground truth; libxml2 is kept as a secondary cross-check
whose disagreements with the oracle are counted (its leniency) but never charged
to PureXML.

The oracle is unit-tested (ContentModelOracleTests), including the libxml2-leniency
case it exists to catch. Over a sweep of seeds the differential now reports zero
oracle-adjudicated false positives and zero false negatives, so PureXML's content
matcher agrees with the independent oracle on every generated case; the default
seed is unchanged. Test-only; no library behaviour change.
This commit is contained in:
Mihaela Mihaljevic 2026-06-29 12:32:08 +02:00 committed by GitHub
parent 77884f99b8
commit 41338c9e3f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 261 additions and 54 deletions

View File

@ -0,0 +1,106 @@
@testable import PureXML
/// An independent, brute-force content-model membership oracle over element names,
/// deliberately implemented separately from PureXML's matcher so the instance
/// differential can adjudicate a disagreement with libxml2: libxml2's content
/// validation has leniency bugs (it accepts some invalid nested/nullable/unbounded
/// models), so "PureXML rejects, libxml2 accepts" is not by itself a PureXML fault.
/// The generated instances are empty `xs:string` elements, so an instance is valid
/// iff its element-name sequence is in this language; this oracle is that decision,
/// verified against an external regex engine on the known cases.
indirect enum ContentModel: Sendable {
case element(String)
case any
case sequence([ContentModel])
case choice([ContentModel])
/// An `xs:all` group; each member is its own `occurs(element, min, 1)`.
case all([ContentModel])
case occurs(ContentModel, Int, Int?)
}
enum ContentModelOracle {
/// Whether the model can match the empty sequence.
static func nullable(_ model: ContentModel) -> Bool {
switch model {
case .element, .any: false
case let .sequence(parts): parts.allSatisfy(nullable)
case let .choice(parts): parts.contains(where: nullable)
case let .all(parts): parts.allSatisfy(nullable)
case let .occurs(sub, low, _): low == 0 || nullable(sub)
}
}
/// Every position reachable after matching `model` starting at `start`.
static func ends(_ model: ContentModel, _ tokens: [String], _ start: Int) -> Set<Int> {
switch model {
case let .element(name):
return start < tokens.count && tokens[start] == name ? [start + 1] : []
case .any:
return start < tokens.count ? [start + 1] : []
case let .sequence(parts):
var positions: Set<Int> = [start]
for part in parts {
var next: Set<Int> = []
for position in positions {
next.formUnion(ends(part, tokens, position))
}
positions = next
if positions.isEmpty { break }
}
return positions
case let .choice(parts):
var out: Set<Int> = []
for part in parts {
out.formUnion(ends(part, tokens, start))
}
return out
case let .all(parts):
return allEnds(parts, Array(repeating: false, count: parts.count), tokens, start)
case let .occurs(sub, low, high):
return occursEnds(sub, low, high, tokens, start)
}
}
/// Positions reachable by matching `sub` between `low` and `high` times. A
/// nullable `sub` is bounded by stopping once the reachable set stops growing.
private static func occursEnds(_ sub: ContentModel, _ low: Int, _ high: Int?, _ tokens: [String], _ start: Int) -> Set<Int> {
var byReps: [Set<Int>] = [[start]]
let cap = min(high ?? Int.max, tokens.count + low + 2)
var reps = 0
while reps < cap {
var next: Set<Int> = []
for position in byReps[reps] {
next.formUnion(ends(sub, tokens, position))
}
byReps.append(next)
reps += 1
let priorPositions = byReps[0 ..< reps].reduce(into: Set<Int>()) { $0.formUnion($1) }
if next.isSubset(of: priorPositions), reps >= low { break }
}
var out: Set<Int> = []
if low == 0 { out.insert(start) }
for count in low ... cap where count < byReps.count {
out.formUnion(byReps[count])
}
return out
}
/// Positions reachable by matching the remaining (not-`done`) members of an
/// `xs:all` group in any order; a member that is nullable may be skipped.
private static func allEnds(_ members: [ContentModel], _ done: [Bool], _ tokens: [String], _ start: Int) -> Set<Int> {
var out: Set<Int> = []
if members.indices.allSatisfy({ done[$0] || nullable(members[$0]) }) { out.insert(start) }
for index in members.indices where !done[index] {
for end in ends(members[index], tokens, start) where end > start {
var next = done
next[index] = true
out.formUnion(allEnds(members, next, tokens, end))
}
}
return out
}
static func matches(_ model: ContentModel, _ tokens: [String]) -> Bool {
ends(model, tokens, 0).contains(tokens.count)
}
}

View File

@ -0,0 +1,60 @@
import Testing
@testable import PureXML
/// The independent content-model membership oracle that adjudicates instance-
/// differential disagreements. Pins the cases that matter, including the libxml2
/// leniency case the oracle exists to catch: a model libxml2 wrongly accepts an
/// invalid instance for, which the oracle (and PureXML) correctly reject.
@Suite("Content-model membership oracle")
struct ContentModelOracleTests {
private func tokens(_ string: String) -> [String] {
string.map(String.init)
}
/// `( (a*|c*){1,3} b? ){1,3}`, the model from the DIFF_SEED=13371337 case.
private var nestedNullable: ContentModel {
let choice = ContentModel.occurs(
.choice([.occurs(.element("a"), 0, nil), .occurs(.sequence([.occurs(.element("c"), 0, nil)]), 1, nil)]),
1, 3,
)
return .occurs(.sequence([choice, .occurs(.element("b"), 0, 1)]), 1, 3)
}
@Test("the nested nullable-repetition model matches exactly its language")
func test_nestedNullable() {
// 'bbabca' is the instance libxml2 LENIENTLY accepts but is NOT in the
// language: three b's anchor three (choice,b?) segments, leaving the
// trailing 'ca' with no fourth segment. The oracle (like PureXML) rejects it.
#expect(!ContentModelOracle.matches(nestedNullable, tokens("bbabca")))
#expect(!ContentModelOracle.matches(nestedNullable, tokens("bbabcab")))
// Members, cross-checked against an external regex engine.
#expect(ContentModelOracle.matches(nestedNullable, tokens("bbab")))
#expect(ContentModelOracle.matches(nestedNullable, tokens("aca")))
#expect(ContentModelOracle.matches(nestedNullable, tokens("abcabc")))
#expect(ContentModelOracle.matches(nestedNullable, tokens("b")))
#expect(ContentModelOracle.matches(nestedNullable, tokens(""))) // one empty outer iteration
}
@Test("sequence, choice, occurrence, and wildcard match as specified")
func test_basics() {
let seq = ContentModel.sequence([.element("a"), .occurs(.element("b"), 0, 1), .element("c")])
#expect(ContentModelOracle.matches(seq, tokens("abc")))
#expect(ContentModelOracle.matches(seq, tokens("ac")))
#expect(!ContentModelOracle.matches(seq, tokens("abbc")))
let choice = ContentModel.occurs(.choice([.element("a"), .element("c")]), 1, 3)
#expect(ContentModelOracle.matches(choice, tokens("aca")))
#expect(!ContentModelOracle.matches(choice, tokens("acac")))
let wildcard = ContentModel.occurs(.any, 0, nil)
#expect(ContentModelOracle.matches(wildcard, tokens("abc")))
}
@Test("an all group matches its members in any order, requiring the non-optional ones")
func test_allGroup() {
let all = ContentModel.all([.occurs(.element("a"), 1, 1), .occurs(.element("b"), 0, 1)])
#expect(ContentModelOracle.matches(all, tokens("a")))
#expect(ContentModelOracle.matches(all, tokens("ab")))
#expect(ContentModelOracle.matches(all, tokens("ba")))
#expect(!ContentModelOracle.matches(all, tokens("b"))) // required 'a' absent
#expect(!ContentModelOracle.matches(all, tokens("aa"))) // 'a' is max 1
}
}

View File

@ -29,18 +29,6 @@ struct InstanceValidationDifferentialTests {
/// compile (skip), or the instance is valid / invalid.
private enum Outcome: Equatable { case schemaInvalid, valid, invalid }
/// How PureXML's verdict relates to the reference for one instance.
private enum Comparison { case falsePositive, falseNegative, agreeValid, agreeInvalid }
private func compare(pureValid: Bool, reference: Outcome) -> Comparison {
switch (pureValid, reference) {
case (false, .valid): .falsePositive // PureXML rejects a valid instance (cardinal-rule blocker)
case (true, .invalid): .falseNegative // PureXML accepts an invalid instance
case (true, _): .agreeValid
case (false, _): .agreeInvalid
}
}
private func libxml2(_ schemaFile: String, _ instance: String, xmllint: String) -> Outcome? {
#if os(WASI)
return nil
@ -70,18 +58,25 @@ struct InstanceValidationDifferentialTests {
var agreeInvalid = 0
var falsePositives: [String] = []
var falseNegatives: [String] = []
/// Instances where libxml2 disagrees with the independent oracle (libxml2's
/// own content-validation leniency), tracked but never blamed on PureXML.
var libxml2Disagreements = 0
}
/// A schema both engines compile: the PureXML document, its source, and the file
/// written for libxml2.
/// A schema both engines compile: the PureXML document, its independent content
/// model (the oracle's ground truth), its source, and the file written for libxml2.
private struct CompiledSchema {
let document: PureXML.Schema.Document
let model: ContentModel
let source: String
let file: String
}
/// Validate several generated instances of one both-compile schema with both engines,
/// folding the outcomes into `tally`.
/// Validate several generated instances of one both-compile schema. Validity is
/// decided by the independent ``ContentModelOracle`` (the generated instances are
/// empty `xs:string` elements, so an instance is valid iff its element-name
/// sequence is in the model's language); libxml2 is a secondary cross-check whose
/// disagreements are counted but not attributed to PureXML.
private func compareInstances(
_ schema: CompiledSchema,
xmllint: String,
@ -90,16 +85,18 @@ struct InstanceValidationDifferentialTests {
) {
for _ in 0 ..< 8 {
let instance = generator.instance()
let pureValid = ((try? schema.document.validate(instance)) ?? [PureXML.Validation.ValidationError]()).isEmpty
guard let reference = libxml2(schema.file, instance, xmllint: xmllint) else { continue }
let pureValid = ((try? schema.document.validate(instance.xml)) ?? [PureXML.Validation.ValidationError]()).isEmpty
guard let reference = libxml2(schema.file, instance.xml, xmllint: xmllint) else { continue }
if reference == .schemaInvalid { return } // libxml2 rejects the schema; skip its instances
let truth = ContentModelOracle.matches(schema.model, instance.tokens)
tally.compared += 1
let detail = "\(schema.source)\n instance: \(instance)"
switch compare(pureValid: pureValid, reference: reference) {
case .falsePositive: if tally.falsePositives.count < 20 { tally.falsePositives.append(detail) }
case .falseNegative: if tally.falseNegatives.count < 20 { tally.falseNegatives.append(detail) }
case .agreeValid: tally.agreeValid += 1
case .agreeInvalid: tally.agreeInvalid += 1
if (reference == .valid) != truth { tally.libxml2Disagreements += 1 }
let detail = "\(schema.source)\n instance: \(instance.xml)\n oracle: \(truth ? "valid" : "invalid"), libxml2: \(reference)"
switch (pureValid, truth) {
case (false, true): if tally.falsePositives.count < 20 { tally.falsePositives.append(detail) }
case (true, false): if tally.falseNegatives.count < 20 { tally.falseNegatives.append(detail) }
case (true, true): tally.agreeValid += 1
case (false, false): tally.agreeInvalid += 1
}
}
}
@ -112,19 +109,22 @@ struct InstanceValidationDifferentialTests {
var tally = Tally()
let schemaFile = NSTemporaryDirectory() + "inst-diff-schema.xsd"
for _ in 0 ..< 1500 {
let schema = generator.schema()
guard let document = try? PureXML.Schema.Document(schema) else { continue } // PureXML compiles
guard (try? schema.write(toFile: schemaFile, atomically: true, encoding: .utf8)) != nil else { continue }
let generated = generator.schema()
guard let document = try? PureXML.Schema.Document(generated.xml) else { continue } // PureXML compiles
guard (try? generated.xml.write(toFile: schemaFile, atomically: true, encoding: .utf8)) != nil else { continue }
defer { try? FileManager.default.removeItem(atPath: schemaFile) }
let compiled = CompiledSchema(document: document, source: schema, file: schemaFile)
let compiled = CompiledSchema(document: document, model: generated.model, source: generated.xml, file: schemaFile)
compareInstances(compiled, xmllint: xmllint, generator: &generator, into: &tally)
}
let report = "compared \(tally.compared) instances (agree-valid \(tally.agreeValid), agree-invalid \(tally.agreeInvalid));"
+ " false positives \(tally.falsePositives.count), false negatives \(tally.falseNegatives.count)"
+ " false positives \(tally.falsePositives.count), false negatives \(tally.falseNegatives.count),"
+ " libxml2 disagreements with the oracle \(tally.libxml2Disagreements)"
FileHandle.standardError.write(Data("instance-diff: \(report)\n".utf8))
try? (tally.falsePositives + ["--- false negatives ---"] + tally.falseNegatives).joined(separator: "\n\n")
.write(toFile: NSTemporaryDirectory() + "instance-diff.txt", atomically: true, encoding: .utf8)
// The cardinal rule: PureXML must never reject a valid instance that libxml2 accepts.
// The cardinal rule, adjudicated against the independent oracle (not libxml2,
// whose content validation is itself lenient): PureXML must never reject an
// instance that is genuinely valid.
#expect(
tally.falsePositives.isEmpty,
"\(tally.falsePositives.count) instance-validation false positives (see \(NSTemporaryDirectory())instance-diff.txt): \(tally.falsePositives.first ?? "")",
@ -138,8 +138,23 @@ struct InstanceValidationDifferentialTests {
/// fair fraction are valid. Kept structurally well-formed so a rejection is a content
/// rejection, not a parse error.
private struct InstanceSchemaGenerator {
/// A generated content model paired with its independent ``ContentModel``, so a
/// disagreement can be adjudicated against the oracle rather than libxml2 alone.
struct Particle {
let xml: String
let model: ContentModel
}
/// A generated occurrence: the attribute text plus its parsed bounds.
private struct Occurrence {
let xml: String
let min: Int
let max: Int?
}
private var rng: InstanceRNG
private var groupNames: [String] = []
private var groupModels: [String: ContentModel] = [:]
private static let alphabet = ["a", "b", "c"]
init(seed: UInt64) {
@ -154,58 +169,84 @@ private struct InstanceSchemaGenerator {
Int.random(in: 0 ..< bound, using: &rng) == 0
}
private mutating func occurrence() -> String {
private mutating func occurrence() -> Occurrence {
switch Int.random(in: 0 ..< 5, using: &rng) {
case 0: ""
case 1: " minOccurs=\"0\""
case 2: " maxOccurs=\"unbounded\""
case 3: " minOccurs=\"0\" maxOccurs=\"unbounded\""
default: " maxOccurs=\"3\""
case 0: Occurrence(xml: "", min: 1, max: 1)
case 1: Occurrence(xml: " minOccurs=\"0\"", min: 0, max: 1)
case 2: Occurrence(xml: " maxOccurs=\"unbounded\"", min: 1, max: nil)
case 3: Occurrence(xml: " minOccurs=\"0\" maxOccurs=\"unbounded\"", min: 0, max: nil)
default: Occurrence(xml: " maxOccurs=\"3\"", min: 1, max: 3)
}
}
private mutating func particle(depth: Int) -> String {
/// Wraps a base model in an occurrence, leaving an exact-once occurrence bare.
private func occurring(_ base: ContentModel, _ occ: Occurrence) -> ContentModel {
occ.min == 1 && occ.max == 1 ? base : .occurs(base, occ.min, occ.max)
}
private mutating func particle(depth: Int) -> Particle {
if depth == 0 || chance(2) {
if chance(5) { return "<xs:any processContents=\"skip\"\(occurrence())/>" }
return "<xs:element name=\"\(pick(Self.alphabet))\" type=\"xs:string\"\(occurrence())/>"
let occ = occurrence()
if chance(5) {
return Particle(xml: "<xs:any processContents=\"skip\"\(occ.xml)/>", model: occurring(.any, occ))
}
let name = pick(Self.alphabet)
return Particle(xml: "<xs:element name=\"\(name)\" type=\"xs:string\"\(occ.xml)/>", model: occurring(.element(name), occ))
}
if !groupNames.isEmpty, chance(3) {
return "<xs:group ref=\"\(pick(groupNames))\"\(occurrence())/>"
let name = pick(groupNames)
let occ = occurrence()
return Particle(xml: "<xs:group ref=\"\(name)\"\(occ.xml)/>", model: occurring(groupModels[name] ?? .sequence([]), occ))
}
let compositor = pick(["sequence", "choice", "all"])
let count = Int.random(in: 1 ... 3, using: &rng)
if compositor == "all" {
let members = (0 ..< count).map { _ in "<xs:element name=\"\(pick(Self.alphabet))\"\(chance(2) ? " minOccurs=\"0\"" : "")/>" }
return "<xs:all>\(members.joined())</xs:all>"
var xml = ""
var members: [ContentModel] = []
for _ in 0 ..< count {
let name = pick(Self.alphabet)
let optional = chance(2)
xml += "<xs:element name=\"\(name)\"\(optional ? " minOccurs=\"0\"" : "")/>"
members.append(.occurs(.element(name), optional ? 0 : 1, 1))
}
return Particle(xml: "<xs:all>\(xml)</xs:all>", model: .all(members))
}
let members = (0 ..< count).map { _ in particle(depth: depth - 1) }.joined()
return "<xs:\(compositor)\(occurrence())>\(members)</xs:\(compositor)>"
let occ = occurrence()
let parts = (0 ..< count).map { _ in particle(depth: depth - 1) }
let base: ContentModel = compositor == "sequence" ? .sequence(parts.map(\.model)) : .choice(parts.map(\.model))
return Particle(xml: "<xs:\(compositor)\(occ.xml)>\(parts.map(\.xml).joined())</xs:\(compositor)>", model: occurring(base, occ))
}
private func wrapModel(_ particle: String) -> String {
if particle.hasPrefix("<xs:sequence") || particle.hasPrefix("<xs:choice") || particle.hasPrefix("<xs:all") {
/// The doc content model is a single compositor; a bare particle is wrapped in a
/// sequence in both the XML and the model so the two stay in lockstep.
private func wrapModel(_ particle: Particle) -> Particle {
if particle.xml.hasPrefix("<xs:sequence") || particle.xml.hasPrefix("<xs:choice") || particle.xml.hasPrefix("<xs:all") {
return particle
}
return "<xs:sequence>\(particle)</xs:sequence>"
return Particle(xml: "<xs:sequence>\(particle.xml)</xs:sequence>", model: .sequence([particle.model]))
}
mutating func schema() -> String {
mutating func schema() -> (xml: String, model: ContentModel) {
groupNames = []
groupModels = [:]
var groups = ""
for index in 0 ..< Int.random(in: 0 ... 2, using: &rng) {
let name = "g\(index)"
let body = wrapModel(particle(depth: 1))
groupNames.append(name)
groups += "<xs:group name=\"\(name)\">\(wrapModel(particle(depth: 1)))</xs:group>"
groupModels[name] = body.model
groups += "<xs:group name=\"\(name)\">\(body.xml)</xs:group>"
}
let body = wrapModel(particle(depth: 3))
return "<xs:schema xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">\(groups)"
+ "<xs:element name=\"doc\"><xs:complexType>\(body)</xs:complexType></xs:element></xs:schema>"
let xml = "<xs:schema xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">\(groups)"
+ "<xs:element name=\"doc\"><xs:complexType>\(body.xml)</xs:complexType></xs:element></xs:schema>"
return (xml, body.model)
}
mutating func instance() -> String {
mutating func instance() -> (xml: String, tokens: [String]) {
let count = Int.random(in: 0 ... 6, using: &rng)
let children = (0 ..< count).map { _ in "<\(pick(Self.alphabet))/>" }.joined()
return "<doc>\(children)</doc>"
let tokens = (0 ..< count).map { _ in pick(Self.alphabet) }
return ("<doc>\(tokens.map { "<\($0)/>" }.joined())</doc>", tokens)
}
}