The schema-compile determinism check has a generated-schema differential; this adds the counterpart for the instance-validation path, exercising the per-instance content matcher (the graph-structured-stack ContentRTNMatcher now in production) over the open input space. For every generated, both-engines-compile schema it validates many generated <doc> instances with PureXML and with libxml2 and compares the verdicts, characterising the cardinal rule's instance half beyond the finite XSTS corpus. It asserts the cardinal rule (PureXML never rejects an instance libxml2 accepts) and reports false negatives for triage. Over 6568 generated validations (1357 both-valid, 5211 both-invalid) across sequences, choices, all groups, group references, occurrences, and wildcards it found zero false positives and zero false negatives: independent confirmation that the matcher agrees with libxml2. Opt-in via INSTANCE_DIFF and xmllint, a no-op in plain swift test and CI; the libxml2 subprocess is guarded for WASI. No behaviour change.
224 lines
10 KiB
Swift
224 lines
10 KiB
Swift
import Foundation
|
|
import Testing
|
|
@testable import PureXML
|
|
|
|
/// A generated-instance differential for content-model VALIDATION against libxml2
|
|
/// (`xmllint --schema`), the reference. The schema-compile determinism check has its own
|
|
/// differential; this one exercises the per-instance content matcher (the
|
|
/// graph-structured-stack `ContentRTNMatcher` now in production) over the open input
|
|
/// space: for each generated, both-compile schema it validates many generated `<doc>`
|
|
/// instances with PureXML and with libxml2 and flags every disagreement.
|
|
///
|
|
/// Opt-in (needs `/usr/bin/xmllint`, set `INSTANCE_DIFF=1`); never runs in plain
|
|
/// `swift test` or CI. A PureXML-rejects / libxml2-accepts disagreement is a FALSE
|
|
/// POSITIVE (the cardinal rule: never reject a valid instance); a PureXML-accepts /
|
|
/// libxml2-rejects is a false negative to triage. Only schemas BOTH engines compile are
|
|
/// used, so a disagreement is about instance validation, not schema compilation.
|
|
@Suite("Instance-validation differential vs libxml2 (opt-in via INSTANCE_DIFF)")
|
|
struct InstanceValidationDifferentialTests {
|
|
private var enabled: Bool {
|
|
ProcessInfo.processInfo.environment["INSTANCE_DIFF"] != nil
|
|
}
|
|
|
|
private var xmllint: String? {
|
|
let path = ProcessInfo.processInfo.environment["XMLLINT"].flatMap { $0.isEmpty ? nil : $0 } ?? "/usr/bin/xmllint"
|
|
return FileManager.default.isExecutableFile(atPath: path) ? path : nil
|
|
}
|
|
|
|
/// libxml2's verdict for `instance` against `schemaFile`: the schema failed to
|
|
/// compile (skip), or the instance is valid / invalid.
|
|
private enum Outcome: Equatable { case schemaInvalid, valid, invalid }
|
|
|
|
/// How PureXML's verdict relates to the reference for one instance.
|
|
private enum Comparison { case falsePositive, falseNegative, agreeValid, agreeInvalid }
|
|
|
|
private func compare(pureValid: Bool, reference: Outcome) -> Comparison {
|
|
switch (pureValid, reference) {
|
|
case (false, .valid): .falsePositive // PureXML rejects a valid instance (cardinal-rule blocker)
|
|
case (true, .invalid): .falseNegative // PureXML accepts an invalid instance
|
|
case (true, _): .agreeValid
|
|
case (false, _): .agreeInvalid
|
|
}
|
|
}
|
|
|
|
private func libxml2(_ schemaFile: String, _ instance: String, xmllint: String) -> Outcome? {
|
|
#if os(WASI)
|
|
return nil
|
|
#else
|
|
let instanceFile = NSTemporaryDirectory() + "inst-diff-\(abs(instance.hashValue)).xml"
|
|
guard (try? instance.write(toFile: instanceFile, atomically: true, encoding: .utf8)) != nil else { return nil }
|
|
defer { try? FileManager.default.removeItem(atPath: instanceFile) }
|
|
let process = Process()
|
|
process.executableURL = URL(fileURLWithPath: xmllint)
|
|
process.arguments = ["--noout", "--schema", schemaFile, instanceFile]
|
|
let pipe = Pipe()
|
|
process.standardError = pipe
|
|
process.standardOutput = pipe
|
|
guard (try? process.run()) != nil else { return nil }
|
|
let data = pipe.fileHandleForReading.readDataToEndOfFile()
|
|
process.waitUntilExit()
|
|
let output = String(decoding: data, as: UTF8.self)
|
|
if output.contains("failed to compile") || output.contains("Schemas parser error") { return .schemaInvalid }
|
|
return process.terminationStatus == 0 ? .valid : .invalid
|
|
#endif
|
|
}
|
|
|
|
/// The running comparison totals; the cardinal-rule check is `falsePositives.isEmpty`.
|
|
private struct Tally {
|
|
var compared = 0
|
|
var agreeValid = 0
|
|
var agreeInvalid = 0
|
|
var falsePositives: [String] = []
|
|
var falseNegatives: [String] = []
|
|
}
|
|
|
|
/// A schema both engines compile: the PureXML document, its source, and the file
|
|
/// written for libxml2.
|
|
private struct CompiledSchema {
|
|
let document: PureXML.Schema.Document
|
|
let source: String
|
|
let file: String
|
|
}
|
|
|
|
/// Validate several generated instances of one both-compile schema with both engines,
|
|
/// folding the outcomes into `tally`.
|
|
private func compareInstances(
|
|
_ schema: CompiledSchema,
|
|
xmllint: String,
|
|
generator: inout InstanceSchemaGenerator,
|
|
into tally: inout Tally,
|
|
) {
|
|
for _ in 0 ..< 8 {
|
|
let instance = generator.instance()
|
|
let pureValid = ((try? schema.document.validate(instance)) ?? [PureXML.Validation.ValidationError]()).isEmpty
|
|
guard let reference = libxml2(schema.file, instance, xmllint: xmllint) else { continue }
|
|
if reference == .schemaInvalid { return } // libxml2 rejects the schema; skip its instances
|
|
tally.compared += 1
|
|
let detail = "\(schema.source)\n instance: \(instance)"
|
|
switch compare(pureValid: pureValid, reference: reference) {
|
|
case .falsePositive: if tally.falsePositives.count < 20 { tally.falsePositives.append(detail) }
|
|
case .falseNegative: if tally.falseNegatives.count < 20 { tally.falseNegatives.append(detail) }
|
|
case .agreeValid: tally.agreeValid += 1
|
|
case .agreeInvalid: tally.agreeInvalid += 1
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test("PureXML's instance-validity verdict agrees with libxml2 over generated documents")
|
|
func test_instanceDifferential() {
|
|
guard enabled, let xmllint else { return }
|
|
var generator = InstanceSchemaGenerator(seed: 0x1A57_C0DE_5EED_BEEF)
|
|
var tally = Tally()
|
|
let schemaFile = NSTemporaryDirectory() + "inst-diff-schema.xsd"
|
|
for _ in 0 ..< 1500 {
|
|
let schema = generator.schema()
|
|
guard let document = try? PureXML.Schema.Document(schema) else { continue } // PureXML compiles
|
|
guard (try? schema.write(toFile: schemaFile, atomically: true, encoding: .utf8)) != nil else { continue }
|
|
defer { try? FileManager.default.removeItem(atPath: schemaFile) }
|
|
let compiled = CompiledSchema(document: document, source: schema, file: schemaFile)
|
|
compareInstances(compiled, xmllint: xmllint, generator: &generator, into: &tally)
|
|
}
|
|
let report = "compared \(tally.compared) instances (agree-valid \(tally.agreeValid), agree-invalid \(tally.agreeInvalid));"
|
|
+ " false positives \(tally.falsePositives.count), false negatives \(tally.falseNegatives.count)"
|
|
FileHandle.standardError.write(Data("instance-diff: \(report)\n".utf8))
|
|
try? (tally.falsePositives + ["--- false negatives ---"] + tally.falseNegatives).joined(separator: "\n\n")
|
|
.write(toFile: NSTemporaryDirectory() + "instance-diff.txt", atomically: true, encoding: .utf8)
|
|
// The cardinal rule: PureXML must never reject a valid instance that libxml2 accepts.
|
|
#expect(
|
|
tally.falsePositives.isEmpty,
|
|
"\(tally.falsePositives.count) instance-validation false positives (see \(NSTemporaryDirectory())instance-diff.txt): \(tally.falsePositives.first ?? "")",
|
|
)
|
|
}
|
|
}
|
|
|
|
/// Generates a `doc` element whose content model varies over the matcher's space
|
|
/// (sequences, choices, `all` groups, group references, occurrences, wildcards) using a
|
|
/// small element alphabet, and matching `<doc>` instances drawn from that alphabet so a
|
|
/// fair fraction are valid. Kept structurally well-formed so a rejection is a content
|
|
/// rejection, not a parse error.
|
|
private struct InstanceSchemaGenerator {
|
|
private var rng: InstanceRNG
|
|
private var groupNames: [String] = []
|
|
private static let alphabet = ["a", "b", "c"]
|
|
|
|
init(seed: UInt64) {
|
|
rng = InstanceRNG(seed: seed)
|
|
}
|
|
|
|
private mutating func pick<T>(_ options: [T]) -> T {
|
|
options[Int.random(in: 0 ..< options.count, using: &rng)]
|
|
}
|
|
|
|
private mutating func chance(_ bound: Int) -> Bool {
|
|
Int.random(in: 0 ..< bound, using: &rng) == 0
|
|
}
|
|
|
|
private mutating func occurrence() -> String {
|
|
switch Int.random(in: 0 ..< 5, using: &rng) {
|
|
case 0: ""
|
|
case 1: " minOccurs=\"0\""
|
|
case 2: " maxOccurs=\"unbounded\""
|
|
case 3: " minOccurs=\"0\" maxOccurs=\"unbounded\""
|
|
default: " maxOccurs=\"3\""
|
|
}
|
|
}
|
|
|
|
private mutating func particle(depth: Int) -> String {
|
|
if depth == 0 || chance(2) {
|
|
if chance(5) { return "<xs:any processContents=\"skip\"\(occurrence())/>" }
|
|
return "<xs:element name=\"\(pick(Self.alphabet))\" type=\"xs:string\"\(occurrence())/>"
|
|
}
|
|
if !groupNames.isEmpty, chance(3) {
|
|
return "<xs:group ref=\"\(pick(groupNames))\"\(occurrence())/>"
|
|
}
|
|
let compositor = pick(["sequence", "choice", "all"])
|
|
let count = Int.random(in: 1 ... 3, using: &rng)
|
|
if compositor == "all" {
|
|
let members = (0 ..< count).map { _ in "<xs:element name=\"\(pick(Self.alphabet))\"\(chance(2) ? " minOccurs=\"0\"" : "")/>" }
|
|
return "<xs:all>\(members.joined())</xs:all>"
|
|
}
|
|
let members = (0 ..< count).map { _ in particle(depth: depth - 1) }.joined()
|
|
return "<xs:\(compositor)\(occurrence())>\(members)</xs:\(compositor)>"
|
|
}
|
|
|
|
private func wrapModel(_ particle: String) -> String {
|
|
if particle.hasPrefix("<xs:sequence") || particle.hasPrefix("<xs:choice") || particle.hasPrefix("<xs:all") {
|
|
return particle
|
|
}
|
|
return "<xs:sequence>\(particle)</xs:sequence>"
|
|
}
|
|
|
|
mutating func schema() -> String {
|
|
groupNames = []
|
|
var groups = ""
|
|
for index in 0 ..< Int.random(in: 0 ... 2, using: &rng) {
|
|
let name = "g\(index)"
|
|
groupNames.append(name)
|
|
groups += "<xs:group name=\"\(name)\">\(wrapModel(particle(depth: 1)))</xs:group>"
|
|
}
|
|
let body = wrapModel(particle(depth: 3))
|
|
return "<xs:schema xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">\(groups)"
|
|
+ "<xs:element name=\"doc\"><xs:complexType>\(body)</xs:complexType></xs:element></xs:schema>"
|
|
}
|
|
|
|
mutating func instance() -> String {
|
|
let count = Int.random(in: 0 ... 6, using: &rng)
|
|
let children = (0 ..< count).map { _ in "<\(pick(Self.alphabet))/>" }.joined()
|
|
return "<doc>\(children)</doc>"
|
|
}
|
|
}
|
|
|
|
private struct InstanceRNG: RandomNumberGenerator {
|
|
private var state: UInt64
|
|
init(seed: UInt64) {
|
|
state = seed == 0 ? 0x9E37_79B9_7F4A_7C15 : seed
|
|
}
|
|
|
|
mutating func next() -> UInt64 {
|
|
state ^= state >> 12
|
|
state ^= state << 25
|
|
state ^= state >> 27
|
|
return state &* 0x2545_F491_4F6C_DD1D
|
|
}
|
|
}
|