Mihaela Mihaljevic
380160cc2b
The two monolithic XSD content/identity rules only satisfied the OpenAPIKit validation idiom at the wrong granularity: no per-constraint removability, no isolable failing/near-miss tests. Split them into 32 named rules, each finding tagged at its origin and cached per XSDContext so the walk still runs once per document (the #101 decision against duplicating XSD's recursive type resolution still holds). Wired the streaming path's ID/IDREF and identity checks through the same framework instead of calling the validators imperatively, and added the unwrapAndLookup combinator the rule already named as required. Also includes the mandated repo-wide swiftformat/swiftlint pass.
276 lines
13 KiB
Swift
276 lines
13 KiB
Swift
import Foundation
|
|
import Testing
|
|
@testable import PureXML
|
|
|
|
/// A generated-instance differential for content-model VALIDATION against libxml2
|
|
/// (`xmllint --schema`), the reference. The schema-compile determinism check has its own
|
|
/// differential; this one exercises the per-instance content matcher (the
|
|
/// graph-structured-stack `ContentRTNMatcher` now in production) over the open input
|
|
/// space: for each generated, both-compile schema it validates many generated `<doc>`
|
|
/// instances with PureXML and with libxml2 and flags every disagreement.
|
|
///
|
|
/// Opt-in (needs `/usr/bin/xmllint`, set `INSTANCE_DIFF=1`); never runs in plain
|
|
/// `swift test` or CI. A PureXML-rejects / libxml2-accepts disagreement is a FALSE
|
|
/// POSITIVE (the cardinal rule: never reject a valid instance); a PureXML-accepts /
|
|
/// libxml2-rejects is a false negative to triage. Only schemas BOTH engines compile are
|
|
/// used, so a disagreement is about instance validation, not schema compilation.
|
|
@Suite("Instance-validation differential vs libxml2 (opt-in via INSTANCE_DIFF)")
|
|
struct InstanceValidationDifferentialTests {
|
|
private var enabled: Bool {
|
|
ProcessInfo.processInfo.environment["INSTANCE_DIFF"] != nil
|
|
}
|
|
|
|
private var xmllint: String? {
|
|
let path = ProcessInfo.processInfo.environment["XMLLINT"].flatMap { $0.isEmpty ? nil : $0 } ?? "/usr/bin/xmllint"
|
|
return FileManager.default.isExecutableFile(atPath: path) ? path : nil
|
|
}
|
|
|
|
/// libxml2's verdict for `instance` against `schemaFile`: the schema failed to
|
|
/// compile (skip), or the instance is valid / invalid.
|
|
private enum Outcome: Equatable { case schemaInvalid, valid, invalid }
|
|
|
|
private func libxml2(_ schemaFile: String, _ instance: String, xmllint: String) -> Outcome? {
|
|
#if os(WASI)
|
|
return nil
|
|
#else
|
|
let instanceFile = NSTemporaryDirectory() + "inst-diff-\(abs(instance.hashValue)).xml"
|
|
guard (try? instance.write(toFile: instanceFile, atomically: true, encoding: .utf8)) != nil else { return nil }
|
|
defer { try? FileManager.default.removeItem(atPath: instanceFile) }
|
|
let process = Process()
|
|
process.executableURL = URL(fileURLWithPath: xmllint)
|
|
process.arguments = ["--noout", "--schema", schemaFile, instanceFile]
|
|
let pipe = Pipe()
|
|
process.standardError = pipe
|
|
process.standardOutput = pipe
|
|
guard (try? process.run()) != nil else { return nil }
|
|
let data = pipe.fileHandleForReading.readDataToEndOfFile()
|
|
process.waitUntilExit()
|
|
let output = String(decoding: data, as: UTF8.self)
|
|
if output.contains("failed to compile") || output.contains("Schemas parser error") {
|
|
return .schemaInvalid
|
|
}
|
|
return process.terminationStatus == 0 ? .valid : .invalid
|
|
#endif
|
|
}
|
|
|
|
/// The running comparison totals; the cardinal-rule check is `falsePositives.isEmpty`.
|
|
private struct Tally {
|
|
var compared = 0
|
|
var agreeValid = 0
|
|
var agreeInvalid = 0
|
|
var falsePositives: [String] = []
|
|
var falseNegatives: [String] = []
|
|
/// Instances where libxml2 disagrees with the independent oracle (libxml2's
|
|
/// own content-validation leniency), tracked but never blamed on PureXML.
|
|
var libxml2Disagreements = 0
|
|
}
|
|
|
|
/// A schema both engines compile: the PureXML document, its independent content
|
|
/// model (the oracle's ground truth), its source, and the file written for libxml2.
|
|
private struct CompiledSchema {
|
|
let document: PureXML.Schema.Document
|
|
let model: ContentModel
|
|
let source: String
|
|
let file: String
|
|
}
|
|
|
|
/// Validate several generated instances of one both-compile schema. Validity is
|
|
/// decided by the independent ``ContentModelOracle`` (the generated instances are
|
|
/// empty `xs:string` elements, so an instance is valid iff its element-name
|
|
/// sequence is in the model's language); libxml2 is a secondary cross-check whose
|
|
/// disagreements are counted but not attributed to PureXML.
|
|
private func compareInstances(
|
|
_ schema: CompiledSchema,
|
|
xmllint: String,
|
|
generator: inout InstanceSchemaGenerator,
|
|
into tally: inout Tally,
|
|
) {
|
|
for _ in 0 ..< 8 {
|
|
let instance = generator.instance()
|
|
let pureValid = ((try? schema.document.validate(instance.xml)) ?? [PureXML.Validation.ValidationError]()).isEmpty
|
|
guard let reference = libxml2(schema.file, instance.xml, xmllint: xmllint) else { continue }
|
|
if reference == .schemaInvalid {
|
|
return
|
|
} // libxml2 rejects the schema; skip its instances
|
|
let truth = ContentModelOracle.matches(schema.model, instance.tokens)
|
|
tally.compared += 1
|
|
if (reference == .valid) != truth {
|
|
tally.libxml2Disagreements += 1
|
|
}
|
|
let detail = "\(schema.source)\n instance: \(instance.xml)\n oracle: \(truth ? "valid" : "invalid"), libxml2: \(reference)"
|
|
switch (pureValid, truth) {
|
|
case (false, true): if tally.falsePositives.count < 20 {
|
|
tally.falsePositives.append(detail)
|
|
}
|
|
case (true, false): if tally.falseNegatives.count < 20 {
|
|
tally.falseNegatives.append(detail)
|
|
}
|
|
case (true, true): tally.agreeValid += 1
|
|
case (false, false): tally.agreeInvalid += 1
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test("PureXML's instance-validity verdict agrees with libxml2 over generated documents")
|
|
func test_instanceDifferential() {
|
|
guard enabled, let xmllint else { return }
|
|
let seed = ProcessInfo.processInfo.environment["DIFF_SEED"].flatMap { UInt64($0) } ?? 0x1A57_C0DE_5EED_BEEF
|
|
var generator = InstanceSchemaGenerator(seed: seed)
|
|
var tally = Tally()
|
|
let schemaFile = NSTemporaryDirectory() + "inst-diff-schema.xsd"
|
|
for _ in 0 ..< 1500 {
|
|
let generated = generator.schema()
|
|
guard let document = try? PureXML.Schema.Document(generated.xml) else { continue } // PureXML compiles
|
|
guard (try? generated.xml.write(toFile: schemaFile, atomically: true, encoding: .utf8)) != nil else { continue }
|
|
defer { try? FileManager.default.removeItem(atPath: schemaFile) }
|
|
let compiled = CompiledSchema(document: document, model: generated.model, source: generated.xml, file: schemaFile)
|
|
compareInstances(compiled, xmllint: xmllint, generator: &generator, into: &tally)
|
|
}
|
|
let report = "compared \(tally.compared) instances (agree-valid \(tally.agreeValid), agree-invalid \(tally.agreeInvalid));"
|
|
+ " false positives \(tally.falsePositives.count), false negatives \(tally.falseNegatives.count),"
|
|
+ " libxml2 disagreements with the oracle \(tally.libxml2Disagreements)"
|
|
FileHandle.standardError.write(Data("instance-diff: \(report)\n".utf8))
|
|
try? (tally.falsePositives + ["--- false negatives ---"] + tally.falseNegatives).joined(separator: "\n\n")
|
|
.write(toFile: NSTemporaryDirectory() + "instance-diff.txt", atomically: true, encoding: .utf8)
|
|
// The cardinal rule, adjudicated against the independent oracle (not libxml2,
|
|
// whose content validation is itself lenient): PureXML must never reject an
|
|
// instance that is genuinely valid.
|
|
#expect(
|
|
tally.falsePositives.isEmpty,
|
|
"\(tally.falsePositives.count) instance-validation false positives (see \(NSTemporaryDirectory())instance-diff.txt): \(tally.falsePositives.first ?? "")",
|
|
)
|
|
}
|
|
}
|
|
|
|
/// Generates a `doc` element whose content model varies over the matcher's space
|
|
/// (sequences, choices, `all` groups, group references, occurrences, wildcards) using a
|
|
/// small element alphabet, and matching `<doc>` instances drawn from that alphabet so a
|
|
/// fair fraction are valid. Kept structurally well-formed so a rejection is a content
|
|
/// rejection, not a parse error.
|
|
private struct InstanceSchemaGenerator {
|
|
/// A generated content model paired with its independent ``ContentModel``, so a
|
|
/// disagreement can be adjudicated against the oracle rather than libxml2 alone.
|
|
struct Particle {
|
|
let xml: String
|
|
let model: ContentModel
|
|
}
|
|
|
|
/// A generated occurrence: the attribute text plus its parsed bounds.
|
|
private struct Occurrence {
|
|
let xml: String
|
|
let min: Int
|
|
let max: Int?
|
|
}
|
|
|
|
private var rng: InstanceRNG
|
|
private var groupNames: [String] = []
|
|
private var groupModels: [String: ContentModel] = [:]
|
|
private static let alphabet = ["a", "b", "c"]
|
|
|
|
init(seed: UInt64) {
|
|
rng = InstanceRNG(seed: seed)
|
|
}
|
|
|
|
private mutating func pick<T>(_ options: [T]) -> T {
|
|
options[Int.random(in: 0 ..< options.count, using: &rng)]
|
|
}
|
|
|
|
private mutating func chance(_ bound: Int) -> Bool {
|
|
Int.random(in: 0 ..< bound, using: &rng) == 0
|
|
}
|
|
|
|
private mutating func occurrence() -> Occurrence {
|
|
switch Int.random(in: 0 ..< 5, using: &rng) {
|
|
case 0: Occurrence(xml: "", min: 1, max: 1)
|
|
case 1: Occurrence(xml: " minOccurs=\"0\"", min: 0, max: 1)
|
|
case 2: Occurrence(xml: " maxOccurs=\"unbounded\"", min: 1, max: nil)
|
|
case 3: Occurrence(xml: " minOccurs=\"0\" maxOccurs=\"unbounded\"", min: 0, max: nil)
|
|
default: Occurrence(xml: " maxOccurs=\"3\"", min: 1, max: 3)
|
|
}
|
|
}
|
|
|
|
/// Wraps a base model in an occurrence, leaving an exact-once occurrence bare.
|
|
private func occurring(_ base: ContentModel, _ occ: Occurrence) -> ContentModel {
|
|
occ.min == 1 && occ.max == 1 ? base : .occurs(base, occ.min, occ.max)
|
|
}
|
|
|
|
private mutating func particle(depth: Int) -> Particle {
|
|
if depth == 0 || chance(2) {
|
|
let occ = occurrence()
|
|
if chance(5) {
|
|
return Particle(xml: "<xs:any processContents=\"skip\"\(occ.xml)/>", model: occurring(.any, occ))
|
|
}
|
|
let name = pick(Self.alphabet)
|
|
return Particle(xml: "<xs:element name=\"\(name)\" type=\"xs:string\"\(occ.xml)/>", model: occurring(.element(name), occ))
|
|
}
|
|
if !groupNames.isEmpty, chance(3) {
|
|
let name = pick(groupNames)
|
|
let occ = occurrence()
|
|
return Particle(xml: "<xs:group ref=\"\(name)\"\(occ.xml)/>", model: occurring(groupModels[name] ?? .sequence([]), occ))
|
|
}
|
|
let compositor = pick(["sequence", "choice", "all"])
|
|
let count = Int.random(in: 1 ... 3, using: &rng)
|
|
if compositor == "all" {
|
|
var xml = ""
|
|
var members: [ContentModel] = []
|
|
for _ in 0 ..< count {
|
|
let name = pick(Self.alphabet)
|
|
let optional = chance(2)
|
|
xml += "<xs:element name=\"\(name)\"\(optional ? " minOccurs=\"0\"" : "")/>"
|
|
members.append(.occurs(.element(name), optional ? 0 : 1, 1))
|
|
}
|
|
return Particle(xml: "<xs:all>\(xml)</xs:all>", model: .all(members))
|
|
}
|
|
let occ = occurrence()
|
|
let parts = (0 ..< count).map { _ in particle(depth: depth - 1) }
|
|
let base: ContentModel = compositor == "sequence" ? .sequence(parts.map(\.model)) : .choice(parts.map(\.model))
|
|
return Particle(xml: "<xs:\(compositor)\(occ.xml)>\(parts.map(\.xml).joined())</xs:\(compositor)>", model: occurring(base, occ))
|
|
}
|
|
|
|
/// The doc content model is a single compositor; a bare particle is wrapped in a
|
|
/// sequence in both the XML and the model so the two stay in lockstep.
|
|
private func wrapModel(_ particle: Particle) -> Particle {
|
|
if particle.xml.hasPrefix("<xs:sequence") || particle.xml.hasPrefix("<xs:choice") || particle.xml.hasPrefix("<xs:all") {
|
|
return particle
|
|
}
|
|
return Particle(xml: "<xs:sequence>\(particle.xml)</xs:sequence>", model: .sequence([particle.model]))
|
|
}
|
|
|
|
mutating func schema() -> (xml: String, model: ContentModel) {
|
|
groupNames = []
|
|
groupModels = [:]
|
|
var groups = ""
|
|
for index in 0 ..< Int.random(in: 0 ... 2, using: &rng) {
|
|
let name = "g\(index)"
|
|
let body = wrapModel(particle(depth: 1))
|
|
groupNames.append(name)
|
|
groupModels[name] = body.model
|
|
groups += "<xs:group name=\"\(name)\">\(body.xml)</xs:group>"
|
|
}
|
|
let body = wrapModel(particle(depth: 3))
|
|
let xml = "<xs:schema xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">\(groups)"
|
|
+ "<xs:element name=\"doc\"><xs:complexType>\(body.xml)</xs:complexType></xs:element></xs:schema>"
|
|
return (xml, body.model)
|
|
}
|
|
|
|
mutating func instance() -> (xml: String, tokens: [String]) {
|
|
let count = Int.random(in: 0 ... 6, using: &rng)
|
|
let tokens = (0 ..< count).map { _ in pick(Self.alphabet) }
|
|
return ("<doc>\(tokens.map { "<\($0)/>" }.joined())</doc>", tokens)
|
|
}
|
|
}
|
|
|
|
private struct InstanceRNG: RandomNumberGenerator {
|
|
private var state: UInt64
|
|
init(seed: UInt64) {
|
|
state = seed == 0 ? 0x9E37_79B9_7F4A_7C15 : seed
|
|
}
|
|
|
|
mutating func next() -> UInt64 {
|
|
state ^= state >> 12
|
|
state ^= state << 25
|
|
state ^= state >> 27
|
|
return state &* 0x2545_F491_4F6C_DD1D
|
|
}
|
|
}
|