Mihaela Mihaljevic
380160cc2b
The two monolithic XSD content/identity rules only satisfied the OpenAPIKit validation idiom at the wrong granularity: no per-constraint removability, no isolable failing/near-miss tests. Split them into 32 named rules, each finding tagged at its origin and cached per XSDContext so the walk still runs once per document (the #101 decision against duplicating XSD's recursive type resolution still holds). Wired the streaming path's ID/IDREF and identity checks through the same framework instead of calling the validators imperatively, and added the unwrapAndLookup combinator the rule already named as required. Also includes the mandated repo-wide swiftformat/swiftlint pass.
288 lines
14 KiB
Swift
288 lines
14 KiB
Swift
import Foundation
|
|
import Testing
|
|
@testable import PureXML
|
|
|
|
/// A generated-value differential for simple-type FACET validation against libxml2
|
|
/// (`xmllint --schema`), the reference. The content-model matcher and the instance
|
|
/// content validator have their own differentials; this one exercises the datatype and
|
|
/// facet value space, the largest correctness surface the others do not touch: for each
|
|
/// generated simple type that restricts a built-in by a subset of its facets, it
|
|
/// validates a pool of generated `<v>` values (chosen at and around the facet
|
|
/// boundaries) with PureXML and with libxml2 and flags every disagreement.
|
|
///
|
|
/// Opt-in (needs `/usr/bin/xmllint`, set `FACET_DIFF=1`); never runs in plain
|
|
/// `swift test` or CI. A PureXML-rejects / libxml2-accepts disagreement is a FALSE
|
|
/// POSITIVE (the cardinal rule: never reject a valid value); a PureXML-accepts /
|
|
/// libxml2-rejects is a false negative to triage. Only schemas BOTH engines compile are
|
|
/// used, so a disagreement is about value validity, not facet-combination validity.
|
|
///
|
|
/// Result (~10k values across xs:int/decimal/double/string/token/boolean/date/dateTime/
|
|
/// nonNegativeInteger restricted by min/maxInclusive/Exclusive, length/min/maxLength,
|
|
/// totalDigits/fractionDigits, pattern, enumeration, whiteSpace): it FOUND one real false
|
|
/// positive, a value of `NaN` rejected by an `xs:double` `enumeration` listing `NaN`
|
|
/// (the comparison used IEEE equality, under which `NaN != NaN`), now fixed in
|
|
/// `OrderedValue.==`; after the fix, ZERO false positives. The remaining disagreements are
|
|
/// a contested datatype reading, not a bug to chase: a value with surrounding whitespace
|
|
/// against a numeric type (e.g. `<v> 7 </v>` against `xs:int`) is valid per the XSD spec,
|
|
/// since the numeric types carry a fixed `whiteSpace="collapse"` that trims it before the
|
|
/// value check; PureXML accepts it (matching the spec and Xerces) where this libxml2 build
|
|
/// is stricter, so rejecting it would itself be a false positive. (`NaN` against a co-
|
|
/// occurring range facet is the dual contested case, excluded from the generator below.)
|
|
@Suite("Facet-validation differential vs libxml2 (opt-in via FACET_DIFF)")
|
|
struct FacetValidationDifferentialTests {
|
|
private var enabled: Bool {
|
|
ProcessInfo.processInfo.environment["FACET_DIFF"] != nil
|
|
}
|
|
|
|
private var xmllint: String? {
|
|
let path = ProcessInfo.processInfo.environment["XMLLINT"].flatMap { $0.isEmpty ? nil : $0 } ?? "/usr/bin/xmllint"
|
|
return FileManager.default.isExecutableFile(atPath: path) ? path : nil
|
|
}
|
|
|
|
private enum Outcome: Equatable { case schemaInvalid, valid, invalid }
|
|
private enum Comparison { case falsePositive, falseNegative, agreeValid, agreeInvalid }
|
|
|
|
private func compare(pureValid: Bool, reference: Outcome) -> Comparison {
|
|
switch (pureValid, reference) {
|
|
case (false, .valid): .falsePositive // PureXML rejects a value libxml2 accepts (cardinal-rule blocker)
|
|
case (true, .invalid): .falseNegative // PureXML accepts a value libxml2 rejects
|
|
case (true, _): .agreeValid
|
|
case (false, _): .agreeInvalid
|
|
}
|
|
}
|
|
|
|
private func libxml2(_ schemaFile: String, _ instance: String, xmllint: String) -> Outcome? {
|
|
#if os(WASI)
|
|
return nil
|
|
#else
|
|
let instanceFile = NSTemporaryDirectory() + "facet-diff-\(abs(instance.hashValue)).xml"
|
|
guard (try? instance.write(toFile: instanceFile, atomically: true, encoding: .utf8)) != nil else { return nil }
|
|
defer { try? FileManager.default.removeItem(atPath: instanceFile) }
|
|
let process = Process()
|
|
process.executableURL = URL(fileURLWithPath: xmllint)
|
|
process.arguments = ["--noout", "--schema", schemaFile, instanceFile]
|
|
let pipe = Pipe()
|
|
process.standardError = pipe
|
|
process.standardOutput = pipe
|
|
guard (try? process.run()) != nil else { return nil }
|
|
let data = pipe.fileHandleForReading.readDataToEndOfFile()
|
|
process.waitUntilExit()
|
|
let output = String(decoding: data, as: UTF8.self)
|
|
if output.contains("failed to compile") || output.contains("Schemas parser error") {
|
|
return .schemaInvalid
|
|
}
|
|
return process.terminationStatus == 0 ? .valid : .invalid
|
|
#endif
|
|
}
|
|
|
|
private struct Tally {
|
|
var compared = 0
|
|
var agreeValid = 0
|
|
var agreeInvalid = 0
|
|
var falsePositives: [String] = []
|
|
var falseNegatives: [String] = []
|
|
}
|
|
|
|
/// A schema both engines compile: the PureXML document, its source, and the file
|
|
/// written for libxml2.
|
|
private struct CompiledSchema {
|
|
let document: PureXML.Schema.Document
|
|
let source: String
|
|
let file: String
|
|
}
|
|
|
|
private func compareValues(
|
|
_ schema: CompiledSchema,
|
|
xmllint: String,
|
|
generator: inout FacetSchemaGenerator,
|
|
into tally: inout Tally,
|
|
) {
|
|
for value in generator.values() {
|
|
let instance = "<v>\(value)</v>"
|
|
let pureValid = ((try? schema.document.validate(instance)) ?? [PureXML.Validation.ValidationError]()).isEmpty
|
|
guard let reference = libxml2(schema.file, instance, xmllint: xmllint) else { continue }
|
|
if reference == .schemaInvalid {
|
|
return
|
|
} // libxml2 rejects the schema; skip its values
|
|
tally.compared += 1
|
|
let detail = "\(schema.source)\n value: '\(value)' (PureXML \(pureValid ? "valid" : "invalid"), libxml2 \(reference))"
|
|
switch compare(pureValid: pureValid, reference: reference) {
|
|
case .falsePositive: if tally.falsePositives.count < 25 {
|
|
tally.falsePositives.append(detail)
|
|
}
|
|
case .falseNegative: if tally.falseNegatives.count < 25 {
|
|
tally.falseNegatives.append(detail)
|
|
}
|
|
case .agreeValid: tally.agreeValid += 1
|
|
case .agreeInvalid: tally.agreeInvalid += 1
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test("PureXML's facet-validity verdict agrees with libxml2 over generated values")
|
|
func test_facetDifferential() {
|
|
guard enabled, let xmllint else { return }
|
|
let seed = ProcessInfo.processInfo.environment["DIFF_SEED"].flatMap { UInt64($0) } ?? 0xFACE_7D1F_F00D_C0DE
|
|
var generator = FacetSchemaGenerator(seed: seed)
|
|
var tally = Tally()
|
|
let schemaFile = NSTemporaryDirectory() + "facet-diff-schema.xsd"
|
|
for _ in 0 ..< 2000 {
|
|
let schema = generator.schema()
|
|
guard let document = try? PureXML.Schema.Document(schema) else { continue } // PureXML compiles
|
|
guard (try? schema.write(toFile: schemaFile, atomically: true, encoding: .utf8)) != nil else { continue }
|
|
defer { try? FileManager.default.removeItem(atPath: schemaFile) }
|
|
let compiled = CompiledSchema(document: document, source: schema, file: schemaFile)
|
|
compareValues(compiled, xmllint: xmllint, generator: &generator, into: &tally)
|
|
}
|
|
let report = "compared \(tally.compared) values (agree-valid \(tally.agreeValid), agree-invalid \(tally.agreeInvalid));"
|
|
+ " false positives \(tally.falsePositives.count), false negatives \(tally.falseNegatives.count)"
|
|
FileHandle.standardError.write(Data("facet-diff: \(report)\n".utf8))
|
|
try? (tally.falsePositives + ["--- false negatives ---"] + tally.falseNegatives).joined(separator: "\n\n")
|
|
.write(toFile: NSTemporaryDirectory() + "facet-diff.txt", atomically: true, encoding: .utf8)
|
|
// The cardinal rule: PureXML must never reject a value that libxml2 accepts.
|
|
#expect(
|
|
tally.falsePositives.isEmpty,
|
|
"\(tally.falsePositives.count) facet-validation false positives (see \(NSTemporaryDirectory())facet-diff.txt): \(tally.falsePositives.first ?? "")",
|
|
)
|
|
}
|
|
}
|
|
|
|
/// One built-in datatype with a pool of candidate facets (each a name and a value, drawn
|
|
/// to sit at and around the boundaries of the value pool) and a pool of `<v>` values
|
|
/// (valid, invalid-lexical, and boundary), so a generated restriction stresses real
|
|
/// facet/value interactions. The facet and value pools are aligned by hand so a boundary
|
|
/// off-by-one surfaces.
|
|
private struct DatatypeProfile {
|
|
let base: String
|
|
let facets: [(name: String, value: String)]
|
|
let values: [String]
|
|
}
|
|
|
|
/// Generates a `<v>` element whose simple type restricts a built-in by a random subset of
|
|
/// its facets, and matching `<v>value</v>` instances drawn from the same profile, so a
|
|
/// fair fraction are valid and a rejection is a value rejection, not a parse error.
|
|
private struct FacetSchemaGenerator {
|
|
private var rng: FacetRNG
|
|
private var current = FacetSchemaGenerator.profiles[0]
|
|
|
|
init(seed: UInt64) {
|
|
rng = FacetRNG(seed: seed)
|
|
}
|
|
|
|
static let profiles: [DatatypeProfile] = [
|
|
DatatypeProfile(
|
|
base: "xs:int",
|
|
facets: [
|
|
("minInclusive", "5"), ("maxInclusive", "20"), ("minExclusive", "4"), ("maxExclusive", "21"),
|
|
("totalDigits", "2"), ("enumeration", "10"), ("enumeration", "15"), ("pattern", "[0-9]+"),
|
|
],
|
|
values: ["5", "20", "4", "21", "10", "15", "0", "99", "100", "-3", "3", "abc", "", " 7 ", "+8", "007", "3.5"],
|
|
),
|
|
DatatypeProfile(
|
|
base: "xs:decimal",
|
|
facets: [
|
|
("minInclusive", "1.5"), ("maxInclusive", "9.5"), ("totalDigits", "3"), ("fractionDigits", "1"),
|
|
("enumeration", "2.5"), ("pattern", "[0-9.]+"),
|
|
],
|
|
values: ["1.5", "9.5", "1.4", "9.6", "2.5", "5.25", "10.5", "100", "abc", "", "3", "2.50", "-1.5"],
|
|
),
|
|
DatatypeProfile(
|
|
base: "xs:string",
|
|
facets: [
|
|
("minLength", "2"), ("maxLength", "6"), ("length", "4"), ("pattern", "[a-z]+"),
|
|
("enumeration", "abc"), ("enumeration", "hello"), ("whiteSpace", "preserve"),
|
|
],
|
|
values: ["abc", "hello", "a", "abcdef", "abcdefg", "ABC", "ab12", "", " sp ", "test", "te"],
|
|
),
|
|
DatatypeProfile(
|
|
base: "xs:token",
|
|
facets: [("maxLength", "5"), ("minLength", "2"), ("pattern", "[a-z ]+"), ("enumeration", "yes"), ("enumeration", "no")],
|
|
values: ["yes", "no", "hello", "abcdef", "a b", "ABC", "", " trim ", " a b "],
|
|
),
|
|
DatatypeProfile(
|
|
base: "xs:boolean",
|
|
facets: [("pattern", "true|false"), ("enumeration", "true"), ("enumeration", "1")],
|
|
values: ["true", "false", "1", "0", "yes", "", "TRUE", " true "],
|
|
),
|
|
DatatypeProfile(
|
|
base: "xs:date",
|
|
facets: [
|
|
("minInclusive", "2020-01-01"), ("maxInclusive", "2020-12-31"),
|
|
("pattern", "2020-.*"), ("enumeration", "2020-06-15"),
|
|
],
|
|
values: ["2020-01-01", "2020-12-31", "2019-12-31", "2021-01-01", "2020-06-15", "2020-13-01", "abc", "", "2020-6-15"],
|
|
),
|
|
DatatypeProfile(
|
|
base: "xs:nonNegativeInteger",
|
|
facets: [("minInclusive", "3"), ("maxInclusive", "30"), ("totalDigits", "2"), ("pattern", "[0-9]+")],
|
|
values: ["3", "30", "2", "31", "0", "-1", "99", "100", "abc", "", "10"],
|
|
),
|
|
DatatypeProfile(
|
|
base: "xs:double",
|
|
facets: [("minInclusive", "1.5"), ("maxInclusive", "100"), ("minExclusive", "0"), ("pattern", "[0-9.eE+-]+")],
|
|
values: ["1.5", "100", "1.4", "100.1", "1.0E2", "1e2", "-0", ".5", "50", "abc", "", "0", "-5"],
|
|
),
|
|
// `NaN`/`INF` are tested only against `enumeration` (value-space identity, the FP the
|
|
// differential caught), NOT range: `NaN` is incomparable for ordering, and whether
|
|
// an enumerated `NaN` then satisfies a co-occurring range facet is a contested
|
|
// reading (strict XSD excludes it; this libxml2 build admits it), so it is left out.
|
|
DatatypeProfile(
|
|
base: "xs:double",
|
|
facets: [("enumeration", "NaN"), ("enumeration", "INF"), ("enumeration", "3.14")],
|
|
values: ["NaN", "INF", "-INF", "3.14", "1.5", "abc", "", "0"],
|
|
),
|
|
DatatypeProfile(
|
|
base: "xs:dateTime",
|
|
facets: [("minInclusive", "2020-01-01T00:00:00"), ("maxInclusive", "2020-12-31T23:59:59"), ("pattern", "2020-.*")],
|
|
values: [
|
|
"2020-06-15T12:00:00", "2020-06-15T12:00:00Z", "2020-06-15T12:00:00+05:00", "2020-06-15T12:00:00-05:00",
|
|
"2020-01-01T00:00:00", "2019-12-31T23:59:59", "2021-01-01T00:00:00", "2020-13-01T00:00:00", "2020-06-15", "abc", "",
|
|
],
|
|
),
|
|
]
|
|
|
|
private mutating func pick<T>(_ options: [T]) -> T {
|
|
options[Int.random(in: 0 ..< options.count, using: &rng)]
|
|
}
|
|
|
|
mutating func schema() -> String {
|
|
current = pick(Self.profiles)
|
|
// A random non-empty subset of the profile's facets (a facet may repeat only for
|
|
// enumeration, which the pool already lists separately).
|
|
var chosen: [(name: String, value: String)] = []
|
|
var seenName: Set<String> = []
|
|
for facet in current.facets where Int.random(in: 0 ..< 2, using: &rng) == 0 {
|
|
if facet.name == "enumeration" || seenName.insert(facet.name).inserted {
|
|
chosen.append(facet)
|
|
}
|
|
}
|
|
if chosen.isEmpty {
|
|
chosen = [pick(current.facets)]
|
|
}
|
|
let restriction = chosen.map { "<xs:\($0.name) value=\"\($0.value)\"/>" }.joined()
|
|
return "<xs:schema xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">"
|
|
+ "<xs:element name=\"v\"><xs:simpleType><xs:restriction base=\"\(current.base)\">"
|
|
+ "\(restriction)</xs:restriction></xs:simpleType></xs:element></xs:schema>"
|
|
}
|
|
|
|
/// The instance values for the current schema's profile (all of them, so every value
|
|
/// is tried against every generated restriction).
|
|
func values() -> [String] {
|
|
current.values
|
|
}
|
|
}
|
|
|
|
private struct FacetRNG: RandomNumberGenerator {
|
|
private var state: UInt64
|
|
init(seed: UInt64) {
|
|
state = seed == 0 ? 0x9E37_79B9_7F4A_7C15 : seed
|
|
}
|
|
|
|
mutating func next() -> UInt64 {
|
|
state ^= state >> 12
|
|
state ^= state << 25
|
|
state ^= state >> 27
|
|
return state &* 0x2545_F491_4F6C_DD1D
|
|
}
|
|
}
|