PureXML/Tests/InstanceValidationDifferentialTests.swift

import Foundation
import Testing
@testable import PureXML

/// A generated-instance differential for content-model VALIDATION against libxml2
/// (`xmllint --schema`), the reference. The schema-compile determinism check has its own
/// differential; this one exercises the per-instance content matcher (the
/// graph-structured-stack `ContentRTNMatcher` now in production) over the open input
/// space: for each generated, both-compile schema it validates many generated `<doc>`
/// instances with PureXML and with libxml2 and flags every disagreement.
///
/// Opt-in (needs `/usr/bin/xmllint`, set `INSTANCE_DIFF=1`); never runs in plain
/// `swift test` or CI. A PureXML-rejects / libxml2-accepts disagreement is a FALSE
/// POSITIVE (the cardinal rule: never reject a valid instance); a PureXML-accepts /
/// libxml2-rejects is a false negative to triage. Only schemas BOTH engines compile are
/// used, so a disagreement is about instance validation, not schema compilation.
@Suite("Instance-validation differential vs libxml2 (opt-in via INSTANCE_DIFF)")
struct InstanceValidationDifferentialTests {
    private var enabled: Bool {
        ProcessInfo.processInfo.environment["INSTANCE_DIFF"] != nil
    }

    private var xmllint: String? {
        let path = ProcessInfo.processInfo.environment["XMLLINT"].flatMap { $0.isEmpty ? nil : $0 } ?? "/usr/bin/xmllint"
        return FileManager.default.isExecutableFile(atPath: path) ? path : nil
    }

    /// libxml2's verdict for `instance` against `schemaFile`: the schema failed to
    /// compile (skip), or the instance is valid / invalid.
    private enum Outcome: Equatable { case schemaInvalid, valid, invalid }

    /// How PureXML's verdict relates to the reference for one instance.
    private enum Comparison { case falsePositive, falseNegative, agreeValid, agreeInvalid }

    private func compare(pureValid: Bool, reference: Outcome) -> Comparison {
        switch (pureValid, reference) {
        case (false, .valid): .falsePositive // PureXML rejects a valid instance (cardinal-rule blocker)
        case (true, .invalid): .falseNegative // PureXML accepts an invalid instance
        case (true, _): .agreeValid
        case (false, _): .agreeInvalid
        }
    }

    private func libxml2(_ schemaFile: String, _ instance: String, xmllint: String) -> Outcome? {
        #if os(WASI)
            return nil
        #else
            let instanceFile = NSTemporaryDirectory() + "inst-diff-\(abs(instance.hashValue)).xml"
            guard (try? instance.write(toFile: instanceFile, atomically: true, encoding: .utf8)) != nil else { return nil }
            defer { try? FileManager.default.removeItem(atPath: instanceFile) }
            let process = Process()
            process.executableURL = URL(fileURLWithPath: xmllint)
            process.arguments = ["--noout", "--schema", schemaFile, instanceFile]
            let pipe = Pipe()
            process.standardError = pipe
            process.standardOutput = pipe
            guard (try? process.run()) != nil else { return nil }
            let data = pipe.fileHandleForReading.readDataToEndOfFile()
            process.waitUntilExit()
            let output = String(decoding: data, as: UTF8.self)
            if output.contains("failed to compile") || output.contains("Schemas parser error") { return .schemaInvalid }
            return process.terminationStatus == 0 ? .valid : .invalid
        #endif
    }

    /// The running comparison totals; the cardinal-rule check is `falsePositives.isEmpty`.
    private struct Tally {
        var compared = 0
        var agreeValid = 0
        var agreeInvalid = 0
        var falsePositives: [String] = []
        var falseNegatives: [String] = []
    }

    /// A schema both engines compile: the PureXML document, its source, and the file
    /// written for libxml2.
    private struct CompiledSchema {
        let document: PureXML.Schema.Document
        let source: String
        let file: String
    }

    /// Validate several generated instances of one both-compile schema with both engines,
    /// folding the outcomes into `tally`.
    private func compareInstances(
        _ schema: CompiledSchema,
        xmllint: String,
        generator: inout InstanceSchemaGenerator,
        into tally: inout Tally,
    ) {
        for _ in 0 ..< 8 {
            let instance = generator.instance()
            let pureValid = ((try? schema.document.validate(instance)) ?? [PureXML.Validation.ValidationError]()).isEmpty
            guard let reference = libxml2(schema.file, instance, xmllint: xmllint) else { continue }
            if reference == .schemaInvalid { return } // libxml2 rejects the schema; skip its instances
            tally.compared += 1
            let detail = "\(schema.source)\n  instance: \(instance)"
            switch compare(pureValid: pureValid, reference: reference) {
            case .falsePositive: if tally.falsePositives.count < 20 { tally.falsePositives.append(detail) }
            case .falseNegative: if tally.falseNegatives.count < 20 { tally.falseNegatives.append(detail) }
            case .agreeValid: tally.agreeValid += 1
            case .agreeInvalid: tally.agreeInvalid += 1
            }
        }
    }

    @Test("PureXML's instance-validity verdict agrees with libxml2 over generated documents")
    func test_instanceDifferential() {
        guard enabled, let xmllint else { return }
        let seed = ProcessInfo.processInfo.environment["DIFF_SEED"].flatMap { UInt64($0) } ?? 0x1A57_C0DE_5EED_BEEF
        var generator = InstanceSchemaGenerator(seed: seed)
        var tally = Tally()
        let schemaFile = NSTemporaryDirectory() + "inst-diff-schema.xsd"
        for _ in 0 ..< 1500 {
            let schema = generator.schema()
            guard let document = try? PureXML.Schema.Document(schema) else { continue } // PureXML compiles
            guard (try? schema.write(toFile: schemaFile, atomically: true, encoding: .utf8)) != nil else { continue }
            defer { try? FileManager.default.removeItem(atPath: schemaFile) }
            let compiled = CompiledSchema(document: document, source: schema, file: schemaFile)
            compareInstances(compiled, xmllint: xmllint, generator: &generator, into: &tally)
        }
        let report = "compared \(tally.compared) instances (agree-valid \(tally.agreeValid), agree-invalid \(tally.agreeInvalid));"
            + " false positives \(tally.falsePositives.count), false negatives \(tally.falseNegatives.count)"
        FileHandle.standardError.write(Data("instance-diff: \(report)\n".utf8))
        try? (tally.falsePositives + ["--- false negatives ---"] + tally.falseNegatives).joined(separator: "\n\n")
            .write(toFile: NSTemporaryDirectory() + "instance-diff.txt", atomically: true, encoding: .utf8)
        // The cardinal rule: PureXML must never reject a valid instance that libxml2 accepts.
        #expect(
            tally.falsePositives.isEmpty,
            "\(tally.falsePositives.count) instance-validation false positives (see \(NSTemporaryDirectory())instance-diff.txt): \(tally.falsePositives.first ?? "")",
        )
    }
}

/// Generates a `doc` element whose content model varies over the matcher's space
/// (sequences, choices, `all` groups, group references, occurrences, wildcards) using a
/// small element alphabet, and matching `<doc>` instances drawn from that alphabet so a
/// fair fraction are valid. Kept structurally well-formed so a rejection is a content
/// rejection, not a parse error.
private struct InstanceSchemaGenerator {
    private var rng: InstanceRNG
    private var groupNames: [String] = []
    private static let alphabet = ["a", "b", "c"]

    init(seed: UInt64) {
        rng = InstanceRNG(seed: seed)
    }

    private mutating func pick<T>(_ options: [T]) -> T {
        options[Int.random(in: 0 ..< options.count, using: &rng)]
    }

    private mutating func chance(_ bound: Int) -> Bool {
        Int.random(in: 0 ..< bound, using: &rng) == 0
    }

    private mutating func occurrence() -> String {
        switch Int.random(in: 0 ..< 5, using: &rng) {
        case 0: ""
        case 1: " minOccurs=\"0\""
        case 2: " maxOccurs=\"unbounded\""
        case 3: " minOccurs=\"0\" maxOccurs=\"unbounded\""
        default: " maxOccurs=\"3\""
        }
    }

    private mutating func particle(depth: Int) -> String {
        if depth == 0 || chance(2) {
            if chance(5) { return "<xs:any processContents=\"skip\"\(occurrence())/>" }
            return "<xs:element name=\"\(pick(Self.alphabet))\" type=\"xs:string\"\(occurrence())/>"
        }
        if !groupNames.isEmpty, chance(3) {
            return "<xs:group ref=\"\(pick(groupNames))\"\(occurrence())/>"
        }
        let compositor = pick(["sequence", "choice", "all"])
        let count = Int.random(in: 1 ... 3, using: &rng)
        if compositor == "all" {
            let members = (0 ..< count).map { _ in "<xs:element name=\"\(pick(Self.alphabet))\"\(chance(2) ? " minOccurs=\"0\"" : "")/>" }
            return "<xs:all>\(members.joined())</xs:all>"
        }
        let members = (0 ..< count).map { _ in particle(depth: depth - 1) }.joined()
        return "<xs:\(compositor)\(occurrence())>\(members)</xs:\(compositor)>"
    }

    private func wrapModel(_ particle: String) -> String {
        if particle.hasPrefix("<xs:sequence") || particle.hasPrefix("<xs:choice") || particle.hasPrefix("<xs:all") {
            return particle
        }
        return "<xs:sequence>\(particle)</xs:sequence>"
    }

    mutating func schema() -> String {
        groupNames = []
        var groups = ""
        for index in 0 ..< Int.random(in: 0 ... 2, using: &rng) {
            let name = "g\(index)"
            groupNames.append(name)
            groups += "<xs:group name=\"\(name)\">\(wrapModel(particle(depth: 1)))</xs:group>"
        }
        let body = wrapModel(particle(depth: 3))
        return "<xs:schema xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">\(groups)"
            + "<xs:element name=\"doc\"><xs:complexType>\(body)</xs:complexType></xs:element></xs:schema>"
    }

    mutating func instance() -> String {
        let count = Int.random(in: 0 ... 6, using: &rng)
        let children = (0 ..< count).map { _ in "<\(pick(Self.alphabet))/>" }.joined()
        return "<doc>\(children)</doc>"
    }
}

private struct InstanceRNG: RandomNumberGenerator {
    private var state: UInt64
    init(seed: UInt64) {
        state = seed == 0 ? 0x9E37_79B9_7F4A_7C15 : seed
    }

    mutating func next() -> UInt64 {
        state ^= state >> 12
        state ^= state << 25
        state ^= state >> 27
        return state &* 0x2545_F491_4F6C_DD1D
    }
}