PureXML/Tests/XSTSDifferentialTests.swift

import Foundation
import Testing
@testable import PureXML

/// Differential schema-validity harness (production-readiness stopper #3:
/// correctness must be characterised against an independent reference, not just a
/// finite labelled suite). For every `.xsd` under `XSTS_ROOT` it compares
/// PureXML's compile verdict with libxml2's (`xmllint --schema`) and writes the
/// disagreements to /tmp/xsts-differential.txt for triage. Opt-in (needs
/// `XSTS_ROOT` and `/usr/bin/xmllint`); never runs in plain `swift test` or CI.
///
/// IMPORTANT, established by triage: libxml2 is the LENIENT party. "PureXML
/// rejects, libxml2 accepts" is overwhelmingly PureXML being *more* correct --
/// enforcing facet consistency (minLength ≤ maxLength), fractionDigits ≤ base,
/// Element Declarations Consistent, whiteSpace restriction, au-props-correct, and
/// notation rules that libxml2 does not. So that direction is NOT a false-positive
/// signal and is not asserted on. The actionable direction for PureXML is the
/// OTHER one -- "PureXML accepts, libxml2 rejects" -- a triage queue of candidate
/// false negatives (stopper #2), each to be confirmed against the spec (libxml2
/// can also be over-strict) before fixing. The authoritative false-positive gate
/// remains the XSTS suite's W3C-labelled `valid-schemas-rejected` (== 0).
///
/// Triage of that queue (2026-06-28, the ~66 entries over msData): it is itself
/// dominated by libxml2 over-strictness and contested-version readings, NOT clean
/// PureXML false negatives. The categories found, each a deliberate-or-correct
/// PureXML choice, not a bug to fix:
///   - Contested datatype version: `gMonth` value `--MM--` is the original 2001
///     REC lexical form the errata replaced with `--MM`; libxml2 uses the errata,
///     XSTS the 2001 REC, and PureXML accepts `--MM--` precisely to keep valid
///     XSTS instances (the iter-2 "gMonth bonus") -- rejecting it would REGRESS
///     valid-instances-rejected. `anyURI` lexical strictness is the same shape.
///   - libxml2 implementation limit: a `maxOccurs` like `999999999999999` is a
///     valid unbounded-precision `nonNegativeInteger`; libxml2 rejects it on its
///     own machine-int limit. PureXML is CORRECT to accept (groupF/H/J/L/N009).
///   - External-document loading: libxml2 rejects when an `<include>/<import>`
///     `schemaLocation` fails to load; PureXML does not fetch externals here, so
///     it compiles the local document (anyURI_a001 and the test* import chains).
///   - Deliberate under-rejection: a regex `{,5}` quantifier is tolerated as a
///     `.badQuantifier` (iter 7) to avoid over-rejecting valid patterns the engine
///     under-supports (elemE*). The cardinal rule (no false positive) governs.
/// The small residue (a `base64Binary` value whose non-padding bits are non-zero,
/// `LMT=`; a `keyref` `refer` to a no-namespace component from a namespaced
/// schema, idC019) are FP-delicate datatype/identity-constraint candidates, not
/// yet adjudicated. So the actionable yield is small and each candidate must clear
/// this same adjudication; the queue is characterised, not a backlog of bugs.
///
/// xmllint and the corpus are used only in this test harness; neither is a
/// dependency of the shipped package.
@Suite("Schema-validity differential vs libxml2 (opt-in via XSTS_ROOT)")
struct XSTSDifferentialTests {
    private var root: String? {
        ProcessInfo.processInfo.environment["XSTS_ROOT"]
    }

    @Test("Characterise PureXML schema validity against libxml2")
    func test_differential() {
        guard let root, xmllintAvailable() else { return } // Opt-in.
        let schemas = xsdFiles(under: root + "/msData")
        var agreeValid = 0, agreeInvalid = 0
        var pureRejectsRefAccepts: [String] = []
        var pureAcceptsRefRejects: [String] = []

        for path in schemas {
            guard let source = try? String(contentsOfFile: path, encoding: .utf8) else { continue }
            let directory = (path as NSString).deletingLastPathComponent
            let loader: (String) -> String? = { location in
                try? String(contentsOfFile: (directory as NSString).appendingPathComponent(location), encoding: .utf8)
            }
            let pureValid = (try? PureXML.Schema.Document(source, schemaLoader: loader)) != nil
            guard let refValid = xmllintSchemaValid(path) else { continue } // skip if xmllint errored opaquely
            switch (pureValid, refValid) {
            case (true, true): agreeValid += 1
            case (false, false): agreeInvalid += 1
            case (false, true): pureRejectsRefAccepts.append(name(path, under: root))
            case (true, false): pureAcceptsRefRejects.append(name(path, under: root))
            }
        }

        let report = """
        differential vs libxml2 over \(schemas.count) schemas:
          agree valid:   \(agreeValid)
          agree invalid: \(agreeInvalid)
          PureXML stricter (rejects, libxml2 accepts; mostly PureXML-correct): \(pureRejectsRefAccepts.count)
          libxml2 stricter (PureXML accepts, libxml2 rejects; false-negative triage queue): \(pureAcceptsRefRejects.count)
        """
        FileHandle.standardError.write(Data((report + "\n").utf8))
        let lines = ["PureXML stricter than libxml2 (triage shows these are largely PureXML-correct):"]
            + pureRejectsRefAccepts.sorted().map { "  " + $0 }
            + ["", "libxml2 stricter than PureXML (candidate false negatives for stopper #2; confirm vs spec):"]
            + pureAcceptsRefRejects.sorted().map { "  " + $0 }
        try? lines.joined(separator: "\n").write(toFile: "/tmp/xsts-differential.txt", atomically: true, encoding: .utf8)

        // This is a characterisation tool: libxml2 is not ground truth (it is
        // lenient in one direction and can be over-strict in the other), so neither
        // disagreement count is a pass/fail correctness gate -- the report is the
        // deliverable. The only invariant asserted is that the harness actually ran
        // over a meaningful corpus and produced data to triage.
        #expect(agreeValid + agreeInvalid > 1000, "differential did not run over a meaningful corpus")
    }

    // MARK: - libxml2 oracle

    private func xmllintAvailable() -> Bool {
        #if os(WASI)
            return false
        #else
            FileManager.default.isExecutableFile(atPath: "/usr/bin/xmllint")
        #endif
    }

    /// libxml2's schema-validity verdict for one `.xsd`, or nil if it could not be
    /// determined. The schema is loaded via `--schema`; a compile failure prints
    /// `Schemas parser error` / `failed to compile`. The instance (`/dev/null`,
    /// empty) always fails to parse, which is ignored -- only schema-compile errors
    /// count toward schema validity.
    private func xmllintSchemaValid(_ path: String) -> Bool? {
        #if os(WASI)
            return nil
        #else
            let process = Process()
            process.executableURL = URL(fileURLWithPath: "/usr/bin/xmllint")
            process.arguments = ["--noout", "--schema", path, "/dev/null"]
            let pipe = Pipe()
            process.standardError = pipe
            process.standardOutput = Pipe()
            do { try process.run() } catch { return nil }
            let data = pipe.fileHandleForReading.readDataToEndOfFile()
            process.waitUntilExit()
            let output = String(decoding: data, as: UTF8.self)
            if output.contains("Schemas parser error") || output.contains("failed to compile") {
                return false
            }
            return true
        #endif
    }

    private func xsdFiles(under directory: String) -> [String] {
        guard let enumerator = FileManager.default.enumerator(atPath: directory) else { return [] }
        var result: [String] = []
        for case let relative as String in enumerator where relative.hasSuffix(".xsd") {
            result.append((directory as NSString).appendingPathComponent(relative))
        }
        return result.sorted()
    }

    /// The path relative to the corpus root for the triage report. The bare
    /// basename is ambiguous: the same name recurs across subdirectories (e.g.
    /// `Facets/Schemas/gMonth_maxInclusive001.xsd` carries `value=""` and is
    /// correctly rejected, while `Facets/gMonth/gMonth_maxInclusive001.xsd`
    /// carries the contested 2001-REC `--MM--` form), so a basename-only entry
    /// hides which file actually disagrees. Reporting the relative path makes the
    /// queue triagable.
    private func name(_ path: String, under root: String) -> String {
        path.hasPrefix(root) ? String(path.dropFirst(root.count).drop(while: { $0 == "/" })) : path
    }
}