test: break down YAML test-suite parse failures by cause

The harness now histograms should-parse failures by ParseError
diagnosticCode, so the conformance gap is actionable rather than a single
number. The breakdown corrects an earlier heuristic guess: multi-document
streams are not the lever (1 of 78 failures). The real clusters are
unexpectedToken=49 (explicit/complex keys, multi-line plain scalars,
tagged keys, document-marker + node combinations), expectedNode=11, and
unsupportedDirective=10.
This commit is contained in:
Mihaela M 2026-06-26 18:26:21 +02:00
parent 9298536e52
commit eaf6a01a0e

View File

@ -49,22 +49,37 @@ struct YAMLTestSuiteConformanceTests {
var okExpected = 0, okPassed = 0, errExpected = 0, errPassed = 0
var shouldParseFailures: [String] = []
var shouldRejectFailures: [String] = []
var failureCauses: [String: Int] = [:]
for testCase in cases {
let parsed = (try? PureYAML.parseStream(testCase.input)) != nil
var thrown: Error?
do {
_ = try PureYAML.parseStream(testCase.input)
} catch {
thrown = error
}
let parsed = thrown == nil
if testCase.expectsError {
errExpected += 1
if parsed { shouldRejectFailures.append(testCase.id) } else { errPassed += 1 }
} else {
okExpected += 1
if parsed { okPassed += 1 } else { shouldParseFailures.append(testCase.id) }
if parsed {
okPassed += 1
} else {
shouldParseFailures.append(testCase.id)
let cause = (thrown as? PureYAML.Parsing.ParseError)?.diagnosticCode ?? "other"
failureCauses[cause, default: 0] += 1
}
}
}
let total = okExpected + errExpected
let passed = okPassed + errPassed
let causes = failureCauses.sorted { $0.value > $1.value }.map { "\($0.key)=\($0.value)" }
report("""
YAML test-suite parse-level: \(passed)/\(total) (\(percent(passed, total)))
should-parse handled: \(okPassed)/\(okExpected) (\(percent(okPassed, okExpected)))
should-error rejected: \(errPassed)/\(errExpected) (\(percent(errPassed, errExpected)))
should-parse failures by cause: \(causes.joined(separator: " "))
should-parse failures (\(shouldParseFailures.count)): \(sample(shouldParseFailures))
should-error misses (\(shouldRejectFailures.count)): \(sample(shouldRejectFailures))
""")