test: break down YAML test-suite parse failures by cause
The harness now histograms should-parse failures by ParseError diagnosticCode, so the conformance gap is actionable rather than a single number. The breakdown corrects an earlier heuristic guess: multi-document streams are not the lever (1 of 78 failures). The real clusters are unexpectedToken=49 (explicit/complex keys, multi-line plain scalars, tagged keys, document-marker + node combinations), expectedNode=11, and unsupportedDirective=10.
This commit is contained in:
parent
9298536e52
commit
eaf6a01a0e
@ -49,22 +49,37 @@ struct YAMLTestSuiteConformanceTests {
|
||||
var okExpected = 0, okPassed = 0, errExpected = 0, errPassed = 0
|
||||
var shouldParseFailures: [String] = []
|
||||
var shouldRejectFailures: [String] = []
|
||||
var failureCauses: [String: Int] = [:]
|
||||
for testCase in cases {
|
||||
let parsed = (try? PureYAML.parseStream(testCase.input)) != nil
|
||||
var thrown: Error?
|
||||
do {
|
||||
_ = try PureYAML.parseStream(testCase.input)
|
||||
} catch {
|
||||
thrown = error
|
||||
}
|
||||
let parsed = thrown == nil
|
||||
if testCase.expectsError {
|
||||
errExpected += 1
|
||||
if parsed { shouldRejectFailures.append(testCase.id) } else { errPassed += 1 }
|
||||
} else {
|
||||
okExpected += 1
|
||||
if parsed { okPassed += 1 } else { shouldParseFailures.append(testCase.id) }
|
||||
if parsed {
|
||||
okPassed += 1
|
||||
} else {
|
||||
shouldParseFailures.append(testCase.id)
|
||||
let cause = (thrown as? PureYAML.Parsing.ParseError)?.diagnosticCode ?? "other"
|
||||
failureCauses[cause, default: 0] += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
let total = okExpected + errExpected
|
||||
let passed = okPassed + errPassed
|
||||
let causes = failureCauses.sorted { $0.value > $1.value }.map { "\($0.key)=\($0.value)" }
|
||||
report("""
|
||||
YAML test-suite parse-level: \(passed)/\(total) (\(percent(passed, total)))
|
||||
should-parse handled: \(okPassed)/\(okExpected) (\(percent(okPassed, okExpected)))
|
||||
should-error rejected: \(errPassed)/\(errExpected) (\(percent(errPassed, errExpected)))
|
||||
should-parse failures by cause: \(causes.joined(separator: " "))
|
||||
should-parse failures (\(shouldParseFailures.count)): \(sample(shouldParseFailures))
|
||||
should-error misses (\(shouldRejectFailures.count)): \(sample(shouldRejectFailures))
|
||||
""")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user