From eaf6a01a0e32ec2565113803cdedc22442cdd394 Mon Sep 17 00:00:00 2001 From: Mihaela M Date: Fri, 26 Jun 2026 18:26:21 +0200 Subject: [PATCH] test: break down YAML test-suite parse failures by cause The harness now histograms should-parse failures by ParseError diagnosticCode, so the conformance gap is actionable rather than a single number. The breakdown corrects an earlier heuristic guess: multi-document streams are not the lever (1 of 78 failures). The real clusters are unexpectedToken=49 (explicit/complex keys, multi-line plain scalars, tagged keys, document-marker + node combinations), expectedNode=11, and unsupportedDirective=10. --- Tests/YAMLTestSuiteConformanceTests.swift | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Tests/YAMLTestSuiteConformanceTests.swift b/Tests/YAMLTestSuiteConformanceTests.swift index 6fc299a..f0888d4 100644 --- a/Tests/YAMLTestSuiteConformanceTests.swift +++ b/Tests/YAMLTestSuiteConformanceTests.swift @@ -49,22 +49,37 @@ struct YAMLTestSuiteConformanceTests { var okExpected = 0, okPassed = 0, errExpected = 0, errPassed = 0 var shouldParseFailures: [String] = [] var shouldRejectFailures: [String] = [] + var failureCauses: [String: Int] = [:] for testCase in cases { - let parsed = (try? PureYAML.parseStream(testCase.input)) != nil + var thrown: Error? + do { + _ = try PureYAML.parseStream(testCase.input) + } catch { + thrown = error + } + let parsed = thrown == nil if testCase.expectsError { errExpected += 1 if parsed { shouldRejectFailures.append(testCase.id) } else { errPassed += 1 } } else { okExpected += 1 - if parsed { okPassed += 1 } else { shouldParseFailures.append(testCase.id) } + if parsed { + okPassed += 1 + } else { + shouldParseFailures.append(testCase.id) + let cause = (thrown as? PureYAML.Parsing.ParseError)?.diagnosticCode ?? "other" + failureCauses[cause, default: 0] += 1 + } } } let total = okExpected + errExpected let passed = okPassed + errPassed + let causes = failureCauses.sorted { $0.value > $1.value }.map { "\($0.key)=\($0.value)" } report(""" YAML test-suite parse-level: \(passed)/\(total) (\(percent(passed, total))) should-parse handled: \(okPassed)/\(okExpected) (\(percent(okPassed, okExpected))) should-error rejected: \(errPassed)/\(errExpected) (\(percent(errPassed, errExpected))) + should-parse failures by cause: \(causes.joined(separator: " ")) should-parse failures (\(shouldParseFailures.count)): \(sample(shouldParseFailures)) should-error misses (\(shouldRejectFailures.count)): \(sample(shouldRejectFailures)) """)