test: add value-level YAML conformance against in.json

Parse-level conformance only asks whether a document parses or is
rejected; a parser can accept a document and build the wrong tree and
still score. The harness now also compares the parsed tree to each case's
in.json, the canonical expected value, the honest metric.

Value-level baseline: 130/170 (76.5%) of comparable single-document
cases. It immediately exposes 40 documents that parse without error but to
the wrong tree, invisible to parse-level: double-quoted multi-line folding
(the newline+indent should fold to one space), scalar resolution (a value
read as null instead of a string), and block-literal indentation. 49 more
do not parse, and 12 multi-doc/non-string-key cases are not comparable
through a single JSON value and are counted as skipped, not failed.

Every future parsing fix is now judged against ground truth, not just
whether it threw.
This commit is contained in:
Mihaela M 2026-06-26 20:18:32 +02:00
parent d90a091ab0
commit c888a113f2
2 changed files with 129 additions and 9 deletions

View File

@ -12,11 +12,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- An opt-in conformance harness against the official YAML test-suite, the canonical
YAML 1.2 corpus (`Tests/YAMLTestSuiteConformanceTests.swift`,
`scripts/fetch-yaml-test-suite.sh`). Gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1`
and a separate fetch, so the data is never vendored. It reports parse-level
conformance (a should-parse case must parse; a should-error case must be
rejected) and histograms the should-parse failures by cause. This turns "best
YAML library" from an assertion into a tracked number; the stricter value-level
check against each case's `in.json` is the next step.
and a separate fetch, so the data is never vendored. It reports two levels:
parse-level (a should-parse case must parse; a should-error case must be
rejected; failures histogrammed by cause) and the stricter, honest value-level,
comparing the parsed tree to each case's `in.json`. Value-level baseline:
130/170 (76.5%) of comparable single-document cases, exposing 40 documents that
parse without error but to the wrong tree (e.g. double-quoted line folding,
scalar resolution, block-scalar indentation) which parse-level cannot see. This
turns "best YAML library" from an assertion into a tracked number.
### Fixed

View File

@ -5,10 +5,11 @@ import Testing
/// Measures PureYAML against the official YAML test-suite (https://github.com/yaml/yaml-test-suite),
/// the canonical YAML 1.2 conformance corpus. Opt-in, like the real-world corpus suite: the data is
/// large and fetched separately, so the run is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1` and
/// `bash scripts/fetch-yaml-test-suite.sh`. It reports parse-level conformance: a should-parse case
/// must parse and a should-error case (one shipping an `error` marker) must be rejected. The
/// stricter value-level check, comparing the parsed tree to each case's `in.json`, is the next step.
/// The suite prints the pass rate to stderr and asserts only that the corpus was found, so it is a
/// `bash scripts/fetch-yaml-test-suite.sh`. It reports two levels:
/// - parse-level: a should-parse case must parse and a should-error case must be rejected;
/// - value-level: a should-parse case must parse to the exact tree its `in.json` describes, the
/// honest metric, since a parser can accept a document and build the wrong tree.
/// Both print their pass rate to stderr and assert only that the corpus was found, so this is a
/// measurement harness, not yet a threshold gate (the threshold is set once the number is known).
@Suite("YAML test-suite conformance")
struct YAMLTestSuiteConformanceTests {
@ -16,6 +17,7 @@ struct YAMLTestSuiteConformanceTests {
let id: String
let input: String
let expectsError: Bool
let expectedJSON: String?
}
private func loadCases() -> [Case]? {
@ -35,6 +37,7 @@ struct YAMLTestSuiteConformanceTests {
id: name,
input: input,
expectsError: manager.fileExists(atPath: dir + "/error"),
expectedJSON: try? String(contentsOfFile: dir + "/in.json", encoding: .utf8),
))
}
return cases
@ -86,6 +89,120 @@ struct YAMLTestSuiteConformanceTests {
#expect(total > 0)
}
/// The honest metric: a should-parse case must parse to the exact tree its `in.json` describes,
/// not merely parse without error. Restricted to single-document cases, since the suite's
/// `in.json` is one JSON value; multi-document and non-string-key cases (which JSON cannot
/// represent) are counted as skipped, not failed, so the rate reflects only what is comparable.
@Test("Value-level conformance: the parsed tree equals the expected in.json")
func valueLevelConformance() throws {
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
return
}
let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh")
var eligible = 0, matched = 0, parseFailed = 0, skipped = 0
var mismatches: [String] = []
for testCase in cases where !testCase.expectsError {
guard let json = testCase.expectedJSON else { continue }
guard let documents = try? PureYAML.parseStream(testCase.input) else {
parseFailed += 1
continue
}
// Compare only single-document cases against their single in.json value.
guard documents.count == 1,
let expected = jsonNode(fromJSONText: json),
let actual = jsonNode(from: documents[0].value)
else {
skipped += 1
continue
}
eligible += 1
if actual == expected { matched += 1 } else { mismatches.append(testCase.id) }
}
report("""
YAML test-suite value-level: \(matched)/\(eligible) (\(percent(matched, eligible))) of comparable cases
(parse-failed: \(parseFailed), skipped multi-doc/non-string-key: \(skipped))
value mismatches (\(mismatches.count)): \(sample(mismatches))
""")
#expect(eligible > 0)
}
/// A JSON-comparable normalization: ints and floats both become a number, so a value-level
/// match reflects structure and resolved value the way `in.json` records them.
private indirect enum JSONNode: Equatable {
case null
case bool(Bool)
case number(Double)
case string(String)
case array([JSONNode])
case object([String: JSONNode])
}
private func jsonNode(from value: PureYAML.Model.Value) -> JSONNode? {
switch value {
case .null: .null
case let .bool(flag): .bool(flag)
case let .int(integer): .number(Double(integer))
case let .double(double): .number(double)
case let .string(text): .string(text)
case let .sequence(items): jsonArray(items)
case let .mapping(mapping): jsonObject(mapping)
}
}
private func jsonArray(_ items: [PureYAML.Model.Value]) -> JSONNode? {
var nodes: [JSONNode] = []
for item in items {
guard let node = jsonNode(from: item) else { return nil }
nodes.append(node)
}
return .array(nodes)
}
private func jsonObject(_ mapping: PureYAML.Model.Mapping) -> JSONNode? {
var object: [String: JSONNode] = [:]
for pair in mapping.pairs {
guard let key = pair.keyNode.stringValue, let node = jsonNode(from: pair.value) else { return nil }
object[key] = node
}
return .object(object)
}
private func jsonNode(fromJSONText text: String) -> JSONNode? {
guard let data = text.data(using: .utf8),
let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed])
else {
return nil
}
return jsonNode(fromJSONObject: object)
}
private func jsonNode(fromJSONObject object: Any) -> JSONNode? {
switch object {
case is NSNull:
return .null
case let number as NSNumber:
return CFGetTypeID(number) == CFBooleanGetTypeID() ? .bool(number.boolValue) : .number(number.doubleValue)
case let text as String:
return .string(text)
case let array as [Any]:
var nodes: [JSONNode] = []
for item in array {
guard let node = jsonNode(fromJSONObject: item) else { return nil }
nodes.append(node)
}
return .array(nodes)
case let dictionary as [String: Any]:
var result: [String: JSONNode] = [:]
for (key, value) in dictionary {
guard let node = jsonNode(fromJSONObject: value) else { return nil }
result[key] = node
}
return .object(result)
default:
return nil
}
}
private func percent(_ part: Int, _ whole: Int) -> String {
whole == 0 ? "n/a" : String(format: "%.1f%%", Double(part) / Double(whole) * 100)
}