test: add value-level YAML conformance against in.json
Parse-level conformance only asks whether a document parses or is rejected; a parser can accept a document and build the wrong tree and still score. The harness now also compares the parsed tree to each case's in.json, the canonical expected value, the honest metric. Value-level baseline: 130/170 (76.5%) of comparable single-document cases. It immediately exposes 40 documents that parse without error but to the wrong tree, invisible to parse-level: double-quoted multi-line folding (the newline+indent should fold to one space), scalar resolution (a value read as null instead of a string), and block-literal indentation. 49 more do not parse, and 12 multi-doc/non-string-key cases are not comparable through a single JSON value and are counted as skipped, not failed. Every future parsing fix is now judged against ground truth, not just whether it threw.
This commit is contained in:
parent
d90a091ab0
commit
c888a113f2
13
CHANGELOG.md
13
CHANGELOG.md
@ -12,11 +12,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- An opt-in conformance harness against the official YAML test-suite, the canonical
|
||||
YAML 1.2 corpus (`Tests/YAMLTestSuiteConformanceTests.swift`,
|
||||
`scripts/fetch-yaml-test-suite.sh`). Gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1`
|
||||
and a separate fetch, so the data is never vendored. It reports parse-level
|
||||
conformance (a should-parse case must parse; a should-error case must be
|
||||
rejected) and histograms the should-parse failures by cause. This turns "best
|
||||
YAML library" from an assertion into a tracked number; the stricter value-level
|
||||
check against each case's `in.json` is the next step.
|
||||
and a separate fetch, so the data is never vendored. It reports two levels:
|
||||
parse-level (a should-parse case must parse; a should-error case must be
|
||||
rejected; failures histogrammed by cause) and the stricter, honest value-level,
|
||||
comparing the parsed tree to each case's `in.json`. Value-level baseline:
|
||||
130/170 (76.5%) of comparable single-document cases, exposing 40 documents that
|
||||
parse without error but to the wrong tree (e.g. double-quoted line folding,
|
||||
scalar resolution, block-scalar indentation) which parse-level cannot see. This
|
||||
turns "best YAML library" from an assertion into a tracked number.
|
||||
|
||||
### Fixed
|
||||
|
||||
|
||||
@ -5,10 +5,11 @@ import Testing
|
||||
/// Measures PureYAML against the official YAML test-suite (https://github.com/yaml/yaml-test-suite),
|
||||
/// the canonical YAML 1.2 conformance corpus. Opt-in, like the real-world corpus suite: the data is
|
||||
/// large and fetched separately, so the run is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1` and
|
||||
/// `bash scripts/fetch-yaml-test-suite.sh`. It reports parse-level conformance: a should-parse case
|
||||
/// must parse and a should-error case (one shipping an `error` marker) must be rejected. The
|
||||
/// stricter value-level check, comparing the parsed tree to each case's `in.json`, is the next step.
|
||||
/// The suite prints the pass rate to stderr and asserts only that the corpus was found, so it is a
|
||||
/// `bash scripts/fetch-yaml-test-suite.sh`. It reports two levels:
|
||||
/// - parse-level: a should-parse case must parse and a should-error case must be rejected;
|
||||
/// - value-level: a should-parse case must parse to the exact tree its `in.json` describes, the
|
||||
/// honest metric, since a parser can accept a document and build the wrong tree.
|
||||
/// Both print their pass rate to stderr and assert only that the corpus was found, so this is a
|
||||
/// measurement harness, not yet a threshold gate (the threshold is set once the number is known).
|
||||
@Suite("YAML test-suite conformance")
|
||||
struct YAMLTestSuiteConformanceTests {
|
||||
@ -16,6 +17,7 @@ struct YAMLTestSuiteConformanceTests {
|
||||
let id: String
|
||||
let input: String
|
||||
let expectsError: Bool
|
||||
let expectedJSON: String?
|
||||
}
|
||||
|
||||
private func loadCases() -> [Case]? {
|
||||
@ -35,6 +37,7 @@ struct YAMLTestSuiteConformanceTests {
|
||||
id: name,
|
||||
input: input,
|
||||
expectsError: manager.fileExists(atPath: dir + "/error"),
|
||||
expectedJSON: try? String(contentsOfFile: dir + "/in.json", encoding: .utf8),
|
||||
))
|
||||
}
|
||||
return cases
|
||||
@ -86,6 +89,120 @@ struct YAMLTestSuiteConformanceTests {
|
||||
#expect(total > 0)
|
||||
}
|
||||
|
||||
/// The honest metric: a should-parse case must parse to the exact tree its `in.json` describes,
|
||||
/// not merely parse without error. Restricted to single-document cases, since the suite's
|
||||
/// `in.json` is one JSON value; multi-document and non-string-key cases (which JSON cannot
|
||||
/// represent) are counted as skipped, not failed, so the rate reflects only what is comparable.
|
||||
@Test("Value-level conformance: the parsed tree equals the expected in.json")
|
||||
func valueLevelConformance() throws {
|
||||
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
|
||||
return
|
||||
}
|
||||
let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh")
|
||||
var eligible = 0, matched = 0, parseFailed = 0, skipped = 0
|
||||
var mismatches: [String] = []
|
||||
for testCase in cases where !testCase.expectsError {
|
||||
guard let json = testCase.expectedJSON else { continue }
|
||||
guard let documents = try? PureYAML.parseStream(testCase.input) else {
|
||||
parseFailed += 1
|
||||
continue
|
||||
}
|
||||
// Compare only single-document cases against their single in.json value.
|
||||
guard documents.count == 1,
|
||||
let expected = jsonNode(fromJSONText: json),
|
||||
let actual = jsonNode(from: documents[0].value)
|
||||
else {
|
||||
skipped += 1
|
||||
continue
|
||||
}
|
||||
eligible += 1
|
||||
if actual == expected { matched += 1 } else { mismatches.append(testCase.id) }
|
||||
}
|
||||
report("""
|
||||
YAML test-suite value-level: \(matched)/\(eligible) (\(percent(matched, eligible))) of comparable cases
|
||||
(parse-failed: \(parseFailed), skipped multi-doc/non-string-key: \(skipped))
|
||||
value mismatches (\(mismatches.count)): \(sample(mismatches))
|
||||
""")
|
||||
#expect(eligible > 0)
|
||||
}
|
||||
|
||||
/// A JSON-comparable normalization: ints and floats both become a number, so a value-level
|
||||
/// match reflects structure and resolved value the way `in.json` records them.
|
||||
private indirect enum JSONNode: Equatable {
|
||||
case null
|
||||
case bool(Bool)
|
||||
case number(Double)
|
||||
case string(String)
|
||||
case array([JSONNode])
|
||||
case object([String: JSONNode])
|
||||
}
|
||||
|
||||
private func jsonNode(from value: PureYAML.Model.Value) -> JSONNode? {
|
||||
switch value {
|
||||
case .null: .null
|
||||
case let .bool(flag): .bool(flag)
|
||||
case let .int(integer): .number(Double(integer))
|
||||
case let .double(double): .number(double)
|
||||
case let .string(text): .string(text)
|
||||
case let .sequence(items): jsonArray(items)
|
||||
case let .mapping(mapping): jsonObject(mapping)
|
||||
}
|
||||
}
|
||||
|
||||
private func jsonArray(_ items: [PureYAML.Model.Value]) -> JSONNode? {
|
||||
var nodes: [JSONNode] = []
|
||||
for item in items {
|
||||
guard let node = jsonNode(from: item) else { return nil }
|
||||
nodes.append(node)
|
||||
}
|
||||
return .array(nodes)
|
||||
}
|
||||
|
||||
private func jsonObject(_ mapping: PureYAML.Model.Mapping) -> JSONNode? {
|
||||
var object: [String: JSONNode] = [:]
|
||||
for pair in mapping.pairs {
|
||||
guard let key = pair.keyNode.stringValue, let node = jsonNode(from: pair.value) else { return nil }
|
||||
object[key] = node
|
||||
}
|
||||
return .object(object)
|
||||
}
|
||||
|
||||
private func jsonNode(fromJSONText text: String) -> JSONNode? {
|
||||
guard let data = text.data(using: .utf8),
|
||||
let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed])
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
return jsonNode(fromJSONObject: object)
|
||||
}
|
||||
|
||||
private func jsonNode(fromJSONObject object: Any) -> JSONNode? {
|
||||
switch object {
|
||||
case is NSNull:
|
||||
return .null
|
||||
case let number as NSNumber:
|
||||
return CFGetTypeID(number) == CFBooleanGetTypeID() ? .bool(number.boolValue) : .number(number.doubleValue)
|
||||
case let text as String:
|
||||
return .string(text)
|
||||
case let array as [Any]:
|
||||
var nodes: [JSONNode] = []
|
||||
for item in array {
|
||||
guard let node = jsonNode(fromJSONObject: item) else { return nil }
|
||||
nodes.append(node)
|
||||
}
|
||||
return .array(nodes)
|
||||
case let dictionary as [String: Any]:
|
||||
var result: [String: JSONNode] = [:]
|
||||
for (key, value) in dictionary {
|
||||
guard let node = jsonNode(fromJSONObject: value) else { return nil }
|
||||
result[key] = node
|
||||
}
|
||||
return .object(result)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
private func percent(_ part: Int, _ whole: Int) -> String {
|
||||
whole == 0 ? "n/a" : String(format: "%.1f%%", Double(part) / Double(whole) * 100)
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user