diff --git a/CHANGELOG.md b/CHANGELOG.md index c1ac992..316bc3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - An opt-in conformance harness against the official YAML test-suite, the canonical YAML 1.2 corpus (`Tests/YAMLTestSuiteConformanceTests.swift`, `scripts/fetch-yaml-test-suite.sh`). Gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1` - and a separate fetch, so the data is never vendored. It reports parse-level - conformance (a should-parse case must parse; a should-error case must be - rejected) and histograms the should-parse failures by cause. This turns "best - YAML library" from an assertion into a tracked number; the stricter value-level - check against each case's `in.json` is the next step. + and a separate fetch, so the data is never vendored. It reports two levels: + parse-level (a should-parse case must parse; a should-error case must be + rejected; failures histogrammed by cause) and the stricter, honest value-level, + comparing the parsed tree to each case's `in.json`. Value-level baseline: + 130/170 (76.5%) of comparable single-document cases, exposing 40 documents that + parse without error but to the wrong tree (e.g. double-quoted line folding, + scalar resolution, block-scalar indentation) which parse-level cannot see. This + turns "best YAML library" from an assertion into a tracked number. ### Fixed diff --git a/Tests/YAMLTestSuiteConformanceTests.swift b/Tests/YAMLTestSuiteConformanceTests.swift index f0888d4..936aa2d 100644 --- a/Tests/YAMLTestSuiteConformanceTests.swift +++ b/Tests/YAMLTestSuiteConformanceTests.swift @@ -5,10 +5,11 @@ import Testing /// Measures PureYAML against the official YAML test-suite (https://github.com/yaml/yaml-test-suite), /// the canonical YAML 1.2 conformance corpus. Opt-in, like the real-world corpus suite: the data is /// large and fetched separately, so the run is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1` and -/// `bash scripts/fetch-yaml-test-suite.sh`. It reports parse-level conformance: a should-parse case -/// must parse and a should-error case (one shipping an `error` marker) must be rejected. The -/// stricter value-level check, comparing the parsed tree to each case's `in.json`, is the next step. -/// The suite prints the pass rate to stderr and asserts only that the corpus was found, so it is a +/// `bash scripts/fetch-yaml-test-suite.sh`. It reports two levels: +/// - parse-level: a should-parse case must parse and a should-error case must be rejected; +/// - value-level: a should-parse case must parse to the exact tree its `in.json` describes, the +/// honest metric, since a parser can accept a document and build the wrong tree. +/// Both print their pass rate to stderr and assert only that the corpus was found, so this is a /// measurement harness, not yet a threshold gate (the threshold is set once the number is known). @Suite("YAML test-suite conformance") struct YAMLTestSuiteConformanceTests { @@ -16,6 +17,7 @@ struct YAMLTestSuiteConformanceTests { let id: String let input: String let expectsError: Bool + let expectedJSON: String? } private func loadCases() -> [Case]? { @@ -35,6 +37,7 @@ struct YAMLTestSuiteConformanceTests { id: name, input: input, expectsError: manager.fileExists(atPath: dir + "/error"), + expectedJSON: try? String(contentsOfFile: dir + "/in.json", encoding: .utf8), )) } return cases @@ -86,6 +89,120 @@ struct YAMLTestSuiteConformanceTests { #expect(total > 0) } + /// The honest metric: a should-parse case must parse to the exact tree its `in.json` describes, + /// not merely parse without error. Restricted to single-document cases, since the suite's + /// `in.json` is one JSON value; multi-document and non-string-key cases (which JSON cannot + /// represent) are counted as skipped, not failed, so the rate reflects only what is comparable. + @Test("Value-level conformance: the parsed tree equals the expected in.json") + func valueLevelConformance() throws { + guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else { + return + } + let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh") + var eligible = 0, matched = 0, parseFailed = 0, skipped = 0 + var mismatches: [String] = [] + for testCase in cases where !testCase.expectsError { + guard let json = testCase.expectedJSON else { continue } + guard let documents = try? PureYAML.parseStream(testCase.input) else { + parseFailed += 1 + continue + } + // Compare only single-document cases against their single in.json value. + guard documents.count == 1, + let expected = jsonNode(fromJSONText: json), + let actual = jsonNode(from: documents[0].value) + else { + skipped += 1 + continue + } + eligible += 1 + if actual == expected { matched += 1 } else { mismatches.append(testCase.id) } + } + report(""" + YAML test-suite value-level: \(matched)/\(eligible) (\(percent(matched, eligible))) of comparable cases + (parse-failed: \(parseFailed), skipped multi-doc/non-string-key: \(skipped)) + value mismatches (\(mismatches.count)): \(sample(mismatches)) + """) + #expect(eligible > 0) + } + + /// A JSON-comparable normalization: ints and floats both become a number, so a value-level + /// match reflects structure and resolved value the way `in.json` records them. + private indirect enum JSONNode: Equatable { + case null + case bool(Bool) + case number(Double) + case string(String) + case array([JSONNode]) + case object([String: JSONNode]) + } + + private func jsonNode(from value: PureYAML.Model.Value) -> JSONNode? { + switch value { + case .null: .null + case let .bool(flag): .bool(flag) + case let .int(integer): .number(Double(integer)) + case let .double(double): .number(double) + case let .string(text): .string(text) + case let .sequence(items): jsonArray(items) + case let .mapping(mapping): jsonObject(mapping) + } + } + + private func jsonArray(_ items: [PureYAML.Model.Value]) -> JSONNode? { + var nodes: [JSONNode] = [] + for item in items { + guard let node = jsonNode(from: item) else { return nil } + nodes.append(node) + } + return .array(nodes) + } + + private func jsonObject(_ mapping: PureYAML.Model.Mapping) -> JSONNode? { + var object: [String: JSONNode] = [:] + for pair in mapping.pairs { + guard let key = pair.keyNode.stringValue, let node = jsonNode(from: pair.value) else { return nil } + object[key] = node + } + return .object(object) + } + + private func jsonNode(fromJSONText text: String) -> JSONNode? { + guard let data = text.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) + else { + return nil + } + return jsonNode(fromJSONObject: object) + } + + private func jsonNode(fromJSONObject object: Any) -> JSONNode? { + switch object { + case is NSNull: + return .null + case let number as NSNumber: + return CFGetTypeID(number) == CFBooleanGetTypeID() ? .bool(number.boolValue) : .number(number.doubleValue) + case let text as String: + return .string(text) + case let array as [Any]: + var nodes: [JSONNode] = [] + for item in array { + guard let node = jsonNode(fromJSONObject: item) else { return nil } + nodes.append(node) + } + return .array(nodes) + case let dictionary as [String: Any]: + var result: [String: JSONNode] = [:] + for (key, value) in dictionary { + guard let node = jsonNode(fromJSONObject: value) else { return nil } + result[key] = node + } + return .object(result) + default: + return nil + } + } + private func percent(_ part: Int, _ whole: Int) -> String { whole == 0 ? "n/a" : String(format: "%.1f%%", Double(part) / Double(whole) * 100) }