Mihaela M
c888a113f2
Parse-level conformance only asks whether a document parses or is rejected; a parser can accept a document and build the wrong tree and still score. The harness now also compares the parsed tree to each case's in.json, the canonical expected value, the honest metric. Value-level baseline: 130/170 (76.5%) of comparable single-document cases. It immediately exposes 40 documents that parse without error but to the wrong tree, invisible to parse-level: double-quoted multi-line folding (the newline+indent should fold to one space), scalar resolution (a value read as null instead of a string), and block-literal indentation. 49 more do not parse, and 12 multi-doc/non-string-key cases are not comparable through a single JSON value and are counted as skipped, not failed. Every future parsing fix is now judged against ground truth, not just whether it threw.
218 lines
9.2 KiB
Swift
218 lines
9.2 KiB
Swift
import Foundation
|
|
@testable import PureYAML
|
|
import Testing
|
|
|
|
/// Measures PureYAML against the official YAML test-suite (https://github.com/yaml/yaml-test-suite),
|
|
/// the canonical YAML 1.2 conformance corpus. Opt-in, like the real-world corpus suite: the data is
|
|
/// large and fetched separately, so the run is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1` and
|
|
/// `bash scripts/fetch-yaml-test-suite.sh`. It reports two levels:
|
|
/// - parse-level: a should-parse case must parse and a should-error case must be rejected;
|
|
/// - value-level: a should-parse case must parse to the exact tree its `in.json` describes, the
|
|
/// honest metric, since a parser can accept a document and build the wrong tree.
|
|
/// Both print their pass rate to stderr and assert only that the corpus was found, so this is a
|
|
/// measurement harness, not yet a threshold gate (the threshold is set once the number is known).
|
|
@Suite("YAML test-suite conformance")
|
|
struct YAMLTestSuiteConformanceTests {
|
|
private struct Case {
|
|
let id: String
|
|
let input: String
|
|
let expectsError: Bool
|
|
let expectedJSON: String?
|
|
}
|
|
|
|
private func loadCases() -> [Case]? {
|
|
let manager = FileManager.default
|
|
let root = ProcessInfo.processInfo.environment["PUREYAML_YAML_TEST_SUITE_DIR"]
|
|
?? manager.currentDirectoryPath + "/.build/yaml-test-suite"
|
|
guard let names = try? manager.contentsOfDirectory(atPath: root) else {
|
|
return nil
|
|
}
|
|
var cases: [Case] = []
|
|
for name in names.sorted() {
|
|
let dir = root + "/" + name
|
|
guard let input = try? String(contentsOfFile: dir + "/in.yaml", encoding: .utf8) else {
|
|
continue
|
|
}
|
|
cases.append(Case(
|
|
id: name,
|
|
input: input,
|
|
expectsError: manager.fileExists(atPath: dir + "/error"),
|
|
expectedJSON: try? String(contentsOfFile: dir + "/in.json", encoding: .utf8),
|
|
))
|
|
}
|
|
return cases
|
|
}
|
|
|
|
@Test("Parse-level conformance against the official yaml-test-suite")
|
|
func parseLevelConformance() throws {
|
|
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
|
|
return
|
|
}
|
|
let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh")
|
|
var okExpected = 0, okPassed = 0, errExpected = 0, errPassed = 0
|
|
var shouldParseFailures: [String] = []
|
|
var shouldRejectFailures: [String] = []
|
|
var failureCauses: [String: Int] = [:]
|
|
for testCase in cases {
|
|
var thrown: Error?
|
|
do {
|
|
_ = try PureYAML.parseStream(testCase.input)
|
|
} catch {
|
|
thrown = error
|
|
}
|
|
let parsed = thrown == nil
|
|
if testCase.expectsError {
|
|
errExpected += 1
|
|
if parsed { shouldRejectFailures.append(testCase.id) } else { errPassed += 1 }
|
|
} else {
|
|
okExpected += 1
|
|
if parsed {
|
|
okPassed += 1
|
|
} else {
|
|
shouldParseFailures.append(testCase.id)
|
|
let cause = (thrown as? PureYAML.Parsing.ParseError)?.diagnosticCode ?? "other"
|
|
failureCauses[cause, default: 0] += 1
|
|
}
|
|
}
|
|
}
|
|
let total = okExpected + errExpected
|
|
let passed = okPassed + errPassed
|
|
let causes = failureCauses.sorted { $0.value > $1.value }.map { "\($0.key)=\($0.value)" }
|
|
report("""
|
|
YAML test-suite parse-level: \(passed)/\(total) (\(percent(passed, total)))
|
|
should-parse handled: \(okPassed)/\(okExpected) (\(percent(okPassed, okExpected)))
|
|
should-error rejected: \(errPassed)/\(errExpected) (\(percent(errPassed, errExpected)))
|
|
should-parse failures by cause: \(causes.joined(separator: " "))
|
|
should-parse failures (\(shouldParseFailures.count)): \(sample(shouldParseFailures))
|
|
should-error misses (\(shouldRejectFailures.count)): \(sample(shouldRejectFailures))
|
|
""")
|
|
#expect(total > 0)
|
|
}
|
|
|
|
/// The honest metric: a should-parse case must parse to the exact tree its `in.json` describes,
|
|
/// not merely parse without error. Restricted to single-document cases, since the suite's
|
|
/// `in.json` is one JSON value; multi-document and non-string-key cases (which JSON cannot
|
|
/// represent) are counted as skipped, not failed, so the rate reflects only what is comparable.
|
|
@Test("Value-level conformance: the parsed tree equals the expected in.json")
|
|
func valueLevelConformance() throws {
|
|
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
|
|
return
|
|
}
|
|
let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh")
|
|
var eligible = 0, matched = 0, parseFailed = 0, skipped = 0
|
|
var mismatches: [String] = []
|
|
for testCase in cases where !testCase.expectsError {
|
|
guard let json = testCase.expectedJSON else { continue }
|
|
guard let documents = try? PureYAML.parseStream(testCase.input) else {
|
|
parseFailed += 1
|
|
continue
|
|
}
|
|
// Compare only single-document cases against their single in.json value.
|
|
guard documents.count == 1,
|
|
let expected = jsonNode(fromJSONText: json),
|
|
let actual = jsonNode(from: documents[0].value)
|
|
else {
|
|
skipped += 1
|
|
continue
|
|
}
|
|
eligible += 1
|
|
if actual == expected { matched += 1 } else { mismatches.append(testCase.id) }
|
|
}
|
|
report("""
|
|
YAML test-suite value-level: \(matched)/\(eligible) (\(percent(matched, eligible))) of comparable cases
|
|
(parse-failed: \(parseFailed), skipped multi-doc/non-string-key: \(skipped))
|
|
value mismatches (\(mismatches.count)): \(sample(mismatches))
|
|
""")
|
|
#expect(eligible > 0)
|
|
}
|
|
|
|
/// A JSON-comparable normalization: ints and floats both become a number, so a value-level
|
|
/// match reflects structure and resolved value the way `in.json` records them.
|
|
private indirect enum JSONNode: Equatable {
|
|
case null
|
|
case bool(Bool)
|
|
case number(Double)
|
|
case string(String)
|
|
case array([JSONNode])
|
|
case object([String: JSONNode])
|
|
}
|
|
|
|
private func jsonNode(from value: PureYAML.Model.Value) -> JSONNode? {
|
|
switch value {
|
|
case .null: .null
|
|
case let .bool(flag): .bool(flag)
|
|
case let .int(integer): .number(Double(integer))
|
|
case let .double(double): .number(double)
|
|
case let .string(text): .string(text)
|
|
case let .sequence(items): jsonArray(items)
|
|
case let .mapping(mapping): jsonObject(mapping)
|
|
}
|
|
}
|
|
|
|
private func jsonArray(_ items: [PureYAML.Model.Value]) -> JSONNode? {
|
|
var nodes: [JSONNode] = []
|
|
for item in items {
|
|
guard let node = jsonNode(from: item) else { return nil }
|
|
nodes.append(node)
|
|
}
|
|
return .array(nodes)
|
|
}
|
|
|
|
private func jsonObject(_ mapping: PureYAML.Model.Mapping) -> JSONNode? {
|
|
var object: [String: JSONNode] = [:]
|
|
for pair in mapping.pairs {
|
|
guard let key = pair.keyNode.stringValue, let node = jsonNode(from: pair.value) else { return nil }
|
|
object[key] = node
|
|
}
|
|
return .object(object)
|
|
}
|
|
|
|
private func jsonNode(fromJSONText text: String) -> JSONNode? {
|
|
guard let data = text.data(using: .utf8),
|
|
let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed])
|
|
else {
|
|
return nil
|
|
}
|
|
return jsonNode(fromJSONObject: object)
|
|
}
|
|
|
|
private func jsonNode(fromJSONObject object: Any) -> JSONNode? {
|
|
switch object {
|
|
case is NSNull:
|
|
return .null
|
|
case let number as NSNumber:
|
|
return CFGetTypeID(number) == CFBooleanGetTypeID() ? .bool(number.boolValue) : .number(number.doubleValue)
|
|
case let text as String:
|
|
return .string(text)
|
|
case let array as [Any]:
|
|
var nodes: [JSONNode] = []
|
|
for item in array {
|
|
guard let node = jsonNode(fromJSONObject: item) else { return nil }
|
|
nodes.append(node)
|
|
}
|
|
return .array(nodes)
|
|
case let dictionary as [String: Any]:
|
|
var result: [String: JSONNode] = [:]
|
|
for (key, value) in dictionary {
|
|
guard let node = jsonNode(fromJSONObject: value) else { return nil }
|
|
result[key] = node
|
|
}
|
|
return .object(result)
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
private func percent(_ part: Int, _ whole: Int) -> String {
|
|
whole == 0 ? "n/a" : String(format: "%.1f%%", Double(part) / Double(whole) * 100)
|
|
}
|
|
|
|
private func sample(_ ids: [String]) -> String {
|
|
ids.prefix(25).joined(separator: " ") + (ids.count > 25 ? " …" : "")
|
|
}
|
|
|
|
private func report(_ text: String) {
|
|
FileHandle.standardError.write(Data("\n\(text)\n".utf8))
|
|
}
|
|
}
|