import Foundation @testable import PureYAML import Testing /// Measures PureYAML against the official YAML test-suite (https://github.com/yaml/yaml-test-suite), /// the canonical YAML 1.2 conformance corpus. Opt-in, like the real-world corpus suite: the data is /// large and fetched separately, so the run is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1` and /// `bash scripts/fetch-yaml-test-suite.sh`. It reports two levels: /// - parse-level: a should-parse case must parse and a should-error case must be rejected; /// - value-level: a should-parse case must parse to the exact tree its `in.json` describes, the /// honest metric, since a parser can accept a document and build the wrong tree. /// Both print their pass rate to stderr and assert only that the corpus was found, so this is a /// measurement harness, not yet a threshold gate (the threshold is set once the number is known). @Suite("YAML test-suite conformance") struct YAMLTestSuiteConformanceTests { private struct Case { let id: String let input: String let expectsError: Bool let expectedJSON: String? } private func loadCases() -> [Case]? { let manager = FileManager.default let root = ProcessInfo.processInfo.environment["PUREYAML_YAML_TEST_SUITE_DIR"] ?? manager.currentDirectoryPath + "/.build/yaml-test-suite" guard let names = try? manager.contentsOfDirectory(atPath: root) else { return nil } var cases: [Case] = [] for name in names.sorted() { let dir = root + "/" + name guard let input = try? String(contentsOfFile: dir + "/in.yaml", encoding: .utf8) else { continue } cases.append(Case( id: name, input: input, expectsError: manager.fileExists(atPath: dir + "/error"), expectedJSON: try? String(contentsOfFile: dir + "/in.json", encoding: .utf8), )) } return cases } @Test("Parse-level conformance against the official yaml-test-suite") func parseLevelConformance() throws { guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else { return } let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh") var okExpected = 0, okPassed = 0, errExpected = 0, errPassed = 0 var shouldParseFailures: [String] = [] var shouldRejectFailures: [String] = [] var failureCauses: [String: Int] = [:] for testCase in cases { var thrown: Error? do { _ = try PureYAML.parseStream(testCase.input) } catch { thrown = error } let parsed = thrown == nil if testCase.expectsError { errExpected += 1 if parsed { shouldRejectFailures.append(testCase.id) } else { errPassed += 1 } } else { okExpected += 1 if parsed { okPassed += 1 } else { shouldParseFailures.append(testCase.id) let cause = (thrown as? PureYAML.Parsing.ParseError)?.diagnosticCode ?? "other" failureCauses[cause, default: 0] += 1 } } } let total = okExpected + errExpected let passed = okPassed + errPassed let causes = failureCauses.sorted { $0.value > $1.value }.map { "\($0.key)=\($0.value)" } report(""" YAML test-suite parse-level: \(passed)/\(total) (\(percent(passed, total))) should-parse handled: \(okPassed)/\(okExpected) (\(percent(okPassed, okExpected))) should-error rejected: \(errPassed)/\(errExpected) (\(percent(errPassed, errExpected))) should-parse failures by cause: \(causes.joined(separator: " ")) should-parse failures (\(shouldParseFailures.count)): \(sample(shouldParseFailures)) should-error misses (\(shouldRejectFailures.count)): \(sample(shouldRejectFailures)) """) #expect(total > 0) } /// The honest metric: a should-parse case must parse to the exact tree its `in.json` describes, /// not merely parse without error. Restricted to single-document cases, since the suite's /// `in.json` is one JSON value; multi-document and non-string-key cases (which JSON cannot /// represent) are counted as skipped, not failed, so the rate reflects only what is comparable. @Test("Value-level conformance: the parsed tree equals the expected in.json") func valueLevelConformance() throws { guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else { return } let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh") var eligible = 0, matched = 0, parseFailed = 0, skipped = 0 var mismatches: [String] = [] for testCase in cases where !testCase.expectsError { guard let json = testCase.expectedJSON else { continue } guard let documents = try? PureYAML.parseStream(testCase.input) else { parseFailed += 1 continue } // Compare only single-document cases against their single in.json value. guard documents.count == 1, let expected = jsonNode(fromJSONText: json), let actual = jsonNode(from: documents[0].value) else { skipped += 1 continue } eligible += 1 if actual == expected { matched += 1 } else { mismatches.append(testCase.id) } } report(""" YAML test-suite value-level: \(matched)/\(eligible) (\(percent(matched, eligible))) of comparable cases (parse-failed: \(parseFailed), skipped multi-doc/non-string-key: \(skipped)) value mismatches (\(mismatches.count)): \(sample(mismatches)) """) #expect(eligible > 0) } /// A JSON-comparable normalization: ints and floats both become a number, so a value-level /// match reflects structure and resolved value the way `in.json` records them. private indirect enum JSONNode: Equatable { case null case bool(Bool) case number(Double) case string(String) case array([JSONNode]) case object([String: JSONNode]) } private func jsonNode(from value: PureYAML.Model.Value) -> JSONNode? { switch value { case .null: .null case let .bool(flag): .bool(flag) case let .int(integer): .number(Double(integer)) case let .double(double): .number(double) case let .string(text): .string(text) case let .sequence(items): jsonArray(items) case let .mapping(mapping): jsonObject(mapping) } } private func jsonArray(_ items: [PureYAML.Model.Value]) -> JSONNode? { var nodes: [JSONNode] = [] for item in items { guard let node = jsonNode(from: item) else { return nil } nodes.append(node) } return .array(nodes) } private func jsonObject(_ mapping: PureYAML.Model.Mapping) -> JSONNode? { var object: [String: JSONNode] = [:] for pair in mapping.pairs { guard let key = pair.keyNode.stringValue, let node = jsonNode(from: pair.value) else { return nil } object[key] = node } return .object(object) } private func jsonNode(fromJSONText text: String) -> JSONNode? { guard let data = text.data(using: .utf8), let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) else { return nil } return jsonNode(fromJSONObject: object) } private func jsonNode(fromJSONObject object: Any) -> JSONNode? { switch object { case is NSNull: return .null case let number as NSNumber: return CFGetTypeID(number) == CFBooleanGetTypeID() ? .bool(number.boolValue) : .number(number.doubleValue) case let text as String: return .string(text) case let array as [Any]: var nodes: [JSONNode] = [] for item in array { guard let node = jsonNode(fromJSONObject: item) else { return nil } nodes.append(node) } return .array(nodes) case let dictionary as [String: Any]: var result: [String: JSONNode] = [:] for (key, value) in dictionary { guard let node = jsonNode(fromJSONObject: value) else { return nil } result[key] = node } return .object(result) default: return nil } } private func percent(_ part: Int, _ whole: Int) -> String { whole == 0 ? "n/a" : String(format: "%.1f%%", Double(part) / Double(whole) * 100) } private func sample(_ ids: [String]) -> String { ids.prefix(25).joined(separator: " ") + (ids.count > 25 ? " …" : "") } private func report(_ text: String) { FileHandle.standardError.write(Data("\n\(text)\n".utf8)) } }