import Foundation @testable import PureYAML import Testing /// Measures PureYAML against the official YAML test-suite (https://github.com/yaml/yaml-test-suite), /// the canonical YAML 1.2 conformance corpus. Opt-in, like the real-world corpus suite: the data is /// large and fetched separately, so the run is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1` and /// `bash scripts/fetch-yaml-test-suite.sh`. It reports two levels: /// - parse-level: a should-parse case must parse and a should-error case must be rejected; /// - value-level: a should-parse case must parse to the exact tree its `in.json` describes, the /// honest metric, since a parser can accept a document and build the wrong tree. /// Both print their pass rate to stderr and assert only that the corpus was found, so this is a /// measurement harness, not yet a threshold gate (the threshold is set once the number is known). @Suite("YAML test-suite conformance") struct YAMLTestSuiteConformanceTests { private struct Case { let id: String let input: String let expectsError: Bool let expectedJSON: String? } private func loadCases() -> [Case]? { let manager = FileManager.default let root = ProcessInfo.processInfo.environment["PUREYAML_YAML_TEST_SUITE_DIR"] ?? manager.currentDirectoryPath + "/.build/yaml-test-suite" guard let names = try? manager.contentsOfDirectory(atPath: root) else { return nil } var cases: [Case] = [] for name in names.sorted() { let dir = root + "/" + name guard let input = try? String(contentsOfFile: dir + "/in.yaml", encoding: .utf8) else { continue } cases.append(Case( id: name, input: input, expectsError: manager.fileExists(atPath: dir + "/error"), expectedJSON: try? String(contentsOfFile: dir + "/in.json", encoding: .utf8), )) } return cases } @Test("Parse-level conformance against the official yaml-test-suite") func parseLevelConformance() throws { guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else { return } let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh") var okExpected = 0, okPassed = 0, errExpected = 0, errPassed = 0 var shouldParseFailures: [String] = [] var shouldRejectFailures: [String] = [] var failureCauses: [String: Int] = [:] for testCase in cases { var thrown: Error? do { _ = try PureYAML.parseStream(testCase.input) } catch { thrown = error } let parsed = thrown == nil if testCase.expectsError { errExpected += 1 if parsed { shouldRejectFailures.append(testCase.id) } else { errPassed += 1 } } else { okExpected += 1 if parsed { okPassed += 1 } else { shouldParseFailures.append(testCase.id) let cause = (thrown as? PureYAML.Parsing.ParseError)?.diagnosticCode ?? "other" failureCauses[cause, default: 0] += 1 } } } let total = okExpected + errExpected let passed = okPassed + errPassed let causes = failureCauses.sorted { $0.value > $1.value }.map { "\($0.key)=\($0.value)" } report(""" YAML test-suite parse-level: \(passed)/\(total) (\(percent(passed, total))) should-parse handled: \(okPassed)/\(okExpected) (\(percent(okPassed, okExpected))) should-error rejected: \(errPassed)/\(errExpected) (\(percent(errPassed, errExpected))) should-parse failures by cause: \(causes.joined(separator: " ")) should-parse failures (\(shouldParseFailures.count)): \(sample(shouldParseFailures)) should-error misses (\(shouldRejectFailures.count)): \(sample(shouldRejectFailures)) """) #expect(total > 0) } /// The honest metric: a should-parse case must parse to the exact tree its `in.json` describes, /// not merely parse without error. Restricted to single-document cases, since the suite's /// `in.json` is one JSON value; multi-document and non-string-key cases (which JSON cannot /// represent) are counted as skipped, not failed, so the rate reflects only what is comparable. @Test("Value-level conformance: the parsed tree equals the expected in.json") func valueLevelConformance() throws { guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else { return } let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh") var eligible = 0, matched = 0, parseFailed = 0, skipped = 0 var mismatches: [String] = [] for testCase in cases where !testCase.expectsError { guard let json = testCase.expectedJSON else { continue } guard let documents = try? PureYAML.parseStream(testCase.input) else { parseFailed += 1 continue } // Compare only single-document cases against their single in.json value. guard documents.count == 1, let expected = jsonNode(fromJSONText: json), let actual = jsonNode(from: documents[0].value) else { skipped += 1 continue } eligible += 1 if actual == expected { matched += 1 } else { mismatches.append(testCase.id) } } report(""" YAML test-suite value-level: \(matched)/\(eligible) (\(percent(matched, eligible))) of comparable cases (parse-failed: \(parseFailed), skipped multi-doc/non-string-key: \(skipped)) value mismatches (\(mismatches.count)): \(sample(mismatches)) """) #expect(eligible > 0) } /// A JSON-comparable normalization: ints and floats both become a number, so a value-level /// match reflects structure and resolved value the way `in.json` records them. `Decodable` so the /// expected `in.json` is read with `JSONDecoder`, which distinguishes a JSON bool from a number /// natively. The previous `JSONSerialization` path needed CoreFoundation (`CFGetTypeID`) for that, /// which is Darwin-only and failed to compile on Linux and Windows. private indirect enum JSONNode: Equatable, Decodable { case null case bool(Bool) case number(Double) case string(String) case array([JSONNode]) case object([String: JSONNode]) init(from decoder: Decoder) throws { let container = try decoder.singleValueContainer() // Order matters: a JSON bool must be tried before a number, since `true`/`false` are a // distinct JSON type the number decode would otherwise not see. if container.decodeNil() { self = .null } else if let bool = try? container.decode(Bool.self) { self = .bool(bool) } else if let number = try? container.decode(Double.self) { self = .number(number) } else if let string = try? container.decode(String.self) { self = .string(string) } else if let array = try? container.decode([JSONNode].self) { self = .array(array) } else if let object = try? container.decode([String: JSONNode].self) { self = .object(object) } else { throw DecodingError.dataCorrupted(.init(codingPath: decoder.codingPath, debugDescription: "unrecognized JSON value")) } } } private func jsonNode(from value: PureYAML.Model.Value) -> JSONNode? { switch value { case .null: .null case let .bool(flag): .bool(flag) case let .int(integer): .number(Double(integer)) case let .double(double): .number(double) case let .string(text): .string(text) case let .sequence(items): jsonArray(items) case let .mapping(mapping): jsonObject(mapping) } } private func jsonArray(_ items: [PureYAML.Model.Value]) -> JSONNode? { var nodes: [JSONNode] = [] for item in items { guard let node = jsonNode(from: item) else { return nil } nodes.append(node) } return .array(nodes) } private func jsonObject(_ mapping: PureYAML.Model.Mapping) -> JSONNode? { var object: [String: JSONNode] = [:] for pair in mapping.pairs { guard let key = pair.keyNode.stringValue, let node = jsonNode(from: pair.value) else { return nil } object[key] = node } return .object(object) } private func jsonNode(fromJSONText text: String) -> JSONNode? { guard let data = text.data(using: .utf8) else { return nil } // The suite's `in.json` may be a top-level fragment (a bare string, number, or null), which // `JSONDecoder` accepts in Swift 6. `JSONNode`'s decoder uses a single-value container, so it // reads fragments and containers alike, cross-platform and without CoreFoundation. return try? JSONDecoder().decode(JSONNode.self, from: data) } private func percent(_ part: Int, _ whole: Int) -> String { whole == 0 ? "n/a" : String(format: "%.1f%%", Double(part) / Double(whole) * 100) } private func sample(_ ids: [String]) -> String { ids.prefix(25).joined(separator: " ") + (ids.count > 25 ? " …" : "") } private func report(_ text: String) { FileHandle.standardError.write(Data("\n\(text)\n".utf8)) } }