import Foundation @testable import PureYAML import Testing /// Full conformance gate against the official YAML test suite /// (github.com/yaml/yaml-test-suite, `data` branch, commit /// 6ad3d2c62885d82fc349026c136ef560838fdf3d). Every official case (numbered /// sub-cases counted) is vendored under `Fixtures/yaml-test-suite//` and /// recorded in `Fixtures/yaml-test-suite-manifest.yaml` with a `status`: /// /// - `passing`: PureYAML already handles the case conformantly (it parses with a /// value matching the JSON oracle when one is present, or it rejects an error /// case). This is the regression guard: if a later change breaks the case, the /// gated sweep fails. /// - `gap`: PureYAML does not yet handle the case conformantly. The sweep asserts /// the gap still reproduces, so a fix is forced to update the manifest rather /// than silently drift. /// /// The behavioral sweep is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1`, mirroring /// the full real-corpus sweep, so the default suite stays fast while the whole suite /// is guarded on demand. The manifest-integrity test below is always on. @Suite("YAML Test Suite Gaps") struct YAMLTestSuiteGapsTests { @Test("Manifest is consistent with its vendored fixture files") func test_manifestMatchesVendoredFixtureFiles() throws { let manifest = try YAMLTestSuiteManifest.load() #expect(manifest.version == 3) #expect(manifest.cases.count == manifest.total) #expect(manifest.total == 402) #expect(Set(manifest.cases.map(\.id)).count == manifest.cases.count) let passing = manifest.cases.count(where: { $0.status == "passing" }) let gaps = manifest.cases.count(where: { $0.status == "gap" }) #expect(passing == manifest.passing) #expect(gaps == manifest.gap) #expect(passing + gaps == manifest.total) let rejects = manifest.cases.count(where: { $0.expected == "reject" }) let parseOK = manifest.cases.count(where: { $0.expected == "parse-ok" }) #expect(rejects + parseOK == manifest.total) for testCase in manifest.cases { #expect(testCase.status == "passing" || testCase.status == "gap", "\(testCase.id): invalid status") #expect(testCase.expected == "reject" || testCase.expected == "parse-ok", "\(testCase.id): invalid expected") #expect(testCase.oracle == "json" || testCase.oracle == "none", "\(testCase.id): invalid oracle") let yaml = try testCase.loadInput() #expect(yaml.utf8.count == testCase.byteCount, "\(testCase.id): byte count drift") #expect(yaml.count(where: { $0 == "\n" }) == testCase.lineCount, "\(testCase.id): line count drift") if testCase.expected == "reject" { #expect(testCase.loadErrorMarker() != nil, "\(testCase.id): reject case is missing its error marker") } if testCase.oracle == "json" { #expect(testCase.loadExpectedJSON() != nil, "\(testCase.id): json oracle case is missing in.json") } else { #expect(testCase.loadExpectedJSON() == nil, "\(testCase.id): oracle is none but an in.json is present") } } } @Test("Every case matches its recorded conformance status") func test_everyCaseMatchesRecordedStatus() throws { guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else { return } let manifest = try YAMLTestSuiteManifest.load() for testCase in manifest.cases { let observed = try testCase.observedStatus() #expect( observed == testCase.status, """ \(testCase.id) "\(testCase.name)": manifest records status '\(testCase.status)' \ but observed '\(observed)'. A 'passing' case that now fails is a regression; \ a 'gap' case that now succeeds should be promoted to 'passing' in the manifest. """, ) } } } // MARK: - Manifest struct YAMLTestSuiteManifest { var version: Int var total: Int var passing: Int var gap: Int var cases: [YAMLTestSuiteCase] static func load() throws -> Self { let url = try #require(Bundle.module.url( forResource: "yaml-test-suite-manifest", withExtension: "yaml", subdirectory: "Fixtures", )) let value = try PureYAML.parse(String(contentsOf: url, encoding: .utf8)) let root = try #require(value.mapping) let version = try #require(root["version"]?.intValue) let total = try #require(root["total"]?.intValue) let passing = try #require(root["passing"]?.intValue) let gap = try #require(root["gap"]?.intValue) let caseValues = try #require(root.sequence("cases")) return try YAMLTestSuiteManifest( version: version, total: total, passing: passing, gap: gap, cases: caseValues.map(YAMLTestSuiteCase.init(value:)), ) } } struct YAMLTestSuiteCase { var id: String var name: String var localPath: String var expected: String var oracle: String var status: String var byteCount: Int var lineCount: Int init(value: PureYAML.Model.Value) throws { let mapping = try #require(value.mapping) id = try mapping.requiredString("id") name = try mapping.requiredString("name") localPath = try mapping.requiredString("localPath") expected = try mapping.requiredString("expected") oracle = try mapping.requiredString("oracle") status = try mapping.requiredString("status") byteCount = try mapping.requiredInt("byteCount") lineCount = try mapping.requiredInt("lineCount") } private var caseSubdirectory: String { let directory = localPath.split(separator: "/").dropLast().joined(separator: "/") return "Fixtures/\(directory)" } func loadInput() throws -> String { let url = try #require(Bundle.module.url(forResource: "in", withExtension: "yaml", subdirectory: caseSubdirectory)) return try String(contentsOf: url, encoding: .utf8) } func loadErrorMarker() -> URL? { Bundle.module.url(forResource: "error", withExtension: nil, subdirectory: caseSubdirectory) } func loadExpectedJSON() -> String? { guard let url = Bundle.module.url(forResource: "in", withExtension: "json", subdirectory: caseSubdirectory) else { return nil } return try? String(contentsOf: url, encoding: .utf8) } /// Recomputes PureYAML's present behavior for this case in the manifest's /// `status` vocabulary, so any drift (a regression in a passing case, or a fix /// that promotes a gap) is detectable. /// /// Conformance is decided by parsing alone: an error case is `passing` when the /// parser rejects it; a value case is `passing` when the parser accepts it and, /// when an in.json oracle is present, the parsed value matches it. Strict /// validation (e.g. duplicate-key detection) is a separate, orthogonal concern /// and is deliberately not part of the conformance decision. func observedStatus() throws -> String { let yaml = try loadInput() var documents: [PureYAML.Stream.Document] = [] var parseThrew = false do { documents = try PureYAML.parseStream(yaml) } catch { parseThrew = true } if expected == "reject" { return parseThrew ? "passing" : "gap" } if parseThrew { return "gap" } guard oracle == "json", let jsonText = loadExpectedJSON() else { return "passing" } // An empty in.json oracle means the conformant output is an empty stream // (no JSON value), as for a comment-only or document-end-only input. A parser // that yields any document then differs from the oracle, so it is a gap. let trimmed = jsonText.trimmingCharacters(in: .whitespacesAndNewlines) if trimmed.isEmpty { return documents.isEmpty ? "passing" : "gap" } guard let expectedDocuments = decodeJSONStream(jsonText) else { return "gap" } if expectedDocuments.count != documents.count { return "gap" } let matches = zip(documents, expectedDocuments).allSatisfy { equalValue($0.value, $1) } return matches ? "passing" : "gap" } } // MARK: - JSON oracle (type-preserving, avoids NSNumber bool ambiguity) indirect enum YAMLTestSuiteJSON: Decodable, Equatable { case null case bool(Bool) case int(Int) case double(Double) case string(String) case array([YAMLTestSuiteJSON]) case object([String: YAMLTestSuiteJSON]) init(from decoder: Decoder) throws { let container = try decoder.singleValueContainer() if container.decodeNil() { self = .null } else if let value = try? container.decode(Bool.self) { self = .bool(value) } else if let value = try? container.decode(Int.self) { self = .int(value) } else if let value = try? container.decode(Double.self) { self = .double(value) } else if let value = try? container.decode(String.self) { self = .string(value) } else if let value = try? container.decode([YAMLTestSuiteJSON].self) { self = .array(value) } else if let value = try? container.decode([String: YAMLTestSuiteJSON].self) { self = .object(value) } else { throw DecodingError.dataCorrupted(.init(codingPath: container.codingPath, debugDescription: "unrecognized JSON")) } } } func decodeJSONStream(_ text: String) -> [YAMLTestSuiteJSON]? { let decoder = JSONDecoder() if let whole = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(text.utf8)) { return [whole] } let parts = splitJSONStream(text) guard !parts.isEmpty else { return nil } var values: [YAMLTestSuiteJSON] = [] for part in parts { guard let value = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(part.utf8)) else { return nil } values.append(value) } return values } /// Splits a concatenated JSON stream (one value per YAML document) into top-level values. func splitJSONStream(_ text: String) -> [String] { var scanner = JSONStreamScanner(text: text) return scanner.scan() } func equalValue(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool { switch (value, json) { case let (.sequence(lhs), .array(rhs)): lhs.count == rhs.count && zip(lhs, rhs).allSatisfy { equalValue($0, $1) } case let (.mapping(lhs), .object(rhs)): equalMapping(lhs, rhs) default: equalScalar(value, json) } } private func equalScalar(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool { switch (value, json) { case (.null, .null): true case let (.bool(lhs), .bool(rhs)): lhs == rhs case let (.int(lhs), .int(rhs)): lhs == rhs case let (.int(lhs), .double(rhs)): Double(lhs) == rhs case let (.double(lhs), .int(rhs)): lhs == Double(rhs) case let (.double(lhs), .double(rhs)): lhs == rhs || (lhs.isNaN && rhs.isNaN) case let (.string(lhs), .string(rhs)): lhs == rhs default: false } } private func equalMapping(_ mapping: PureYAML.Model.Mapping, _ object: [String: YAMLTestSuiteJSON]) -> Bool { guard mapping.pairs.count == object.count else { return false } var dictionary: [String: PureYAML.Model.Value] = [:] for pair in mapping.pairs { dictionary[pair.key] = pair.value } guard dictionary.count == object.count else { return false } for (key, expected) in object { guard let actual = dictionary[key], equalValue(actual, expected) else { return false } } return true } private extension PureYAML.Model.Mapping { func requiredString(_ key: String) throws -> String { try #require(self[key]?.stringValue) } func requiredInt(_ key: String) throws -> Int { try #require(self[key]?.intValue) } } private extension PureYAML.Model.Value { var stringValue: String? { guard case let .string(value) = self else { return nil } return value } var intValue: Int? { guard case let .int(value) = self else { return nil } return value } }