PureYAML/Tests/YAMLTestSuiteGapsTests.swift
Mihaela Mihaljevic ce0585f77e test(conformance): track all 402 suite cases; fix 3 regressions
The gap-only manifest stopped guarding a case once it was fixed, so later
work silently regressed previously-fixed cases (caught only by an external
full-suite run, not the gated sweep or full corpus). Replace it with a full
manifest of all 402 official cases (Tests/Fixtures/yaml-test-suite-manifest.yaml),
each carrying its conformant expectation and a passing/gap status, with every
case's files vendored. The gated sweep now asserts every passing case still
passes (the regression guard) and every gap still reproduces, so any regression
fails the build.

The honest gate-enforced count is 329/402 passing (the earlier 348 came from a
more lenient one-time oracle).

Repairs three regressions the new gate revealed: the bucket-6 tab rule
over-rejected tabs in block-scalar content and as separation before a scalar
value (root cause: the batch scanner and the incremental TokenCursor had
divergent line-start loops; unified onto a single scanStep, fixing 96NN/00,
96NN/01, R4YG, Y79Y/001, DK95/00, DK95/03, DK95/07, 6CA3, Q5MG, J3BT); an anchor
on a prior line before an alias key wrongly rejected 26DV; and a tag-only flow
key mis-parsed WZ62. Genuine tab-indentation errors (4EJS, Y79Y/000) still reject.
2026-06-24 00:16:28 +02:00

398 lines
14 KiB
Swift

import Foundation
@testable import PureYAML
import Testing
/// Full conformance gate against the official YAML test suite
/// (github.com/yaml/yaml-test-suite, `data` branch, commit
/// 6ad3d2c62885d82fc349026c136ef560838fdf3d). Every official case (numbered
/// sub-cases counted) is vendored under `Fixtures/yaml-test-suite/<id>/` and
/// recorded in `Fixtures/yaml-test-suite-manifest.yaml` with a `status`:
///
/// - `passing`: PureYAML already handles the case conformantly (it parses with a
/// value matching the JSON oracle when one is present, or it rejects an error
/// case). This is the regression guard: if a later change breaks the case, the
/// gated sweep fails.
/// - `gap`: PureYAML does not yet handle the case conformantly. The sweep asserts
/// the gap still reproduces, so a fix is forced to update the manifest rather
/// than silently drift.
///
/// The behavioral sweep is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1`, mirroring
/// the full real-corpus sweep, so the default suite stays fast while the whole suite
/// is guarded on demand. The manifest-integrity test below is always on.
@Suite("YAML Test Suite Gaps")
struct YAMLTestSuiteGapsTests {
@Test("Manifest is consistent with its vendored fixture files")
func test_manifestMatchesVendoredFixtureFiles() throws {
let manifest = try YAMLTestSuiteManifest.load()
#expect(manifest.version == 2)
#expect(manifest.cases.count == manifest.total)
#expect(manifest.total == 402)
#expect(Set(manifest.cases.map(\.id)).count == manifest.cases.count)
let passing = manifest.cases.count(where: { $0.status == "passing" })
let gaps = manifest.cases.count(where: { $0.status == "gap" })
#expect(passing == manifest.passing)
#expect(gaps == manifest.gap)
#expect(passing + gaps == manifest.total)
let rejects = manifest.cases.count(where: { $0.expected == "reject" })
let parseOK = manifest.cases.count(where: { $0.expected == "parse-ok" })
#expect(rejects + parseOK == manifest.total)
for testCase in manifest.cases {
#expect(testCase.status == "passing" || testCase.status == "gap", "\(testCase.id): invalid status")
#expect(testCase.expected == "reject" || testCase.expected == "parse-ok", "\(testCase.id): invalid expected")
#expect(testCase.oracle == "json" || testCase.oracle == "none", "\(testCase.id): invalid oracle")
let yaml = try testCase.loadInput()
#expect(yaml.utf8.count == testCase.byteCount, "\(testCase.id): byte count drift")
#expect(yaml.count(where: { $0 == "\n" }) == testCase.lineCount, "\(testCase.id): line count drift")
if testCase.expected == "reject" {
#expect(testCase.loadErrorMarker() != nil, "\(testCase.id): reject case is missing its error marker")
}
if testCase.oracle == "json" {
#expect(testCase.loadExpectedJSON() != nil, "\(testCase.id): json oracle case is missing in.json")
} else {
#expect(testCase.loadExpectedJSON() == nil, "\(testCase.id): oracle is none but an in.json is present")
}
}
}
@Test("Every case matches its recorded conformance status")
func test_everyCaseMatchesRecordedStatus() throws {
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
return
}
let manifest = try YAMLTestSuiteManifest.load()
for testCase in manifest.cases {
let observed = try testCase.observedStatus()
#expect(
observed == testCase.status,
"""
\(testCase.id) "\(testCase.name)": manifest records status '\(testCase.status)' \
but observed '\(observed)'. A 'passing' case that now fails is a regression; \
a 'gap' case that now succeeds should be promoted to 'passing' in the manifest.
""",
)
}
}
}
// MARK: - Manifest
struct YAMLTestSuiteManifest {
var version: Int
var total: Int
var passing: Int
var gap: Int
var cases: [YAMLTestSuiteCase]
static func load() throws -> Self {
let url = try #require(Bundle.module.url(
forResource: "yaml-test-suite-manifest",
withExtension: "yaml",
subdirectory: "Fixtures",
))
let value = try PureYAML.parse(String(contentsOf: url, encoding: .utf8))
let root = try #require(value.mapping)
let version = try #require(root["version"]?.intValue)
let total = try #require(root["total"]?.intValue)
let passing = try #require(root["passing"]?.intValue)
let gap = try #require(root["gap"]?.intValue)
let caseValues = try #require(root.sequence("cases"))
return try YAMLTestSuiteManifest(
version: version,
total: total,
passing: passing,
gap: gap,
cases: caseValues.map(YAMLTestSuiteCase.init(value:)),
)
}
}
struct YAMLTestSuiteCase {
var id: String
var name: String
var localPath: String
var expected: String
var oracle: String
var status: String
var byteCount: Int
var lineCount: Int
init(value: PureYAML.Model.Value) throws {
let mapping = try #require(value.mapping)
id = try mapping.requiredString("id")
name = try mapping.requiredString("name")
localPath = try mapping.requiredString("localPath")
expected = try mapping.requiredString("expected")
oracle = try mapping.requiredString("oracle")
status = try mapping.requiredString("status")
byteCount = try mapping.requiredInt("byteCount")
lineCount = try mapping.requiredInt("lineCount")
}
private var caseSubdirectory: String {
let directory = localPath.split(separator: "/").dropLast().joined(separator: "/")
return "Fixtures/\(directory)"
}
func loadInput() throws -> String {
let url = try #require(Bundle.module.url(forResource: "in", withExtension: "yaml", subdirectory: caseSubdirectory))
return try String(contentsOf: url, encoding: .utf8)
}
func loadErrorMarker() -> URL? {
Bundle.module.url(forResource: "error", withExtension: nil, subdirectory: caseSubdirectory)
}
func loadExpectedJSON() -> String? {
guard let url = Bundle.module.url(forResource: "in", withExtension: "json", subdirectory: caseSubdirectory) else {
return nil
}
return try? String(contentsOf: url, encoding: .utf8)
}
/// Recomputes PureYAML's present behavior for this case in the manifest's
/// `status` vocabulary, so any drift (a regression in a passing case, or a fix
/// that promotes a gap) is detectable.
///
/// Conformance is decided by parsing alone: an error case is `passing` when the
/// parser rejects it; a value case is `passing` when the parser accepts it and,
/// when an in.json oracle is present, the parsed value matches it. Strict
/// validation (e.g. duplicate-key detection) is a separate, orthogonal concern
/// and is deliberately not part of the conformance decision.
func observedStatus() throws -> String {
let yaml = try loadInput()
var documents: [PureYAML.Stream.Document] = []
var parseThrew = false
do {
documents = try PureYAML.parseStream(yaml)
} catch {
parseThrew = true
}
if expected == "reject" {
return parseThrew ? "passing" : "gap"
}
if parseThrew {
return "gap"
}
guard oracle == "json", let jsonText = loadExpectedJSON() else {
return "passing"
}
// An empty in.json oracle means the conformant output is an empty stream
// (no JSON value), as for a comment-only or document-end-only input. A parser
// that yields any document then differs from the oracle, so it is a gap.
let trimmed = jsonText.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty {
return documents.isEmpty ? "passing" : "gap"
}
guard let expectedDocuments = decodeJSONStream(jsonText) else {
return "gap"
}
if expectedDocuments.count != documents.count {
return "gap"
}
let matches = zip(documents, expectedDocuments).allSatisfy { equalValue($0.value, $1) }
return matches ? "passing" : "gap"
}
}
// MARK: - JSON oracle (type-preserving, avoids NSNumber bool ambiguity)
indirect enum YAMLTestSuiteJSON: Decodable, Equatable {
case null
case bool(Bool)
case int(Int)
case double(Double)
case string(String)
case array([YAMLTestSuiteJSON])
case object([String: YAMLTestSuiteJSON])
init(from decoder: Decoder) throws {
let container = try decoder.singleValueContainer()
if container.decodeNil() {
self = .null
} else if let value = try? container.decode(Bool.self) {
self = .bool(value)
} else if let value = try? container.decode(Int.self) {
self = .int(value)
} else if let value = try? container.decode(Double.self) {
self = .double(value)
} else if let value = try? container.decode(String.self) {
self = .string(value)
} else if let value = try? container.decode([YAMLTestSuiteJSON].self) {
self = .array(value)
} else if let value = try? container.decode([String: YAMLTestSuiteJSON].self) {
self = .object(value)
} else {
throw DecodingError.dataCorrupted(.init(codingPath: container.codingPath, debugDescription: "unrecognized JSON"))
}
}
}
func decodeJSONStream(_ text: String) -> [YAMLTestSuiteJSON]? {
let decoder = JSONDecoder()
if let whole = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(text.utf8)) {
return [whole]
}
let parts = splitJSONStream(text)
guard !parts.isEmpty else {
return nil
}
var values: [YAMLTestSuiteJSON] = []
for part in parts {
guard let value = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(part.utf8)) else {
return nil
}
values.append(value)
}
return values
}
/// Splits a concatenated JSON stream (one value per YAML document) into top-level values.
func splitJSONStream(_ text: String) -> [String] {
var scanner = JSONStreamScanner(text: text)
return scanner.scan()
}
private struct JSONStreamScanner {
let text: String
var out: [String] = []
var depth = 0
var inString = false
var escaped = false
var start: String.Index?
mutating func scan() -> [String] {
var index = text.startIndex
while index < text.endIndex {
if inString {
stepInString(text[index])
} else {
stepStructural(text[index], at: index)
}
index = text.index(after: index)
}
if let begin = start {
out.append(String(text[begin...]))
}
return out.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty }
}
private mutating func stepInString(_ character: Character) {
if escaped {
escaped = false
} else if character == "\\" {
escaped = true
} else if character == "\"" {
inString = false
}
}
private mutating func stepStructural(_ character: Character, at index: String.Index) {
switch character {
case "\"":
if start == nil { start = index }
inString = true
case "{", "[":
if start == nil { start = index }
depth += 1
case "}", "]":
depth -= 1
if depth == 0, let begin = start {
out.append(String(text[begin ... index]))
start = nil
}
case " ", "\n", "\r", "\t":
if depth == 0, let begin = start {
out.append(String(text[begin ..< index]))
start = nil
}
default:
if depth == 0, start == nil { start = index }
}
}
}
func equalValue(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool {
switch (value, json) {
case let (.sequence(lhs), .array(rhs)):
lhs.count == rhs.count && zip(lhs, rhs).allSatisfy { equalValue($0, $1) }
case let (.mapping(lhs), .object(rhs)):
equalMapping(lhs, rhs)
default:
equalScalar(value, json)
}
}
private func equalScalar(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool {
switch (value, json) {
case (.null, .null):
true
case let (.bool(lhs), .bool(rhs)):
lhs == rhs
case let (.int(lhs), .int(rhs)):
lhs == rhs
case let (.int(lhs), .double(rhs)):
Double(lhs) == rhs
case let (.double(lhs), .int(rhs)):
lhs == Double(rhs)
case let (.double(lhs), .double(rhs)):
lhs == rhs || (lhs.isNaN && rhs.isNaN)
case let (.string(lhs), .string(rhs)):
lhs == rhs
default:
false
}
}
private func equalMapping(_ mapping: PureYAML.Model.Mapping, _ object: [String: YAMLTestSuiteJSON]) -> Bool {
guard mapping.pairs.count == object.count else {
return false
}
var dictionary: [String: PureYAML.Model.Value] = [:]
for pair in mapping.pairs {
dictionary[pair.key] = pair.value
}
guard dictionary.count == object.count else {
return false
}
for (key, expected) in object {
guard let actual = dictionary[key], equalValue(actual, expected) else {
return false
}
}
return true
}
private extension PureYAML.Model.Mapping {
func requiredString(_ key: String) throws -> String {
try #require(self[key]?.stringValue)
}
func requiredInt(_ key: String) throws -> Int {
try #require(self[key]?.intValue)
}
}
private extension PureYAML.Model.Value {
var stringValue: String? {
guard case let .string(value) = self else {
return nil
}
return value
}
var intValue: Int? {
guard case let .int(value) = self else {
return nil
}
return value
}
}