The gap-only manifest stopped guarding a case once it was fixed, so later work silently regressed previously-fixed cases (caught only by an external full-suite run, not the gated sweep or full corpus). Replace it with a full manifest of all 402 official cases (Tests/Fixtures/yaml-test-suite-manifest.yaml), each carrying its conformant expectation and a passing/gap status, with every case's files vendored. The gated sweep now asserts every passing case still passes (the regression guard) and every gap still reproduces, so any regression fails the build. The honest gate-enforced count is 329/402 passing (the earlier 348 came from a more lenient one-time oracle). Repairs three regressions the new gate revealed: the bucket-6 tab rule over-rejected tabs in block-scalar content and as separation before a scalar value (root cause: the batch scanner and the incremental TokenCursor had divergent line-start loops; unified onto a single scanStep, fixing 96NN/00, 96NN/01, R4YG, Y79Y/001, DK95/00, DK95/03, DK95/07, 6CA3, Q5MG, J3BT); an anchor on a prior line before an alias key wrongly rejected 26DV; and a tag-only flow key mis-parsed WZ62. Genuine tab-indentation errors (4EJS, Y79Y/000) still reject.
398 lines
14 KiB
Swift
398 lines
14 KiB
Swift
import Foundation
|
|
@testable import PureYAML
|
|
import Testing
|
|
|
|
/// Full conformance gate against the official YAML test suite
|
|
/// (github.com/yaml/yaml-test-suite, `data` branch, commit
|
|
/// 6ad3d2c62885d82fc349026c136ef560838fdf3d). Every official case (numbered
|
|
/// sub-cases counted) is vendored under `Fixtures/yaml-test-suite/<id>/` and
|
|
/// recorded in `Fixtures/yaml-test-suite-manifest.yaml` with a `status`:
|
|
///
|
|
/// - `passing`: PureYAML already handles the case conformantly (it parses with a
|
|
/// value matching the JSON oracle when one is present, or it rejects an error
|
|
/// case). This is the regression guard: if a later change breaks the case, the
|
|
/// gated sweep fails.
|
|
/// - `gap`: PureYAML does not yet handle the case conformantly. The sweep asserts
|
|
/// the gap still reproduces, so a fix is forced to update the manifest rather
|
|
/// than silently drift.
|
|
///
|
|
/// The behavioral sweep is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1`, mirroring
|
|
/// the full real-corpus sweep, so the default suite stays fast while the whole suite
|
|
/// is guarded on demand. The manifest-integrity test below is always on.
|
|
@Suite("YAML Test Suite Gaps")
|
|
struct YAMLTestSuiteGapsTests {
|
|
@Test("Manifest is consistent with its vendored fixture files")
|
|
func test_manifestMatchesVendoredFixtureFiles() throws {
|
|
let manifest = try YAMLTestSuiteManifest.load()
|
|
|
|
#expect(manifest.version == 2)
|
|
#expect(manifest.cases.count == manifest.total)
|
|
#expect(manifest.total == 402)
|
|
#expect(Set(manifest.cases.map(\.id)).count == manifest.cases.count)
|
|
|
|
let passing = manifest.cases.count(where: { $0.status == "passing" })
|
|
let gaps = manifest.cases.count(where: { $0.status == "gap" })
|
|
#expect(passing == manifest.passing)
|
|
#expect(gaps == manifest.gap)
|
|
#expect(passing + gaps == manifest.total)
|
|
|
|
let rejects = manifest.cases.count(where: { $0.expected == "reject" })
|
|
let parseOK = manifest.cases.count(where: { $0.expected == "parse-ok" })
|
|
#expect(rejects + parseOK == manifest.total)
|
|
|
|
for testCase in manifest.cases {
|
|
#expect(testCase.status == "passing" || testCase.status == "gap", "\(testCase.id): invalid status")
|
|
#expect(testCase.expected == "reject" || testCase.expected == "parse-ok", "\(testCase.id): invalid expected")
|
|
#expect(testCase.oracle == "json" || testCase.oracle == "none", "\(testCase.id): invalid oracle")
|
|
|
|
let yaml = try testCase.loadInput()
|
|
#expect(yaml.utf8.count == testCase.byteCount, "\(testCase.id): byte count drift")
|
|
#expect(yaml.count(where: { $0 == "\n" }) == testCase.lineCount, "\(testCase.id): line count drift")
|
|
|
|
if testCase.expected == "reject" {
|
|
#expect(testCase.loadErrorMarker() != nil, "\(testCase.id): reject case is missing its error marker")
|
|
}
|
|
if testCase.oracle == "json" {
|
|
#expect(testCase.loadExpectedJSON() != nil, "\(testCase.id): json oracle case is missing in.json")
|
|
} else {
|
|
#expect(testCase.loadExpectedJSON() == nil, "\(testCase.id): oracle is none but an in.json is present")
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test("Every case matches its recorded conformance status")
|
|
func test_everyCaseMatchesRecordedStatus() throws {
|
|
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
|
|
return
|
|
}
|
|
|
|
let manifest = try YAMLTestSuiteManifest.load()
|
|
for testCase in manifest.cases {
|
|
let observed = try testCase.observedStatus()
|
|
#expect(
|
|
observed == testCase.status,
|
|
"""
|
|
\(testCase.id) "\(testCase.name)": manifest records status '\(testCase.status)' \
|
|
but observed '\(observed)'. A 'passing' case that now fails is a regression; \
|
|
a 'gap' case that now succeeds should be promoted to 'passing' in the manifest.
|
|
""",
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
// MARK: - Manifest
|
|
|
|
struct YAMLTestSuiteManifest {
|
|
var version: Int
|
|
var total: Int
|
|
var passing: Int
|
|
var gap: Int
|
|
var cases: [YAMLTestSuiteCase]
|
|
|
|
static func load() throws -> Self {
|
|
let url = try #require(Bundle.module.url(
|
|
forResource: "yaml-test-suite-manifest",
|
|
withExtension: "yaml",
|
|
subdirectory: "Fixtures",
|
|
))
|
|
let value = try PureYAML.parse(String(contentsOf: url, encoding: .utf8))
|
|
let root = try #require(value.mapping)
|
|
let version = try #require(root["version"]?.intValue)
|
|
let total = try #require(root["total"]?.intValue)
|
|
let passing = try #require(root["passing"]?.intValue)
|
|
let gap = try #require(root["gap"]?.intValue)
|
|
let caseValues = try #require(root.sequence("cases"))
|
|
return try YAMLTestSuiteManifest(
|
|
version: version,
|
|
total: total,
|
|
passing: passing,
|
|
gap: gap,
|
|
cases: caseValues.map(YAMLTestSuiteCase.init(value:)),
|
|
)
|
|
}
|
|
}
|
|
|
|
struct YAMLTestSuiteCase {
|
|
var id: String
|
|
var name: String
|
|
var localPath: String
|
|
var expected: String
|
|
var oracle: String
|
|
var status: String
|
|
var byteCount: Int
|
|
var lineCount: Int
|
|
|
|
init(value: PureYAML.Model.Value) throws {
|
|
let mapping = try #require(value.mapping)
|
|
id = try mapping.requiredString("id")
|
|
name = try mapping.requiredString("name")
|
|
localPath = try mapping.requiredString("localPath")
|
|
expected = try mapping.requiredString("expected")
|
|
oracle = try mapping.requiredString("oracle")
|
|
status = try mapping.requiredString("status")
|
|
byteCount = try mapping.requiredInt("byteCount")
|
|
lineCount = try mapping.requiredInt("lineCount")
|
|
}
|
|
|
|
private var caseSubdirectory: String {
|
|
let directory = localPath.split(separator: "/").dropLast().joined(separator: "/")
|
|
return "Fixtures/\(directory)"
|
|
}
|
|
|
|
func loadInput() throws -> String {
|
|
let url = try #require(Bundle.module.url(forResource: "in", withExtension: "yaml", subdirectory: caseSubdirectory))
|
|
return try String(contentsOf: url, encoding: .utf8)
|
|
}
|
|
|
|
func loadErrorMarker() -> URL? {
|
|
Bundle.module.url(forResource: "error", withExtension: nil, subdirectory: caseSubdirectory)
|
|
}
|
|
|
|
func loadExpectedJSON() -> String? {
|
|
guard let url = Bundle.module.url(forResource: "in", withExtension: "json", subdirectory: caseSubdirectory) else {
|
|
return nil
|
|
}
|
|
return try? String(contentsOf: url, encoding: .utf8)
|
|
}
|
|
|
|
/// Recomputes PureYAML's present behavior for this case in the manifest's
|
|
/// `status` vocabulary, so any drift (a regression in a passing case, or a fix
|
|
/// that promotes a gap) is detectable.
|
|
///
|
|
/// Conformance is decided by parsing alone: an error case is `passing` when the
|
|
/// parser rejects it; a value case is `passing` when the parser accepts it and,
|
|
/// when an in.json oracle is present, the parsed value matches it. Strict
|
|
/// validation (e.g. duplicate-key detection) is a separate, orthogonal concern
|
|
/// and is deliberately not part of the conformance decision.
|
|
func observedStatus() throws -> String {
|
|
let yaml = try loadInput()
|
|
var documents: [PureYAML.Stream.Document] = []
|
|
var parseThrew = false
|
|
do {
|
|
documents = try PureYAML.parseStream(yaml)
|
|
} catch {
|
|
parseThrew = true
|
|
}
|
|
|
|
if expected == "reject" {
|
|
return parseThrew ? "passing" : "gap"
|
|
}
|
|
if parseThrew {
|
|
return "gap"
|
|
}
|
|
guard oracle == "json", let jsonText = loadExpectedJSON() else {
|
|
return "passing"
|
|
}
|
|
// An empty in.json oracle means the conformant output is an empty stream
|
|
// (no JSON value), as for a comment-only or document-end-only input. A parser
|
|
// that yields any document then differs from the oracle, so it is a gap.
|
|
let trimmed = jsonText.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
if trimmed.isEmpty {
|
|
return documents.isEmpty ? "passing" : "gap"
|
|
}
|
|
guard let expectedDocuments = decodeJSONStream(jsonText) else {
|
|
return "gap"
|
|
}
|
|
if expectedDocuments.count != documents.count {
|
|
return "gap"
|
|
}
|
|
let matches = zip(documents, expectedDocuments).allSatisfy { equalValue($0.value, $1) }
|
|
return matches ? "passing" : "gap"
|
|
}
|
|
}
|
|
|
|
// MARK: - JSON oracle (type-preserving, avoids NSNumber bool ambiguity)
|
|
|
|
indirect enum YAMLTestSuiteJSON: Decodable, Equatable {
|
|
case null
|
|
case bool(Bool)
|
|
case int(Int)
|
|
case double(Double)
|
|
case string(String)
|
|
case array([YAMLTestSuiteJSON])
|
|
case object([String: YAMLTestSuiteJSON])
|
|
|
|
init(from decoder: Decoder) throws {
|
|
let container = try decoder.singleValueContainer()
|
|
if container.decodeNil() {
|
|
self = .null
|
|
} else if let value = try? container.decode(Bool.self) {
|
|
self = .bool(value)
|
|
} else if let value = try? container.decode(Int.self) {
|
|
self = .int(value)
|
|
} else if let value = try? container.decode(Double.self) {
|
|
self = .double(value)
|
|
} else if let value = try? container.decode(String.self) {
|
|
self = .string(value)
|
|
} else if let value = try? container.decode([YAMLTestSuiteJSON].self) {
|
|
self = .array(value)
|
|
} else if let value = try? container.decode([String: YAMLTestSuiteJSON].self) {
|
|
self = .object(value)
|
|
} else {
|
|
throw DecodingError.dataCorrupted(.init(codingPath: container.codingPath, debugDescription: "unrecognized JSON"))
|
|
}
|
|
}
|
|
}
|
|
|
|
func decodeJSONStream(_ text: String) -> [YAMLTestSuiteJSON]? {
|
|
let decoder = JSONDecoder()
|
|
if let whole = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(text.utf8)) {
|
|
return [whole]
|
|
}
|
|
let parts = splitJSONStream(text)
|
|
guard !parts.isEmpty else {
|
|
return nil
|
|
}
|
|
var values: [YAMLTestSuiteJSON] = []
|
|
for part in parts {
|
|
guard let value = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(part.utf8)) else {
|
|
return nil
|
|
}
|
|
values.append(value)
|
|
}
|
|
return values
|
|
}
|
|
|
|
/// Splits a concatenated JSON stream (one value per YAML document) into top-level values.
|
|
func splitJSONStream(_ text: String) -> [String] {
|
|
var scanner = JSONStreamScanner(text: text)
|
|
return scanner.scan()
|
|
}
|
|
|
|
private struct JSONStreamScanner {
|
|
let text: String
|
|
var out: [String] = []
|
|
var depth = 0
|
|
var inString = false
|
|
var escaped = false
|
|
var start: String.Index?
|
|
|
|
mutating func scan() -> [String] {
|
|
var index = text.startIndex
|
|
while index < text.endIndex {
|
|
if inString {
|
|
stepInString(text[index])
|
|
} else {
|
|
stepStructural(text[index], at: index)
|
|
}
|
|
index = text.index(after: index)
|
|
}
|
|
if let begin = start {
|
|
out.append(String(text[begin...]))
|
|
}
|
|
return out.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty }
|
|
}
|
|
|
|
private mutating func stepInString(_ character: Character) {
|
|
if escaped {
|
|
escaped = false
|
|
} else if character == "\\" {
|
|
escaped = true
|
|
} else if character == "\"" {
|
|
inString = false
|
|
}
|
|
}
|
|
|
|
private mutating func stepStructural(_ character: Character, at index: String.Index) {
|
|
switch character {
|
|
case "\"":
|
|
if start == nil { start = index }
|
|
inString = true
|
|
case "{", "[":
|
|
if start == nil { start = index }
|
|
depth += 1
|
|
case "}", "]":
|
|
depth -= 1
|
|
if depth == 0, let begin = start {
|
|
out.append(String(text[begin ... index]))
|
|
start = nil
|
|
}
|
|
case " ", "\n", "\r", "\t":
|
|
if depth == 0, let begin = start {
|
|
out.append(String(text[begin ..< index]))
|
|
start = nil
|
|
}
|
|
default:
|
|
if depth == 0, start == nil { start = index }
|
|
}
|
|
}
|
|
}
|
|
|
|
func equalValue(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool {
|
|
switch (value, json) {
|
|
case let (.sequence(lhs), .array(rhs)):
|
|
lhs.count == rhs.count && zip(lhs, rhs).allSatisfy { equalValue($0, $1) }
|
|
case let (.mapping(lhs), .object(rhs)):
|
|
equalMapping(lhs, rhs)
|
|
default:
|
|
equalScalar(value, json)
|
|
}
|
|
}
|
|
|
|
private func equalScalar(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool {
|
|
switch (value, json) {
|
|
case (.null, .null):
|
|
true
|
|
case let (.bool(lhs), .bool(rhs)):
|
|
lhs == rhs
|
|
case let (.int(lhs), .int(rhs)):
|
|
lhs == rhs
|
|
case let (.int(lhs), .double(rhs)):
|
|
Double(lhs) == rhs
|
|
case let (.double(lhs), .int(rhs)):
|
|
lhs == Double(rhs)
|
|
case let (.double(lhs), .double(rhs)):
|
|
lhs == rhs || (lhs.isNaN && rhs.isNaN)
|
|
case let (.string(lhs), .string(rhs)):
|
|
lhs == rhs
|
|
default:
|
|
false
|
|
}
|
|
}
|
|
|
|
private func equalMapping(_ mapping: PureYAML.Model.Mapping, _ object: [String: YAMLTestSuiteJSON]) -> Bool {
|
|
guard mapping.pairs.count == object.count else {
|
|
return false
|
|
}
|
|
var dictionary: [String: PureYAML.Model.Value] = [:]
|
|
for pair in mapping.pairs {
|
|
dictionary[pair.key] = pair.value
|
|
}
|
|
guard dictionary.count == object.count else {
|
|
return false
|
|
}
|
|
for (key, expected) in object {
|
|
guard let actual = dictionary[key], equalValue(actual, expected) else {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
private extension PureYAML.Model.Mapping {
|
|
func requiredString(_ key: String) throws -> String {
|
|
try #require(self[key]?.stringValue)
|
|
}
|
|
|
|
func requiredInt(_ key: String) throws -> Int {
|
|
try #require(self[key]?.intValue)
|
|
}
|
|
}
|
|
|
|
private extension PureYAML.Model.Value {
|
|
var stringValue: String? {
|
|
guard case let .string(value) = self else {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
|
|
var intValue: Int? {
|
|
guard case let .int(value) = self else {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
}
|