PureYAML/Tests/YAMLTestSuiteGapsTests.swift
Mihaela Mihaljevic cd05976fac feat(parsing): handle directives and document-end boundaries
Three root causes: (1) a '%' begins a directive only in directive position (before document content, outside flow); everywhere else it is plain content, fixing misfires inside plain scalars, flow mappings, and block scalars. (2) %YAML accepts any 1.x (a higher minor is treated as 1.2 without error), rejects major != 1 and duplicate %YAML, and requires a following '---'; reserved and %TAG directives are accepted and ignored. A directive not followed by a document start is rejected with the new directiveWithoutDocument error. (3) An explicit '...' end marker closes the current document so following content begins a fresh document in parseStream; single-document parse still rejects multi-document input at compose time.

Tests/UnsupportedYAMLGapsFixtures.swift, StreamParsingTests, ScannerTests, and ParsingErrorTests were updated in lockstep to assert the now-correct behavior (percent-after-content is plain text; '...' then content is two documents; %YAML 1.3 is valid while 2.0 and duplicate %YAML are rejected); no test was weakened and multi-document rejection for single-document parse is still covered.

Flips 15 official YAML test suite cases (295/402); SF5V, 9MMA, B63P, H7TQ now correctly rejected. 6LVF and W4TN are reclassified to json-mismatch (blocked by other scalar gaps). Updates the manifest, fixtures, counts, and roadmap.
2026-06-23 21:31:58 +02:00

363 lines
12 KiB
Swift

import Foundation
@testable import PureYAML
import Testing
/// Conformance gaps measured against the official YAML test suite
/// (github.com/yaml/yaml-test-suite, `data` branch). The vendored cases under
/// `Fixtures/yaml-test-suite/` are exactly the suite entries PureYAML does not
/// yet handle correctly. The manifest records each gap's current behavior so a
/// future fix is forced to update it (the gated test below reports drift).
///
/// The behavioral sweep is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1`, mirroring
/// the full real-corpus sweep, so the default suite stays green while the gaps are
/// tracked rather than silenced.
@Suite("YAML Test Suite Gaps")
struct YAMLTestSuiteGapsTests {
@Test("Gap manifest matches vendored fixture files")
func test_gapManifestMatchesVendoredFixtureFiles() throws {
let manifest = try YAMLTestSuiteManifest.load()
#expect(manifest.version == 1)
#expect(manifest.cases.count == 107)
#expect(manifest.cases.count(where: { $0.category == "false-reject" }) == 40)
#expect(manifest.cases.count(where: { $0.category == "json-mismatch" }) == 20)
#expect(manifest.cases.count(where: { $0.category == "accepted-invalid" }) == 47)
#expect(Set(manifest.cases.map(\.id)).count == manifest.cases.count)
for testCase in manifest.cases {
let yaml = try testCase.loadInput()
#expect(yaml.utf8.count == testCase.byteCount, "\(testCase.id): byte count drift")
#expect(yaml.count(where: { $0 == "\n" }) == testCase.lineCount, "\(testCase.id): line count drift")
switch testCase.category {
case "accepted-invalid":
#expect(testCase.expected == "reject")
#expect(testCase.loadErrorMarker() != nil, "\(testCase.id): missing error marker")
default:
#expect(testCase.expected == "parse-ok")
}
if testCase.oracle == "json" {
#expect(testCase.loadExpectedJSON() != nil, "\(testCase.id): missing in.json oracle")
}
}
}
@Test("Recorded conformance gaps still reproduce")
func test_recordedConformanceGapsStillReproduce() throws {
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
return
}
let manifest = try YAMLTestSuiteManifest.load()
for testCase in manifest.cases {
let observed = try testCase.observedStatus()
#expect(
observed == testCase.current,
"\(testCase.id) \"\(testCase.name)\": recorded gap '\(testCase.current)' but observed '\(observed)'. If this case is now handled correctly, update the manifest.",
)
}
}
}
// MARK: - Manifest
struct YAMLTestSuiteManifest {
var version: Int
var cases: [YAMLTestSuiteCase]
static func load() throws -> Self {
let url = try #require(Bundle.module.url(
forResource: "yaml-test-suite-gaps",
withExtension: "yaml",
subdirectory: "Fixtures",
))
let value = try PureYAML.parse(String(contentsOf: url, encoding: .utf8))
let root = try #require(value.mapping)
let version = try #require(root["version"]?.intValue)
let caseValues = try #require(root.sequence("cases"))
return try YAMLTestSuiteManifest(
version: version,
cases: caseValues.map(YAMLTestSuiteCase.init(value:)),
)
}
}
struct YAMLTestSuiteCase {
var id: String
var name: String
var localPath: String
var category: String
var expected: String
var oracle: String
var current: String
var byteCount: Int
var lineCount: Int
init(value: PureYAML.Model.Value) throws {
let mapping = try #require(value.mapping)
id = try mapping.requiredString("id")
name = try mapping.requiredString("name")
localPath = try mapping.requiredString("localPath")
category = try mapping.requiredString("category")
expected = try mapping.requiredString("expected")
oracle = try mapping.requiredString("oracle")
current = try mapping.requiredString("current")
byteCount = try mapping.requiredInt("byteCount")
lineCount = try mapping.requiredInt("lineCount")
}
private var caseSubdirectory: String {
let directory = localPath.split(separator: "/").dropLast().joined(separator: "/")
return "Fixtures/\(directory)"
}
func loadInput() throws -> String {
let url = try #require(Bundle.module.url(forResource: "in", withExtension: "yaml", subdirectory: caseSubdirectory))
return try String(contentsOf: url, encoding: .utf8)
}
func loadErrorMarker() -> URL? {
Bundle.module.url(forResource: "error", withExtension: nil, subdirectory: caseSubdirectory)
}
func loadExpectedJSON() -> String? {
guard let url = Bundle.module.url(forResource: "in", withExtension: "json", subdirectory: caseSubdirectory) else {
return nil
}
return try? String(contentsOf: url, encoding: .utf8)
}
/// Recomputes PureYAML's present behavior for this case, in the same vocabulary
/// the manifest's `current` field uses, so drift (including fixes) is detectable.
func observedStatus() throws -> String {
let yaml = try loadInput()
var documents: [PureYAML.Stream.Document] = []
var parseThrew = false
do {
documents = try PureYAML.parseStream(yaml)
} catch {
parseThrew = true
}
var validateThrew = false
if !parseThrew {
do {
_ = try PureYAML.validate(documents)
} catch {
validateThrew = true
}
}
if category == "accepted-invalid" {
return (parseThrew || validateThrew) ? "reject" : "accepted"
}
if parseThrew {
return "parse-error"
}
guard oracle == "json", let jsonText = loadExpectedJSON(), let expectedDocuments = decodeJSONStream(jsonText) else {
return "parse-ok"
}
if expectedDocuments.count != documents.count {
return "value-mismatch"
}
let matches = zip(documents, expectedDocuments).allSatisfy { equalValue($0.value, $1) }
return matches ? "value-match" : "value-mismatch"
}
}
// MARK: - JSON oracle (type-preserving, avoids NSNumber bool ambiguity)
indirect enum YAMLTestSuiteJSON: Decodable, Equatable {
case null
case bool(Bool)
case int(Int)
case double(Double)
case string(String)
case array([YAMLTestSuiteJSON])
case object([String: YAMLTestSuiteJSON])
init(from decoder: Decoder) throws {
let container = try decoder.singleValueContainer()
if container.decodeNil() {
self = .null
} else if let value = try? container.decode(Bool.self) {
self = .bool(value)
} else if let value = try? container.decode(Int.self) {
self = .int(value)
} else if let value = try? container.decode(Double.self) {
self = .double(value)
} else if let value = try? container.decode(String.self) {
self = .string(value)
} else if let value = try? container.decode([YAMLTestSuiteJSON].self) {
self = .array(value)
} else if let value = try? container.decode([String: YAMLTestSuiteJSON].self) {
self = .object(value)
} else {
throw DecodingError.dataCorrupted(.init(codingPath: container.codingPath, debugDescription: "unrecognized JSON"))
}
}
}
func decodeJSONStream(_ text: String) -> [YAMLTestSuiteJSON]? {
let decoder = JSONDecoder()
if let whole = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(text.utf8)) {
return [whole]
}
let parts = splitJSONStream(text)
guard !parts.isEmpty else {
return nil
}
var values: [YAMLTestSuiteJSON] = []
for part in parts {
guard let value = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(part.utf8)) else {
return nil
}
values.append(value)
}
return values
}
/// Splits a concatenated JSON stream (one value per YAML document) into top-level values.
func splitJSONStream(_ text: String) -> [String] {
var scanner = JSONStreamScanner(text: text)
return scanner.scan()
}
private struct JSONStreamScanner {
let text: String
var out: [String] = []
var depth = 0
var inString = false
var escaped = false
var start: String.Index?
mutating func scan() -> [String] {
var index = text.startIndex
while index < text.endIndex {
if inString {
stepInString(text[index])
} else {
stepStructural(text[index], at: index)
}
index = text.index(after: index)
}
if let begin = start {
out.append(String(text[begin...]))
}
return out.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty }
}
private mutating func stepInString(_ character: Character) {
if escaped {
escaped = false
} else if character == "\\" {
escaped = true
} else if character == "\"" {
inString = false
}
}
private mutating func stepStructural(_ character: Character, at index: String.Index) {
switch character {
case "\"":
if start == nil { start = index }
inString = true
case "{", "[":
if start == nil { start = index }
depth += 1
case "}", "]":
depth -= 1
if depth == 0, let begin = start {
out.append(String(text[begin ... index]))
start = nil
}
case " ", "\n", "\r", "\t":
if depth == 0, let begin = start {
out.append(String(text[begin ..< index]))
start = nil
}
default:
if depth == 0, start == nil { start = index }
}
}
}
func equalValue(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool {
switch (value, json) {
case let (.sequence(lhs), .array(rhs)):
lhs.count == rhs.count && zip(lhs, rhs).allSatisfy { equalValue($0, $1) }
case let (.mapping(lhs), .object(rhs)):
equalMapping(lhs, rhs)
default:
equalScalar(value, json)
}
}
private func equalScalar(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool {
switch (value, json) {
case (.null, .null):
true
case let (.bool(lhs), .bool(rhs)):
lhs == rhs
case let (.int(lhs), .int(rhs)):
lhs == rhs
case let (.int(lhs), .double(rhs)):
Double(lhs) == rhs
case let (.double(lhs), .int(rhs)):
lhs == Double(rhs)
case let (.double(lhs), .double(rhs)):
lhs == rhs || (lhs.isNaN && rhs.isNaN)
case let (.string(lhs), .string(rhs)):
lhs == rhs
default:
false
}
}
private func equalMapping(_ mapping: PureYAML.Model.Mapping, _ object: [String: YAMLTestSuiteJSON]) -> Bool {
guard mapping.pairs.count == object.count else {
return false
}
var dictionary: [String: PureYAML.Model.Value] = [:]
for pair in mapping.pairs {
dictionary[pair.key] = pair.value
}
guard dictionary.count == object.count else {
return false
}
for (key, expected) in object {
guard let actual = dictionary[key], equalValue(actual, expected) else {
return false
}
}
return true
}
private extension PureYAML.Model.Mapping {
func requiredString(_ key: String) throws -> String {
try #require(self[key]?.stringValue)
}
func requiredInt(_ key: String) throws -> Int {
try #require(self[key]?.intValue)
}
}
private extension PureYAML.Model.Value {
var stringValue: String? {
guard case let .string(value) = self else {
return nil
}
return value
}
var intValue: Int? {
guard case let .int(value) = self else {
return nil
}
return value
}
}