Three root causes: (1) a '%' begins a directive only in directive position (before document content, outside flow); everywhere else it is plain content, fixing misfires inside plain scalars, flow mappings, and block scalars. (2) %YAML accepts any 1.x (a higher minor is treated as 1.2 without error), rejects major != 1 and duplicate %YAML, and requires a following '---'; reserved and %TAG directives are accepted and ignored. A directive not followed by a document start is rejected with the new directiveWithoutDocument error. (3) An explicit '...' end marker closes the current document so following content begins a fresh document in parseStream; single-document parse still rejects multi-document input at compose time. Tests/UnsupportedYAMLGapsFixtures.swift, StreamParsingTests, ScannerTests, and ParsingErrorTests were updated in lockstep to assert the now-correct behavior (percent-after-content is plain text; '...' then content is two documents; %YAML 1.3 is valid while 2.0 and duplicate %YAML are rejected); no test was weakened and multi-document rejection for single-document parse is still covered. Flips 15 official YAML test suite cases (295/402); SF5V, 9MMA, B63P, H7TQ now correctly rejected. 6LVF and W4TN are reclassified to json-mismatch (blocked by other scalar gaps). Updates the manifest, fixtures, counts, and roadmap.
363 lines
12 KiB
Swift
363 lines
12 KiB
Swift
import Foundation
|
|
@testable import PureYAML
|
|
import Testing
|
|
|
|
/// Conformance gaps measured against the official YAML test suite
|
|
/// (github.com/yaml/yaml-test-suite, `data` branch). The vendored cases under
|
|
/// `Fixtures/yaml-test-suite/` are exactly the suite entries PureYAML does not
|
|
/// yet handle correctly. The manifest records each gap's current behavior so a
|
|
/// future fix is forced to update it (the gated test below reports drift).
|
|
///
|
|
/// The behavioral sweep is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1`, mirroring
|
|
/// the full real-corpus sweep, so the default suite stays green while the gaps are
|
|
/// tracked rather than silenced.
|
|
@Suite("YAML Test Suite Gaps")
|
|
struct YAMLTestSuiteGapsTests {
|
|
@Test("Gap manifest matches vendored fixture files")
|
|
func test_gapManifestMatchesVendoredFixtureFiles() throws {
|
|
let manifest = try YAMLTestSuiteManifest.load()
|
|
|
|
#expect(manifest.version == 1)
|
|
#expect(manifest.cases.count == 107)
|
|
#expect(manifest.cases.count(where: { $0.category == "false-reject" }) == 40)
|
|
#expect(manifest.cases.count(where: { $0.category == "json-mismatch" }) == 20)
|
|
#expect(manifest.cases.count(where: { $0.category == "accepted-invalid" }) == 47)
|
|
#expect(Set(manifest.cases.map(\.id)).count == manifest.cases.count)
|
|
|
|
for testCase in manifest.cases {
|
|
let yaml = try testCase.loadInput()
|
|
#expect(yaml.utf8.count == testCase.byteCount, "\(testCase.id): byte count drift")
|
|
#expect(yaml.count(where: { $0 == "\n" }) == testCase.lineCount, "\(testCase.id): line count drift")
|
|
|
|
switch testCase.category {
|
|
case "accepted-invalid":
|
|
#expect(testCase.expected == "reject")
|
|
#expect(testCase.loadErrorMarker() != nil, "\(testCase.id): missing error marker")
|
|
default:
|
|
#expect(testCase.expected == "parse-ok")
|
|
}
|
|
|
|
if testCase.oracle == "json" {
|
|
#expect(testCase.loadExpectedJSON() != nil, "\(testCase.id): missing in.json oracle")
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test("Recorded conformance gaps still reproduce")
|
|
func test_recordedConformanceGapsStillReproduce() throws {
|
|
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
|
|
return
|
|
}
|
|
|
|
let manifest = try YAMLTestSuiteManifest.load()
|
|
for testCase in manifest.cases {
|
|
let observed = try testCase.observedStatus()
|
|
#expect(
|
|
observed == testCase.current,
|
|
"\(testCase.id) \"\(testCase.name)\": recorded gap '\(testCase.current)' but observed '\(observed)'. If this case is now handled correctly, update the manifest.",
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
// MARK: - Manifest
|
|
|
|
struct YAMLTestSuiteManifest {
|
|
var version: Int
|
|
var cases: [YAMLTestSuiteCase]
|
|
|
|
static func load() throws -> Self {
|
|
let url = try #require(Bundle.module.url(
|
|
forResource: "yaml-test-suite-gaps",
|
|
withExtension: "yaml",
|
|
subdirectory: "Fixtures",
|
|
))
|
|
let value = try PureYAML.parse(String(contentsOf: url, encoding: .utf8))
|
|
let root = try #require(value.mapping)
|
|
let version = try #require(root["version"]?.intValue)
|
|
let caseValues = try #require(root.sequence("cases"))
|
|
return try YAMLTestSuiteManifest(
|
|
version: version,
|
|
cases: caseValues.map(YAMLTestSuiteCase.init(value:)),
|
|
)
|
|
}
|
|
}
|
|
|
|
struct YAMLTestSuiteCase {
|
|
var id: String
|
|
var name: String
|
|
var localPath: String
|
|
var category: String
|
|
var expected: String
|
|
var oracle: String
|
|
var current: String
|
|
var byteCount: Int
|
|
var lineCount: Int
|
|
|
|
init(value: PureYAML.Model.Value) throws {
|
|
let mapping = try #require(value.mapping)
|
|
id = try mapping.requiredString("id")
|
|
name = try mapping.requiredString("name")
|
|
localPath = try mapping.requiredString("localPath")
|
|
category = try mapping.requiredString("category")
|
|
expected = try mapping.requiredString("expected")
|
|
oracle = try mapping.requiredString("oracle")
|
|
current = try mapping.requiredString("current")
|
|
byteCount = try mapping.requiredInt("byteCount")
|
|
lineCount = try mapping.requiredInt("lineCount")
|
|
}
|
|
|
|
private var caseSubdirectory: String {
|
|
let directory = localPath.split(separator: "/").dropLast().joined(separator: "/")
|
|
return "Fixtures/\(directory)"
|
|
}
|
|
|
|
func loadInput() throws -> String {
|
|
let url = try #require(Bundle.module.url(forResource: "in", withExtension: "yaml", subdirectory: caseSubdirectory))
|
|
return try String(contentsOf: url, encoding: .utf8)
|
|
}
|
|
|
|
func loadErrorMarker() -> URL? {
|
|
Bundle.module.url(forResource: "error", withExtension: nil, subdirectory: caseSubdirectory)
|
|
}
|
|
|
|
func loadExpectedJSON() -> String? {
|
|
guard let url = Bundle.module.url(forResource: "in", withExtension: "json", subdirectory: caseSubdirectory) else {
|
|
return nil
|
|
}
|
|
return try? String(contentsOf: url, encoding: .utf8)
|
|
}
|
|
|
|
/// Recomputes PureYAML's present behavior for this case, in the same vocabulary
|
|
/// the manifest's `current` field uses, so drift (including fixes) is detectable.
|
|
func observedStatus() throws -> String {
|
|
let yaml = try loadInput()
|
|
var documents: [PureYAML.Stream.Document] = []
|
|
var parseThrew = false
|
|
do {
|
|
documents = try PureYAML.parseStream(yaml)
|
|
} catch {
|
|
parseThrew = true
|
|
}
|
|
var validateThrew = false
|
|
if !parseThrew {
|
|
do {
|
|
_ = try PureYAML.validate(documents)
|
|
} catch {
|
|
validateThrew = true
|
|
}
|
|
}
|
|
|
|
if category == "accepted-invalid" {
|
|
return (parseThrew || validateThrew) ? "reject" : "accepted"
|
|
}
|
|
|
|
if parseThrew {
|
|
return "parse-error"
|
|
}
|
|
|
|
guard oracle == "json", let jsonText = loadExpectedJSON(), let expectedDocuments = decodeJSONStream(jsonText) else {
|
|
return "parse-ok"
|
|
}
|
|
if expectedDocuments.count != documents.count {
|
|
return "value-mismatch"
|
|
}
|
|
let matches = zip(documents, expectedDocuments).allSatisfy { equalValue($0.value, $1) }
|
|
return matches ? "value-match" : "value-mismatch"
|
|
}
|
|
}
|
|
|
|
// MARK: - JSON oracle (type-preserving, avoids NSNumber bool ambiguity)
|
|
|
|
indirect enum YAMLTestSuiteJSON: Decodable, Equatable {
|
|
case null
|
|
case bool(Bool)
|
|
case int(Int)
|
|
case double(Double)
|
|
case string(String)
|
|
case array([YAMLTestSuiteJSON])
|
|
case object([String: YAMLTestSuiteJSON])
|
|
|
|
init(from decoder: Decoder) throws {
|
|
let container = try decoder.singleValueContainer()
|
|
if container.decodeNil() {
|
|
self = .null
|
|
} else if let value = try? container.decode(Bool.self) {
|
|
self = .bool(value)
|
|
} else if let value = try? container.decode(Int.self) {
|
|
self = .int(value)
|
|
} else if let value = try? container.decode(Double.self) {
|
|
self = .double(value)
|
|
} else if let value = try? container.decode(String.self) {
|
|
self = .string(value)
|
|
} else if let value = try? container.decode([YAMLTestSuiteJSON].self) {
|
|
self = .array(value)
|
|
} else if let value = try? container.decode([String: YAMLTestSuiteJSON].self) {
|
|
self = .object(value)
|
|
} else {
|
|
throw DecodingError.dataCorrupted(.init(codingPath: container.codingPath, debugDescription: "unrecognized JSON"))
|
|
}
|
|
}
|
|
}
|
|
|
|
func decodeJSONStream(_ text: String) -> [YAMLTestSuiteJSON]? {
|
|
let decoder = JSONDecoder()
|
|
if let whole = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(text.utf8)) {
|
|
return [whole]
|
|
}
|
|
let parts = splitJSONStream(text)
|
|
guard !parts.isEmpty else {
|
|
return nil
|
|
}
|
|
var values: [YAMLTestSuiteJSON] = []
|
|
for part in parts {
|
|
guard let value = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(part.utf8)) else {
|
|
return nil
|
|
}
|
|
values.append(value)
|
|
}
|
|
return values
|
|
}
|
|
|
|
/// Splits a concatenated JSON stream (one value per YAML document) into top-level values.
|
|
func splitJSONStream(_ text: String) -> [String] {
|
|
var scanner = JSONStreamScanner(text: text)
|
|
return scanner.scan()
|
|
}
|
|
|
|
private struct JSONStreamScanner {
|
|
let text: String
|
|
var out: [String] = []
|
|
var depth = 0
|
|
var inString = false
|
|
var escaped = false
|
|
var start: String.Index?
|
|
|
|
mutating func scan() -> [String] {
|
|
var index = text.startIndex
|
|
while index < text.endIndex {
|
|
if inString {
|
|
stepInString(text[index])
|
|
} else {
|
|
stepStructural(text[index], at: index)
|
|
}
|
|
index = text.index(after: index)
|
|
}
|
|
if let begin = start {
|
|
out.append(String(text[begin...]))
|
|
}
|
|
return out.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty }
|
|
}
|
|
|
|
private mutating func stepInString(_ character: Character) {
|
|
if escaped {
|
|
escaped = false
|
|
} else if character == "\\" {
|
|
escaped = true
|
|
} else if character == "\"" {
|
|
inString = false
|
|
}
|
|
}
|
|
|
|
private mutating func stepStructural(_ character: Character, at index: String.Index) {
|
|
switch character {
|
|
case "\"":
|
|
if start == nil { start = index }
|
|
inString = true
|
|
case "{", "[":
|
|
if start == nil { start = index }
|
|
depth += 1
|
|
case "}", "]":
|
|
depth -= 1
|
|
if depth == 0, let begin = start {
|
|
out.append(String(text[begin ... index]))
|
|
start = nil
|
|
}
|
|
case " ", "\n", "\r", "\t":
|
|
if depth == 0, let begin = start {
|
|
out.append(String(text[begin ..< index]))
|
|
start = nil
|
|
}
|
|
default:
|
|
if depth == 0, start == nil { start = index }
|
|
}
|
|
}
|
|
}
|
|
|
|
func equalValue(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool {
|
|
switch (value, json) {
|
|
case let (.sequence(lhs), .array(rhs)):
|
|
lhs.count == rhs.count && zip(lhs, rhs).allSatisfy { equalValue($0, $1) }
|
|
case let (.mapping(lhs), .object(rhs)):
|
|
equalMapping(lhs, rhs)
|
|
default:
|
|
equalScalar(value, json)
|
|
}
|
|
}
|
|
|
|
private func equalScalar(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool {
|
|
switch (value, json) {
|
|
case (.null, .null):
|
|
true
|
|
case let (.bool(lhs), .bool(rhs)):
|
|
lhs == rhs
|
|
case let (.int(lhs), .int(rhs)):
|
|
lhs == rhs
|
|
case let (.int(lhs), .double(rhs)):
|
|
Double(lhs) == rhs
|
|
case let (.double(lhs), .int(rhs)):
|
|
lhs == Double(rhs)
|
|
case let (.double(lhs), .double(rhs)):
|
|
lhs == rhs || (lhs.isNaN && rhs.isNaN)
|
|
case let (.string(lhs), .string(rhs)):
|
|
lhs == rhs
|
|
default:
|
|
false
|
|
}
|
|
}
|
|
|
|
private func equalMapping(_ mapping: PureYAML.Model.Mapping, _ object: [String: YAMLTestSuiteJSON]) -> Bool {
|
|
guard mapping.pairs.count == object.count else {
|
|
return false
|
|
}
|
|
var dictionary: [String: PureYAML.Model.Value] = [:]
|
|
for pair in mapping.pairs {
|
|
dictionary[pair.key] = pair.value
|
|
}
|
|
guard dictionary.count == object.count else {
|
|
return false
|
|
}
|
|
for (key, expected) in object {
|
|
guard let actual = dictionary[key], equalValue(actual, expected) else {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
private extension PureYAML.Model.Mapping {
|
|
func requiredString(_ key: String) throws -> String {
|
|
try #require(self[key]?.stringValue)
|
|
}
|
|
|
|
func requiredInt(_ key: String) throws -> Int {
|
|
try #require(self[key]?.intValue)
|
|
}
|
|
}
|
|
|
|
private extension PureYAML.Model.Value {
|
|
var stringValue: String? {
|
|
guard case let .string(value) = self else {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
|
|
var intValue: Int? {
|
|
guard case let .int(value) = self else {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
}
|