PureYAML/Tests/CanonicalEventConformanceTests.swift
Mihaela Mihaljevic 190b80316b fix(parsing): emit no document for comment-only or empty streams
A stream whose only content is comments or whitespace has zero documents in YAML 1.2 (confirmed by both libyaml and the suite's test.event), but PureYAML was emitting a spurious empty null document into the event stream. Stop emitting it; the single-document value API maps an empty document stream to null, so PureYAML.parse("") is unchanged.

Found two ways: the canonical event gate (8G76, 98YD, AVM7) and, independently, differential testing against libyaml built from the CYaml source over the real-world corpus, where a comment-only values.yaml diverged. Raises exact canonical-event conformance to 267 of 308; the real-world differential agreement rises to 113 of 114. Adds a gated PureYAML yeast-event dumper used to drive the libyaml differential.
2026-06-24 02:27:38 +02:00

188 lines
8.1 KiB
Swift

import Foundation
@testable import PureYAML
import Testing
/// Authoritative conformance gate: serialize PureYAML's parse events into the
/// official YAML test suite "event" (yeast) format and compare EXACTLY to each
/// case's vendored `test.event`. Unlike the JSON-projection oracle, this checks
/// anchors, tags, scalar styles, flow vs block, and explicit document markers,
/// for every valid case (including those with no `in.json`). No normalization.
@Suite("Canonical Event Conformance")
struct CanonicalEventConformanceTests {
/// Valid suite cases whose event stream PureYAML does NOT yet emit exactly.
/// This is the authoritative gap set: a fix removes an id (and the test fails
/// until it is removed), a regression adds one (and the test fails). Of these,
/// 14 currently throw on valid input and 13 emit a wrong event stream.
static let knownEventGaps: Set<String> = [
// false-rejects (parseEvents throws on valid input)
"57H4", "58MP", "5T43", "6BFJ", "8XYN", "FH7J", "LX3P", "M5C3", "M5DY",
"Q9WF", "SKE5", "UKK6/00", "V9D5", "W5VH",
// wrong event stream (parses, but not the canonical events)
"2SXE", "4ABK", "652Z", "6CK3", "7BMT", "7FWL", "7ZZ5",
"9KAX", "CFD4", "DBG4", "DC7X", "DK95/05", "HM87/00", "HM87/01",
"HS5T", "HWV9", "JEF9/02", "M2N8/00", "M2N8/01", "M7A3", "NB6Z", "PW8X",
"QT73", "U3XV", "UGM3", "UV7Q", "Y2GN",
]
@Test("Every valid case emits canonical events, except the recorded gaps")
func test_canonicalEventConformance() throws {
var observedGaps: Set<String> = []
for testCase in CanonicalEventCorpus.validCases() {
guard let yaml = testCase.input, let expected = testCase.expectedEvents else { continue }
let events: [PureYAML.Parsing.Event]
do {
events = try PureYAML.Parsing.Parser().parseEvents(yaml)
} catch {
observedGaps.insert(testCase.id)
continue
}
let actual = events.map(CanonicalEventCorpus.serialize).joined(separator: "\n")
if actual != expected.trimmingTrailingNewline {
observedGaps.insert(testCase.id)
}
}
let regressions = observedGaps.subtracting(Self.knownEventGaps).sorted()
let fixed = Self.knownEventGaps.subtracting(observedGaps).sorted()
#expect(regressions.isEmpty, "Canonical-event REGRESSIONS (now wrong, were correct): \(regressions.joined(separator: " "))")
#expect(fixed.isEmpty, "Canonical-event cases now FIXED; remove from knownEventGaps: \(fixed.joined(separator: " "))")
}
@Test("differential: dump PureYAML yeast events for files under a directory")
func test_dumpYeast() throws {
guard let dir = ProcessInfo.processInfo.environment["PUREYAML_YEAST_DIR"] else { return }
let fileManager = FileManager.default
guard let walker = fileManager.enumerator(atPath: dir) else { return }
for case let path as String in walker where path.hasSuffix(".yaml") || path.hasSuffix(".yml") {
let full = dir + "/" + path
guard let yaml = try? String(contentsOfFile: full, encoding: .utf8) else { continue }
print("@@@FILE@@@ \(path)")
do {
let events = try PureYAML.Parsing.Parser().parseEvents(yaml)
print(events.map(CanonicalEventCorpus.serialize).joined(separator: "\n"))
} catch {
print("@@@ERROR@@@ \(error)")
}
print("@@@END@@@")
}
}
@Test("triage: dump first-line diff for each recorded gap")
func test_triage() throws {
guard ProcessInfo.processInfo.environment["PUREYAML_TRIAGE_EVENTS"] == "1" else { return }
for id in Self.knownEventGaps.sorted() {
let testCase = CanonicalEventCorpus.Case(id: id)
guard let yaml = testCase.input, let expectedRaw = testCase.expectedEvents else { continue }
let expected = expectedRaw.trimmingTrailingNewline.split(separator: "\n").map(String.init)
let events: [PureYAML.Parsing.Event]
do {
events = try PureYAML.Parsing.Parser().parseEvents(yaml)
} catch {
print("\(id) THROWS: \(error)")
continue
}
let actual = events.map(CanonicalEventCorpus.serialize)
var line = 0
while line < min(expected.count, actual.count), expected[line] == actual[line] {
line += 1
}
let exp = line < expected.count ? expected[line] : "(end)"
let act = line < actual.count ? actual[line] : "(end)"
print("\(id) @\(line): EXP[\(exp)] ACT[\(act)]")
}
}
}
enum CanonicalEventCorpus {
struct Case { let id: String }
static func validCases() -> [Case] {
// Every vendored case that is NOT an error case (no `error` marker).
guard let base = Bundle.module.url(forResource: "yaml-test-suite", withExtension: nil, subdirectory: "Fixtures") else { return [] }
let fileManager = FileManager.default
guard let walker = fileManager.enumerator(at: base, includingPropertiesForKeys: nil) else { return [] }
var out: [Case] = []
for case let url as URL in walker where url.lastPathComponent == "in.yaml" {
let dir = url.deletingLastPathComponent()
let id = dir.path.replacingOccurrences(of: base.path + "/", with: "")
if !fileManager.fileExists(atPath: dir.appendingPathComponent("error").path) {
out.append(Case(id: id))
}
}
return out.sorted { $0.id < $1.id }
}
static func url(_ id: String, _ name: String, _ ext: String?) -> URL? {
Bundle.module.url(forResource: name, withExtension: ext, subdirectory: "Fixtures/yaml-test-suite/" + id)
}
static func serialize(_ event: PureYAML.Parsing.Event) -> String {
switch event {
case .streamStart: "+STR"
case .streamEnd: "-STR"
case let .documentStart(explicit, _): explicit ? "+DOC ---" : "+DOC"
case let .documentEnd(explicit, _): explicit ? "-DOC ..." : "-DOC"
case let .sequenceStart(anchor, tag, style, _): "+SEQ" + (style == .flow ? " []" : "") + props(anchor, tag)
case .sequenceEnd: "-SEQ"
case let .mappingStart(anchor, tag, style, _): "+MAP" + (style == .flow ? " {}" : "") + props(anchor, tag)
case .mappingEnd: "-MAP"
case let .scalar(value, anchor, tag, style, _): "=VAL" + props(anchor, tag) + " " + styleChar(style) + escape(value)
case let .alias(anchor, _): "=ALI *" + anchor
}
}
static func props(_ anchor: String?, _ tag: String?) -> String {
var result = ""
if let anchor { result += " &" + anchor }
if let tag { result += " <" + tag + ">" }
return result
}
static func styleChar(_ style: PureYAML.Parsing.ScalarStyle) -> String {
switch style {
case .plain: ":"
case .singleQuoted: "'"
case .doubleQuoted: "\""
case .literal: "|"
case .folded: ">"
}
}
static func escape(_ value: String) -> String {
var out = ""
for scalar in value.unicodeScalars {
switch scalar {
case "\\": out += "\\\\"
case "\t": out += "\\t"
case "\n": out += "\\n"
case "\r": out += "\\r"
case "\u{08}": out += "\\b"
case "\u{00}": out += "\\0"
default: out.unicodeScalars.append(scalar)
}
}
return out
}
}
extension CanonicalEventCorpus.Case {
var input: String? {
guard let url = CanonicalEventCorpus.url(id, "in", "yaml") else { return nil }
return try? String(contentsOf: url, encoding: .utf8)
}
var expectedEvents: String? {
guard let url = CanonicalEventCorpus.url(id, "test", "event") else { return nil }
return try? String(contentsOf: url, encoding: .utf8)
}
}
private extension String {
var trimmingTrailingNewline: String {
var copy = self
while copy.hasSuffix("\n") {
copy.removeLast()
}
return copy
}
}