A stream whose only content is comments or whitespace has zero documents in YAML 1.2 (confirmed by both libyaml and the suite's test.event), but PureYAML was emitting a spurious empty null document into the event stream. Stop emitting it; the single-document value API maps an empty document stream to null, so PureYAML.parse("") is unchanged.
Found two ways: the canonical event gate (8G76, 98YD, AVM7) and, independently, differential testing against libyaml built from the CYaml source over the real-world corpus, where a comment-only values.yaml diverged. Raises exact canonical-event conformance to 267 of 308; the real-world differential agreement rises to 113 of 114. Adds a gated PureYAML yeast-event dumper used to drive the libyaml differential.
188 lines
8.1 KiB
Swift
188 lines
8.1 KiB
Swift
import Foundation
|
|
@testable import PureYAML
|
|
import Testing
|
|
|
|
/// Authoritative conformance gate: serialize PureYAML's parse events into the
|
|
/// official YAML test suite "event" (yeast) format and compare EXACTLY to each
|
|
/// case's vendored `test.event`. Unlike the JSON-projection oracle, this checks
|
|
/// anchors, tags, scalar styles, flow vs block, and explicit document markers,
|
|
/// for every valid case (including those with no `in.json`). No normalization.
|
|
@Suite("Canonical Event Conformance")
|
|
struct CanonicalEventConformanceTests {
|
|
/// Valid suite cases whose event stream PureYAML does NOT yet emit exactly.
|
|
/// This is the authoritative gap set: a fix removes an id (and the test fails
|
|
/// until it is removed), a regression adds one (and the test fails). Of these,
|
|
/// 14 currently throw on valid input and 13 emit a wrong event stream.
|
|
static let knownEventGaps: Set<String> = [
|
|
// false-rejects (parseEvents throws on valid input)
|
|
"57H4", "58MP", "5T43", "6BFJ", "8XYN", "FH7J", "LX3P", "M5C3", "M5DY",
|
|
"Q9WF", "SKE5", "UKK6/00", "V9D5", "W5VH",
|
|
// wrong event stream (parses, but not the canonical events)
|
|
"2SXE", "4ABK", "652Z", "6CK3", "7BMT", "7FWL", "7ZZ5",
|
|
"9KAX", "CFD4", "DBG4", "DC7X", "DK95/05", "HM87/00", "HM87/01",
|
|
"HS5T", "HWV9", "JEF9/02", "M2N8/00", "M2N8/01", "M7A3", "NB6Z", "PW8X",
|
|
"QT73", "U3XV", "UGM3", "UV7Q", "Y2GN",
|
|
]
|
|
|
|
@Test("Every valid case emits canonical events, except the recorded gaps")
|
|
func test_canonicalEventConformance() throws {
|
|
var observedGaps: Set<String> = []
|
|
for testCase in CanonicalEventCorpus.validCases() {
|
|
guard let yaml = testCase.input, let expected = testCase.expectedEvents else { continue }
|
|
let events: [PureYAML.Parsing.Event]
|
|
do {
|
|
events = try PureYAML.Parsing.Parser().parseEvents(yaml)
|
|
} catch {
|
|
observedGaps.insert(testCase.id)
|
|
continue
|
|
}
|
|
let actual = events.map(CanonicalEventCorpus.serialize).joined(separator: "\n")
|
|
if actual != expected.trimmingTrailingNewline {
|
|
observedGaps.insert(testCase.id)
|
|
}
|
|
}
|
|
let regressions = observedGaps.subtracting(Self.knownEventGaps).sorted()
|
|
let fixed = Self.knownEventGaps.subtracting(observedGaps).sorted()
|
|
#expect(regressions.isEmpty, "Canonical-event REGRESSIONS (now wrong, were correct): \(regressions.joined(separator: " "))")
|
|
#expect(fixed.isEmpty, "Canonical-event cases now FIXED; remove from knownEventGaps: \(fixed.joined(separator: " "))")
|
|
}
|
|
|
|
@Test("differential: dump PureYAML yeast events for files under a directory")
|
|
func test_dumpYeast() throws {
|
|
guard let dir = ProcessInfo.processInfo.environment["PUREYAML_YEAST_DIR"] else { return }
|
|
let fileManager = FileManager.default
|
|
guard let walker = fileManager.enumerator(atPath: dir) else { return }
|
|
for case let path as String in walker where path.hasSuffix(".yaml") || path.hasSuffix(".yml") {
|
|
let full = dir + "/" + path
|
|
guard let yaml = try? String(contentsOfFile: full, encoding: .utf8) else { continue }
|
|
print("@@@FILE@@@ \(path)")
|
|
do {
|
|
let events = try PureYAML.Parsing.Parser().parseEvents(yaml)
|
|
print(events.map(CanonicalEventCorpus.serialize).joined(separator: "\n"))
|
|
} catch {
|
|
print("@@@ERROR@@@ \(error)")
|
|
}
|
|
print("@@@END@@@")
|
|
}
|
|
}
|
|
|
|
@Test("triage: dump first-line diff for each recorded gap")
|
|
func test_triage() throws {
|
|
guard ProcessInfo.processInfo.environment["PUREYAML_TRIAGE_EVENTS"] == "1" else { return }
|
|
for id in Self.knownEventGaps.sorted() {
|
|
let testCase = CanonicalEventCorpus.Case(id: id)
|
|
guard let yaml = testCase.input, let expectedRaw = testCase.expectedEvents else { continue }
|
|
let expected = expectedRaw.trimmingTrailingNewline.split(separator: "\n").map(String.init)
|
|
let events: [PureYAML.Parsing.Event]
|
|
do {
|
|
events = try PureYAML.Parsing.Parser().parseEvents(yaml)
|
|
} catch {
|
|
print("\(id) THROWS: \(error)")
|
|
continue
|
|
}
|
|
let actual = events.map(CanonicalEventCorpus.serialize)
|
|
var line = 0
|
|
while line < min(expected.count, actual.count), expected[line] == actual[line] {
|
|
line += 1
|
|
}
|
|
let exp = line < expected.count ? expected[line] : "(end)"
|
|
let act = line < actual.count ? actual[line] : "(end)"
|
|
print("\(id) @\(line): EXP[\(exp)] ACT[\(act)]")
|
|
}
|
|
}
|
|
}
|
|
|
|
enum CanonicalEventCorpus {
|
|
struct Case { let id: String }
|
|
|
|
static func validCases() -> [Case] {
|
|
// Every vendored case that is NOT an error case (no `error` marker).
|
|
guard let base = Bundle.module.url(forResource: "yaml-test-suite", withExtension: nil, subdirectory: "Fixtures") else { return [] }
|
|
let fileManager = FileManager.default
|
|
guard let walker = fileManager.enumerator(at: base, includingPropertiesForKeys: nil) else { return [] }
|
|
var out: [Case] = []
|
|
for case let url as URL in walker where url.lastPathComponent == "in.yaml" {
|
|
let dir = url.deletingLastPathComponent()
|
|
let id = dir.path.replacingOccurrences(of: base.path + "/", with: "")
|
|
if !fileManager.fileExists(atPath: dir.appendingPathComponent("error").path) {
|
|
out.append(Case(id: id))
|
|
}
|
|
}
|
|
return out.sorted { $0.id < $1.id }
|
|
}
|
|
|
|
static func url(_ id: String, _ name: String, _ ext: String?) -> URL? {
|
|
Bundle.module.url(forResource: name, withExtension: ext, subdirectory: "Fixtures/yaml-test-suite/" + id)
|
|
}
|
|
|
|
static func serialize(_ event: PureYAML.Parsing.Event) -> String {
|
|
switch event {
|
|
case .streamStart: "+STR"
|
|
case .streamEnd: "-STR"
|
|
case let .documentStart(explicit, _): explicit ? "+DOC ---" : "+DOC"
|
|
case let .documentEnd(explicit, _): explicit ? "-DOC ..." : "-DOC"
|
|
case let .sequenceStart(anchor, tag, style, _): "+SEQ" + (style == .flow ? " []" : "") + props(anchor, tag)
|
|
case .sequenceEnd: "-SEQ"
|
|
case let .mappingStart(anchor, tag, style, _): "+MAP" + (style == .flow ? " {}" : "") + props(anchor, tag)
|
|
case .mappingEnd: "-MAP"
|
|
case let .scalar(value, anchor, tag, style, _): "=VAL" + props(anchor, tag) + " " + styleChar(style) + escape(value)
|
|
case let .alias(anchor, _): "=ALI *" + anchor
|
|
}
|
|
}
|
|
|
|
static func props(_ anchor: String?, _ tag: String?) -> String {
|
|
var result = ""
|
|
if let anchor { result += " &" + anchor }
|
|
if let tag { result += " <" + tag + ">" }
|
|
return result
|
|
}
|
|
|
|
static func styleChar(_ style: PureYAML.Parsing.ScalarStyle) -> String {
|
|
switch style {
|
|
case .plain: ":"
|
|
case .singleQuoted: "'"
|
|
case .doubleQuoted: "\""
|
|
case .literal: "|"
|
|
case .folded: ">"
|
|
}
|
|
}
|
|
|
|
static func escape(_ value: String) -> String {
|
|
var out = ""
|
|
for scalar in value.unicodeScalars {
|
|
switch scalar {
|
|
case "\\": out += "\\\\"
|
|
case "\t": out += "\\t"
|
|
case "\n": out += "\\n"
|
|
case "\r": out += "\\r"
|
|
case "\u{08}": out += "\\b"
|
|
case "\u{00}": out += "\\0"
|
|
default: out.unicodeScalars.append(scalar)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
}
|
|
|
|
extension CanonicalEventCorpus.Case {
|
|
var input: String? {
|
|
guard let url = CanonicalEventCorpus.url(id, "in", "yaml") else { return nil }
|
|
return try? String(contentsOf: url, encoding: .utf8)
|
|
}
|
|
|
|
var expectedEvents: String? {
|
|
guard let url = CanonicalEventCorpus.url(id, "test", "event") else { return nil }
|
|
return try? String(contentsOf: url, encoding: .utf8)
|
|
}
|
|
}
|
|
|
|
private extension String {
|
|
var trimmingTrailingNewline: String {
|
|
var copy = self
|
|
while copy.hasSuffix("\n") {
|
|
copy.removeLast()
|
|
}
|
|
return copy
|
|
}
|
|
}
|