Serialize parse events to the official yaml-test-suite yeast format and compare exactly to each case's test.event: the strongest oracle for the parser, checking anchors, tags, scalar styles, flow vs block, and aliases for every valid case, including those without an in.json that the JSON-projection oracle cannot judge. It runs opt-in (PUREYAML_RUN_YAML_TEST_SUITE=1) against the fetched suite and is a gate, not a measurement: knownEventGaps records the 31 valid cases whose event stream is not yet exact, so closing one must remove its id and a regression adds one. No case is silently skipped. One documented normalization: the event API does not carry the explicit document-marker flag, so +DOC/-DOC are compared without their ---/... suffix. The first run exposed real deviations the parse+value metric missed: 4ABK truncates http://foo.com at the colon, FH7J and PW8X bind a property to the collection instead of the empty scalar, 6BFJ binds the wrong anchor.
171 lines
7.6 KiB
Swift
171 lines
7.6 KiB
Swift
import Foundation
|
|
@testable import PureYAML
|
|
import Testing
|
|
|
|
/// Canonical event-stream conformance gate: the strongest oracle for the parser. It serializes
|
|
/// PureYAML's parse events into the official YAML test-suite "event" (yeast) format and compares them
|
|
/// exactly to each case's `test.event`. Unlike the JSON-projection oracle, this checks anchors, tags,
|
|
/// scalar styles, flow vs block, and aliases, for every valid case including those without an `in.json`.
|
|
///
|
|
/// Opt-in like the other suite gates, since the corpus is fetched, not vendored:
|
|
/// bash scripts/fetch-yaml-test-suite.sh
|
|
/// PUREYAML_RUN_YAML_TEST_SUITE=1 swift test --filter CanonicalEventConformanceTests
|
|
///
|
|
/// `knownEventGaps` is the authoritative gap set: every valid case whose event stream PureYAML does not
|
|
/// yet emit exactly. It is a gate, not a measurement: a fix must remove an id (the test fails until it
|
|
/// is removed), and a regression adds one (the test fails). No case is ever silently skipped.
|
|
///
|
|
/// One documented normalization. `Parsing.Event` does not carry the explicit document-marker flag, so
|
|
/// `+DOC`/`-DOC` lines are compared without their ` ---`/` ...` suffix. The marker distinction is the
|
|
/// single property this gate does not yet check; enriching the event API to carry the flag removes the
|
|
/// normalization and this caveat with it.
|
|
@Suite("Canonical Event Conformance")
|
|
struct CanonicalEventConformanceTests {
|
|
private struct Case {
|
|
let id: String
|
|
let input: String
|
|
let expectedEvents: String
|
|
}
|
|
|
|
/// The authoritative event-level frontier: valid cases that parse+value-match but whose canonical
|
|
/// event stream PureYAML does not yet emit exactly. Each is a precise, recorded gap (first-line
|
|
/// event diff available via the triage below), not a silent skip. Closing one removes its id; a
|
|
/// regression adds one. Audited 2026-06-29: every entry is a real parser deviation (e.g. 4ABK
|
|
/// truncates `http://foo.com` at the `:`; FH7J/PW8X bind a property to the collection instead of
|
|
/// the empty scalar), not a serializer artifact.
|
|
static let knownEventGaps: Set<String> = [
|
|
"4ABK", "4FJ6", "6BFJ", "6CK3", "7BMT", "7FWL", "87E4", "8UDB", "9KAX", "9MMW",
|
|
"CN3R", "CT4Q", "DC7X", "FH7J", "HWV9", "L9U5", "LP6E", "LQZ7", "M5C3", "M5DY",
|
|
"PW8X", "QF4Y", "QT73", "RZP5", "U3XV", "UGM3", "UT92", "V9D5", "XW4D", "ZF4X", "ZK9H",
|
|
]
|
|
|
|
@Test("Every valid case emits canonical events, except the recorded gaps")
|
|
func test_canonicalEventConformance() throws {
|
|
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
|
|
return
|
|
}
|
|
let cases = try #require(loadValidCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh")
|
|
var observedGaps: Set<String> = []
|
|
var compared = 0
|
|
for testCase in cases {
|
|
let events: [PureYAML.Parsing.Event]
|
|
do {
|
|
events = try PureYAML.Parsing.Parser().parseEvents(testCase.input)
|
|
} catch {
|
|
observedGaps.insert(testCase.id)
|
|
continue
|
|
}
|
|
compared += 1
|
|
let actual = Self.normalizeMarkers(events.map(Self.serialize).joined(separator: "\n"))
|
|
if actual != Self.normalizeMarkers(Self.trimmingTrailingNewlines(testCase.expectedEvents)) {
|
|
observedGaps.insert(testCase.id)
|
|
}
|
|
}
|
|
let regressions = observedGaps.subtracting(Self.knownEventGaps).sorted()
|
|
let fixed = Self.knownEventGaps.subtracting(observedGaps).sorted()
|
|
report("""
|
|
Canonical event conformance: \(cases.count - observedGaps.count)/\(cases.count) valid cases emit exact canonical events
|
|
recorded gaps: \(observedGaps.count) (\(compared) parsed, \(cases.count - compared) threw)
|
|
""")
|
|
#expect(regressions.isEmpty, "Canonical-event REGRESSIONS (now wrong, were correct): \(regressions.joined(separator: " "))")
|
|
#expect(fixed.isEmpty, "Canonical-event cases now FIXED; remove from knownEventGaps: \(fixed.joined(separator: " "))")
|
|
#expect(compared > 100, "non-vacuity: only \(compared) cases compared; the corpus did not load")
|
|
}
|
|
|
|
private func loadValidCases() -> [Case]? {
|
|
let manager = FileManager.default
|
|
let root = ProcessInfo.processInfo.environment["PUREYAML_YAML_TEST_SUITE_DIR"]
|
|
?? manager.currentDirectoryPath + "/.build/yaml-test-suite"
|
|
guard let names = try? manager.contentsOfDirectory(atPath: root) else {
|
|
return nil
|
|
}
|
|
var cases: [Case] = []
|
|
for name in names.sorted() {
|
|
let dir = root + "/" + name
|
|
guard !manager.fileExists(atPath: dir + "/error"),
|
|
let input = try? String(contentsOfFile: dir + "/in.yaml", encoding: .utf8),
|
|
let events = try? String(contentsOfFile: dir + "/test.event", encoding: .utf8)
|
|
else {
|
|
continue
|
|
}
|
|
cases.append(Case(id: name, input: input, expectedEvents: events))
|
|
}
|
|
return cases
|
|
}
|
|
|
|
// MARK: - Yeast serialization
|
|
|
|
static func serialize(_ event: PureYAML.Parsing.Event) -> String {
|
|
switch event {
|
|
case .streamStart: "+STR"
|
|
case .streamEnd: "-STR"
|
|
case .documentStart: "+DOC"
|
|
case .documentEnd: "-DOC"
|
|
case let .sequenceStart(anchor, tag, style, _): "+SEQ" + (style == .flow ? " []" : "") + props(anchor, tag)
|
|
case .sequenceEnd: "-SEQ"
|
|
case let .mappingStart(anchor, tag, style, _): "+MAP" + (style == .flow ? " {}" : "") + props(anchor, tag)
|
|
case .mappingEnd: "-MAP"
|
|
case let .scalar(value, anchor, tag, style, _, _): "=VAL" + props(anchor, tag) + " " + styleChar(style) + escape(value)
|
|
case let .alias(anchor, _): "=ALI *" + anchor
|
|
}
|
|
}
|
|
|
|
static func props(_ anchor: String?, _ tag: String?) -> String {
|
|
var result = ""
|
|
if let anchor { result += " &" + anchor }
|
|
if let tag { result += " <" + tag + ">" }
|
|
return result
|
|
}
|
|
|
|
static func styleChar(_ style: PureYAML.Parsing.ScalarStyle) -> String {
|
|
switch style {
|
|
case .plain: ":"
|
|
case .singleQuoted: "'"
|
|
case .doubleQuoted: "\""
|
|
case .literal: "|"
|
|
case .folded: ">"
|
|
}
|
|
}
|
|
|
|
static func escape(_ value: String) -> String {
|
|
var out = ""
|
|
for scalar in value.unicodeScalars {
|
|
switch scalar {
|
|
case "\\": out += "\\\\"
|
|
case "\t": out += "\\t"
|
|
case "\n": out += "\\n"
|
|
case "\r": out += "\\r"
|
|
case "\u{08}": out += "\\b"
|
|
case "\u{00}": out += "\\0"
|
|
default: out.unicodeScalars.append(scalar)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
/// Drops the explicit `---`/`...` suffix from document-boundary lines, since the event API does not
|
|
/// yet record whether a document marker was explicit. Applied to both sides for symmetry.
|
|
static func normalizeMarkers(_ events: String) -> String {
|
|
events
|
|
.split(separator: "\n", omittingEmptySubsequences: false)
|
|
.map { line in
|
|
if line == "+DOC ---" { return "+DOC" }
|
|
if line == "-DOC ..." { return "-DOC" }
|
|
return String(line)
|
|
}
|
|
.joined(separator: "\n")
|
|
}
|
|
|
|
static func trimmingTrailingNewlines(_ value: String) -> String {
|
|
var copy = value
|
|
while copy.hasSuffix("\n") {
|
|
copy.removeLast()
|
|
}
|
|
return copy
|
|
}
|
|
|
|
private func report(_ text: String) {
|
|
FileHandle.standardError.write(Data("\n\(text)\n".utf8))
|
|
}
|
|
}
|