NodetteHQ/PureYAML
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

test(conformance): add the canonical event-stream conformance gate

Browse Source 
Serialize parse events to the official yaml-test-suite yeast format and compare
exactly to each case's test.event: the strongest oracle for the parser, checking
anchors, tags, scalar styles, flow vs block, and aliases for every valid case,
including those without an in.json that the JSON-projection oracle cannot judge.

It runs opt-in (PUREYAML_RUN_YAML_TEST_SUITE=1) against the fetched suite and is
a gate, not a measurement: knownEventGaps records the 31 valid cases whose event
stream is not yet exact, so closing one must remove its id and a regression adds
one. No case is silently skipped. One documented normalization: the event API
does not carry the explicit document-marker flag, so +DOC/-DOC are compared
without their ---/... suffix.

The first run exposed real deviations the parse+value metric missed: 4ABK
truncates http://foo.com at the colon, FH7J and PW8X bind a property to the
collection instead of the empty scalar, 6BFJ binds the wrong anchor.
This commit is contained in:
Mihaela M 2026-06-29 10:13:45 +02:00
parent af4a00f229
commit 93621b8c84
1 changed files with 170 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
Tests/CanonicalEventConformanceTests.swift Normal file
View File

@ -0,0 +1,170 @@
import Foundation
@testable import PureYAML
import Testing
/// Canonical event-stream conformance gate: the strongest oracle for the parser. It serializes
/// PureYAML's parse events into the official YAML test-suite "event" (yeast) format and compares them
/// exactly to each case's `test.event`. Unlike the JSON-projection oracle, this checks anchors, tags,
/// scalar styles, flow vs block, and aliases, for every valid case including those without an `in.json`.
///
/// Opt-in like the other suite gates, since the corpus is fetched, not vendored:
/// bash scripts/fetch-yaml-test-suite.sh
/// PUREYAML_RUN_YAML_TEST_SUITE=1 swift test --filter CanonicalEventConformanceTests
///
/// `knownEventGaps` is the authoritative gap set: every valid case whose event stream PureYAML does not
/// yet emit exactly. It is a gate, not a measurement: a fix must remove an id (the test fails until it
/// is removed), and a regression adds one (the test fails). No case is ever silently skipped.
///
/// One documented normalization. `Parsing.Event` does not carry the explicit document-marker flag, so
/// `+DOC`/`-DOC` lines are compared without their ` ---`/` ...` suffix. The marker distinction is the
/// single property this gate does not yet check; enriching the event API to carry the flag removes the
/// normalization and this caveat with it.
@Suite("Canonical Event Conformance")
struct CanonicalEventConformanceTests {
private struct Case {
let id: String
let input: String
let expectedEvents: String
}
/// The authoritative event-level frontier: valid cases that parse+value-match but whose canonical
/// event stream PureYAML does not yet emit exactly. Each is a precise, recorded gap (first-line
/// event diff available via the triage below), not a silent skip. Closing one removes its id; a
/// regression adds one. Audited 2026-06-29: every entry is a real parser deviation (e.g. 4ABK
/// truncates `http://foo.com` at the `:`; FH7J/PW8X bind a property to the collection instead of
/// the empty scalar), not a serializer artifact.
static let knownEventGaps: Set<String> = [
"4ABK", "4FJ6", "6BFJ", "6CK3", "7BMT", "7FWL", "87E4", "8UDB", "9KAX", "9MMW",
"CN3R", "CT4Q", "DC7X", "FH7J", "HWV9", "L9U5", "LP6E", "LQZ7", "M5C3", "M5DY",
"PW8X", "QF4Y", "QT73", "RZP5", "U3XV", "UGM3", "UT92", "V9D5", "XW4D", "ZF4X", "ZK9H",
]
@Test("Every valid case emits canonical events, except the recorded gaps")
func test_canonicalEventConformance() throws {
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
return
}
let cases = try #require(loadValidCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh")
var observedGaps: Set<String> = []
var compared = 0
for testCase in cases {
let events: [PureYAML.Parsing.Event]
do {
events = try PureYAML.Parsing.Parser().parseEvents(testCase.input)
} catch {
observedGaps.insert(testCase.id)
continue
}
compared += 1
let actual = Self.normalizeMarkers(events.map(Self.serialize).joined(separator: "\n"))
if actual != Self.normalizeMarkers(Self.trimmingTrailingNewlines(testCase.expectedEvents)) {
observedGaps.insert(testCase.id)
}
}
let regressions = observedGaps.subtracting(Self.knownEventGaps).sorted()
let fixed = Self.knownEventGaps.subtracting(observedGaps).sorted()
report("""
Canonical event conformance: \(cases.count - observedGaps.count)/\(cases.count) valid cases emit exact canonical events
recorded gaps: \(observedGaps.count) (\(compared) parsed, \(cases.count - compared) threw)
""")
#expect(regressions.isEmpty, "Canonical-event REGRESSIONS (now wrong, were correct): \(regressions.joined(separator: " "))")
#expect(fixed.isEmpty, "Canonical-event cases now FIXED; remove from knownEventGaps: \(fixed.joined(separator: " "))")
#expect(compared > 100, "non-vacuity: only \(compared) cases compared; the corpus did not load")
}
private func loadValidCases() -> [Case]? {
let manager = FileManager.default
let root = ProcessInfo.processInfo.environment["PUREYAML_YAML_TEST_SUITE_DIR"]
?? manager.currentDirectoryPath + "/.build/yaml-test-suite"
guard let names = try? manager.contentsOfDirectory(atPath: root) else {
return nil
}
var cases: [Case] = []
for name in names.sorted() {
let dir = root + "/" + name
guard !manager.fileExists(atPath: dir + "/error"),
let input = try? String(contentsOfFile: dir + "/in.yaml", encoding: .utf8),
let events = try? String(contentsOfFile: dir + "/test.event", encoding: .utf8)
else {
continue
}
cases.append(Case(id: name, input: input, expectedEvents: events))
}
return cases
}
// MARK: - Yeast serialization
static func serialize(_ event: PureYAML.Parsing.Event) -> String {
switch event {
case .streamStart: "+STR"
case .streamEnd: "-STR"
case .documentStart: "+DOC"
case .documentEnd: "-DOC"
case let .sequenceStart(anchor, tag, style, _): "+SEQ" + (style == .flow ? " []" : "") + props(anchor, tag)
case .sequenceEnd: "-SEQ"
case let .mappingStart(anchor, tag, style, _): "+MAP" + (style == .flow ? " {}" : "") + props(anchor, tag)
case .mappingEnd: "-MAP"
case let .scalar(value, anchor, tag, style, _, _): "=VAL" + props(anchor, tag) + " " + styleChar(style) + escape(value)
case let .alias(anchor, _): "=ALI *" + anchor
}
}
static func props(_ anchor: String?, _ tag: String?) -> String {
var result = ""
if let anchor { result += " &" + anchor }
if let tag { result += " <" + tag + ">" }
return result
}
static func styleChar(_ style: PureYAML.Parsing.ScalarStyle) -> String {
switch style {
case .plain: ":"
case .singleQuoted: "'"
case .doubleQuoted: "\""
case .literal: "|"
case .folded: ">"
}
}
static func escape(_ value: String) -> String {
var out = ""
for scalar in value.unicodeScalars {
switch scalar {
case "\\": out += "\\\\"
case "\t": out += "\\t"
case "\n": out += "\\n"
case "\r": out += "\\r"
case "\u{08}": out += "\\b"
case "\u{00}": out += "\\0"
default: out.unicodeScalars.append(scalar)
}
}
return out
}
/// Drops the explicit `---`/`...` suffix from document-boundary lines, since the event API does not
/// yet record whether a document marker was explicit. Applied to both sides for symmetry.
static func normalizeMarkers(_ events: String) -> String {
events
.split(separator: "\n", omittingEmptySubsequences: false)
.map { line in
if line == "+DOC ---" { return "+DOC" }
if line == "-DOC ..." { return "-DOC" }
return String(line)
}
.joined(separator: "\n")
}
static func trimmingTrailingNewlines(_ value: String) -> String {
var copy = value
while copy.hasSuffix("\n") {
copy.removeLast()
}
return copy
}
private func report(_ text: String) {
FileHandle.standardError.write(Data("\n\(text)\n".utf8))
}
}