Mihaela Mihaljevic
e8987e4cfd
The conformance number was measured against a lossy oracle: the JSON projection (in.json) discards anchors, tags, and scalar styles, and exists for only about half the cases, so "passing" often meant only "parse did not throw". Measured against the official suite's own test.event event streams, the true exact conformance is 254 of 308 valid cases (40 emit a wrong event stream, 14 false-reject), well below the JSON-proxy figure. Add a canonical-event gate (Tests/CanonicalEventConformanceTests.swift) that serializes the parser's events into the yaml-test-suite yeast format and compares them exactly to each case's test.event, checking anchors, tags, scalar styles, flow vs block, and explicit document markers, for every valid case. The gate records the gap set and fails on any regression or unrecorded fix. All 402 test.event files are vendored alongside their inputs. To make the comparison exact, the parser Event model now records whether a document boundary was explicit (documentStart/documentEnd carry an `explicit` flag), distinguishing `---`/`...` from an implicit boundary. This resolves 89 cases that were previously correct but unverifiable without normalization. No parser behavior changed; this is measurement and model fidelity. The 40 wrong-event and 14 false-reject cases are the real remaining gaps, to be fixed against this gate.
329 lines
14 KiB
Swift
329 lines
14 KiB
Swift
extension PureYAML.Parsing {
|
|
struct TokenEventParser {
|
|
var cursor: TokenCursor
|
|
let scalarParser: Parser
|
|
var events: [Event]?
|
|
var eventSink: ((Event) throws -> Void)?
|
|
|
|
/// Current node recursion depth, tracked so deeply nested input fails
|
|
/// with a diagnostic instead of overflowing the stack.
|
|
var depth = 0
|
|
|
|
/// Maximum node nesting the parser will follow before failing with
|
|
/// ``ParseError/nestingTooDeep(line:column:)``.
|
|
///
|
|
/// This bound is the single safety invariant for deeply nested input,
|
|
/// and it is *necessary*, not merely convenient. A ``Model/Value`` is a
|
|
/// recursive tree, so composing it, emitting it, comparing it, hashing
|
|
/// it, and -- crucially -- deallocating it all recurse with the nesting
|
|
/// depth. Swift tears down a recursive value type recursively, and that
|
|
/// teardown cannot be made iterative cheaply, so the only way to keep
|
|
/// every operation within the call stack is to bound the depth of the
|
|
/// tree where it is built: here, while parsing.
|
|
///
|
|
/// Because every parsing entry point routes through ``parseNode``, this
|
|
/// one bound caps the depth of any parsed tree, which transitively keeps
|
|
/// the parser, the event composers, the emitter, equality, hashing, and
|
|
/// deallocation all well inside their limits. Measured on the main
|
|
/// thread's 8 MB stack: the parser overflows near depth 225 and the
|
|
/// tree operations near depth 50,000, while real-world specs nest below
|
|
/// ~40. So 64 accepts every realistic document yet rejects pathological
|
|
/// input -- for example a 100 KB `[[[...]]]` -- cleanly and early.
|
|
///
|
|
/// The bound cannot simply be raised: above ~225 the recursive parser
|
|
/// itself overflows before the guard fires (lifting it would require an
|
|
/// iterative parser), and a tree past ~5,000 deep crashes on `deinit`
|
|
/// regardless, so a higher limit would trade a parse-time failure for an
|
|
/// un-fixable teardown crash. 64 is the largest value that is both safe
|
|
/// here and sufficient for real input.
|
|
///
|
|
/// Caveat: a tree built *without* the parser -- for example by encoding
|
|
/// a pathologically deep `Codable` value -- bypasses this bound, and
|
|
/// operations on such a tree are O(depth) in stack like any recursive
|
|
/// value type. Keeping those shallow is the caller's responsibility, the
|
|
/// same way the standard library cannot prevent unbounded recursion in
|
|
/// user code.
|
|
static let maxNestingDepth = 64
|
|
|
|
init(
|
|
tokens: [Token],
|
|
scalarParser: Parser,
|
|
) {
|
|
cursor = TokenCursor(tokens: tokens)
|
|
self.scalarParser = scalarParser
|
|
events = []
|
|
eventSink = nil
|
|
}
|
|
|
|
mutating func parse() throws -> [Event] {
|
|
try parseEvents()
|
|
return events ?? []
|
|
}
|
|
|
|
init(
|
|
tokens: [Token],
|
|
scalarParser: Parser,
|
|
eventSink: @escaping (Event) throws -> Void,
|
|
) {
|
|
cursor = TokenCursor(tokens: tokens)
|
|
self.scalarParser = scalarParser
|
|
events = nil
|
|
self.eventSink = eventSink
|
|
}
|
|
|
|
init(
|
|
yaml: String,
|
|
scalarParser: Parser,
|
|
eventSink: @escaping (Event) throws -> Void,
|
|
) {
|
|
cursor = TokenCursor(yaml: yaml)
|
|
self.scalarParser = scalarParser
|
|
events = nil
|
|
self.eventSink = eventSink
|
|
}
|
|
|
|
mutating func parseStreaming() throws {
|
|
try parseEvents()
|
|
}
|
|
|
|
mutating func parseEvents() throws {
|
|
let streamStart = try expect("stream start") { kind in
|
|
if case .streamStart = kind {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
try emit(.streamStart(mark: streamStart.mark))
|
|
var didParseDocument = false
|
|
while cursor.current?.kind.isStreamEnd != true {
|
|
try parseDocument()
|
|
didParseDocument = true
|
|
}
|
|
if !didParseDocument {
|
|
let mark = cursor.current?.mark ?? streamStart.endMark
|
|
try emit(.documentStart(explicit: false, mark: mark))
|
|
try appendEmptyDocument(at: mark)
|
|
try emit(.documentEnd(explicit: false, mark: mark))
|
|
}
|
|
let streamEnd = try expect("stream end") { kind in
|
|
if case .streamEnd = kind {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
try emit(.streamEnd(mark: streamEnd.mark))
|
|
}
|
|
|
|
mutating func emit(_ event: Event) throws {
|
|
if let eventSink {
|
|
try eventSink(event)
|
|
} else {
|
|
events?.append(event)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.TokenEventParser {
|
|
mutating func parseDocument() throws {
|
|
let (startMark, explicitStart) = try parseDocumentStartMark()
|
|
try emit(.documentStart(explicit: explicitStart, mark: startMark))
|
|
|
|
if cursor.current?.kind.isDocumentEnd == true {
|
|
let end = try expect("document end") { $0.isDocumentEnd }
|
|
try appendEmptyDocument(at: end.mark)
|
|
try emit(.documentEnd(explicit: true, mark: end.endMark))
|
|
return
|
|
}
|
|
|
|
if cursor.current?.kind.isDocumentStart == true || cursor.current?.kind.isStreamEnd == true {
|
|
try appendEmptyDocument(at: cursor.current?.mark ?? startMark)
|
|
try emit(.documentEnd(explicit: false, mark: cursor.current?.mark ?? startMark))
|
|
return
|
|
}
|
|
|
|
var endMark = try parseDocumentRootNode().endMark
|
|
while cursor.current?.kind.isDedent == true {
|
|
let dedent = try expect("dedent") { $0.isDedent }
|
|
endMark = dedent.endMark
|
|
}
|
|
if cursor.current?.kind.isDocumentEnd == true {
|
|
// An explicit `...` end marker closes the document outright. Whatever
|
|
// follows -- a bare node, an explicit `---`, or the end of the stream
|
|
// -- starts a fresh document on the next loop iteration, so emit the
|
|
// end here and return rather than demanding a document boundary token.
|
|
let end = try expect("document end") { $0.isDocumentEnd }
|
|
try emit(.documentEnd(explicit: true, mark: end.endMark))
|
|
return
|
|
}
|
|
if let current = cursor.current, !canEndDocument(at: current) {
|
|
throw PureYAML.Parsing.ParseError.unexpectedToken(
|
|
expected: "document boundary or stream end",
|
|
actual: current.kind.description,
|
|
line: current.mark.line,
|
|
column: current.mark.column,
|
|
)
|
|
}
|
|
try emit(.documentEnd(explicit: false, mark: endMark))
|
|
}
|
|
|
|
mutating func parseDocumentRootNode() throws -> PureYAML.Parsing.NodeResult {
|
|
guard cursor.current?.kind.isIndent == true else {
|
|
return try parseNode()
|
|
}
|
|
_ = try expect("indent") { $0.isIndent }
|
|
let result = try parseNode()
|
|
if cursor.current?.kind.isDedent == true {
|
|
_ = try expect("dedent") { $0.isDedent }
|
|
}
|
|
return result
|
|
}
|
|
|
|
mutating func parseDocumentStartMark() throws -> (mark: PureYAML.Parsing.Mark, explicit: Bool) {
|
|
guard cursor.current?.kind.isDocumentStart == true else {
|
|
return (cursor.current?.mark ?? cursor.previous?.endMark ?? .start, false)
|
|
}
|
|
let start = try cursor.advance()
|
|
return (start?.mark ?? .start, true)
|
|
}
|
|
|
|
mutating func appendEmptyDocument(at mark: PureYAML.Parsing.Mark) throws {
|
|
try emit(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: mark))
|
|
}
|
|
|
|
func canEndDocument(at token: PureYAML.Parsing.Token) -> Bool {
|
|
token.kind.isDocumentStart || token.kind.isStreamEnd
|
|
}
|
|
|
|
mutating func parseNode() throws -> PureYAML.Parsing.NodeResult {
|
|
let properties = try consumeProperties()
|
|
return try parseNode(properties: properties)
|
|
}
|
|
|
|
/// Reject documents nested past ``maxNestingDepth`` with a diagnostic so a
|
|
/// pathologically nested input fails cleanly instead of overflowing the
|
|
/// recursive-descent stack.
|
|
func enforceNestingLimit(properties: PureYAML.Parsing.NodeProperties) throws {
|
|
guard depth > Self.maxNestingDepth else {
|
|
return
|
|
}
|
|
let mark = properties.mark ?? cursor.current?.mark ?? cursor.previous?.endMark ?? .start
|
|
throw PureYAML.Parsing.ParseError.nestingTooDeep(line: mark.line, column: mark.column)
|
|
}
|
|
|
|
mutating func parseNode(
|
|
properties initialProperties: PureYAML.Parsing.NodeProperties,
|
|
) throws -> PureYAML.Parsing.NodeResult {
|
|
depth += 1
|
|
defer { depth -= 1 }
|
|
try enforceNestingLimit(properties: initialProperties)
|
|
var properties = initialProperties
|
|
try properties.mergeUnset(from: consumeProperties())
|
|
|
|
guard let token = cursor.current, !token.kind.isTerminator else {
|
|
return try emptyScalarOrThrow(properties: properties)
|
|
}
|
|
|
|
switch token.kind {
|
|
case let .alias(anchor):
|
|
return try parseAliasNode(anchor: anchor, token: token, properties: properties)
|
|
case .blockEntry:
|
|
return try parseBlockSequence(properties: properties)
|
|
case .blockScalarHeader:
|
|
return try parseBlockScalar(properties: properties)
|
|
case .flowMappingStart:
|
|
return try parseFlowMapping(properties: properties)
|
|
case .flowSequenceStart:
|
|
return try parseFlowSequence(properties: properties)
|
|
case .mappingKey, .mappingValue:
|
|
return try parseBlockMappingNode(token: token, properties: properties)
|
|
case .indent:
|
|
return try parseIndentedNode(properties: properties)
|
|
case let .scalar(value, style):
|
|
return try parseScalarNode(token: token, value: value, style: style, properties: properties)
|
|
default:
|
|
throw PureYAML.Parsing.ParseError.expectedNode(line: token.mark.line, column: token.mark.column)
|
|
}
|
|
}
|
|
|
|
/// Parses an alias node. An alias immediately followed by a `:` value
|
|
/// indicator is the key of a block mapping (`*alias : value`), not a
|
|
/// standalone alias, so route into the mapping and let the key parser
|
|
/// consume it; otherwise emit the alias event directly.
|
|
mutating func parseAliasNode(
|
|
anchor: String,
|
|
token: PureYAML.Parsing.Token,
|
|
properties: PureYAML.Parsing.NodeProperties,
|
|
) throws -> PureYAML.Parsing.NodeResult {
|
|
// An alias is a complete reference to an already-anchored node and cannot
|
|
// itself carry an anchor or tag. libyaml handles ALIAS as the first branch
|
|
// of yaml_parser_parse_node (parser.c:563-570), so the property branches
|
|
// are unreachable for an alias; mirror that by rejecting `&b *alias`. The
|
|
// rejection applies only when the property sits on the *same line* as the
|
|
// alias: a property scanned on the previous line (e.g. `key: &anchor` then an
|
|
// indented `*alias : value`) anchors the enclosing mapping, not the alias key.
|
|
let hasProperties = properties.anchor != nil || properties.tag != nil
|
|
if hasProperties, properties.mark?.line == token.mark.line {
|
|
let mark = properties.mark ?? token.mark
|
|
throw PureYAML.Parsing.ParseError.expectedNode(line: mark.line, column: mark.column)
|
|
}
|
|
// An alias immediately followed by a `:` is the first key of a block mapping;
|
|
// any property that reached here belongs to the mapping (it was on a prior
|
|
// line, per the check above), so route into the mapping carrying it.
|
|
if try cursor.peek()?.kind.isMappingValue == true {
|
|
return try parseBlockMapping(
|
|
properties: properties,
|
|
includeIndentedContinuation: false,
|
|
)
|
|
}
|
|
_ = try cursor.advance()
|
|
let mark = properties.mark ?? token.mark
|
|
try emit(.alias(anchor: anchor, mark: mark))
|
|
return PureYAML.Parsing.NodeResult(kind: .alias, endMark: token.endMark)
|
|
}
|
|
|
|
/// Parses a block mapping introduced by a `mappingKey`/`mappingValue`
|
|
/// indicator. Properties on the same line as the first key indicator belong
|
|
/// to that key node, not to the mapping: `&a a: b` anchors the key `a`,
|
|
/// whereas `&a` alone on a line before the mapping anchors the map.
|
|
mutating func parseBlockMappingNode(
|
|
token: PureYAML.Parsing.Token,
|
|
properties: PureYAML.Parsing.NodeProperties,
|
|
) throws -> PureYAML.Parsing.NodeResult {
|
|
if properties != .none, let mark = properties.mark, mark.line == token.mark.line, token.kind.isMappingKey {
|
|
return try parseBlockMapping(
|
|
properties: .none,
|
|
includeIndentedContinuation: false,
|
|
firstKeyProperties: properties,
|
|
)
|
|
}
|
|
return try parseBlockMapping(
|
|
properties: properties,
|
|
includeIndentedContinuation: false,
|
|
)
|
|
}
|
|
|
|
mutating func parseScalarNode(
|
|
token: PureYAML.Parsing.Token,
|
|
value: String,
|
|
style: PureYAML.Parsing.ScalarStyle,
|
|
properties: PureYAML.Parsing.NodeProperties,
|
|
) throws -> PureYAML.Parsing.NodeResult {
|
|
if try canParsePlainScalarContinuation() {
|
|
return try parsePlainScalarWithContinuation(properties: properties)
|
|
}
|
|
_ = try cursor.advance()
|
|
let decoded = try decodeScalar(value, style: style, line: token.mark.line)
|
|
let mark = properties.mark ?? token.mark
|
|
try emit(.scalar(
|
|
value: decoded,
|
|
anchor: properties.anchor,
|
|
tag: properties.tag,
|
|
style: style,
|
|
mark: mark,
|
|
))
|
|
return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: token.endMark)
|
|
}
|
|
}
|