Add Scanner.scanStep(), the first increment of true chunked input streaming (#4): Reader gains isFinal/append(_:)/markFinal(), and peek(_:) flags ranDry instead of asserting end of input when the buffer runs out mid-fetch and more chunks may still arrive. scanStep() snapshots the scanner before each fetchNextToken() and restores it whenever a fetch runs dry, so a construct spanning a chunk boundary is retried after more input lands rather than misread as an early end of input. Every existing call site keeps the default isFinal: true, so scanAll() and batch parsing are unchanged; ranDry can never be set when isFinal is true, by construction. A token already queued is not always safe to release, since libyaml's simple-key mechanism can still splice a KEY token in front of it, so scanStep() ports libyaml's yaml_parser_fetch_more_tokens release condition. Proof: StreamingScannerDifferentialTests.swift drives scanStep() at several chunk sizes over representative cases always, and over the entire yaml-test-suite manifest (402 cases) behind PUREYAML_RUN_YAML_TEST_SUITE=1, asserting the token sequence matches scanAll() exactly. That proof caught a real bug: lastTokenForbidsInlineSequence() read indentEngine.tokens.last, which goes empty once tokens are released incrementally, silently breaking the key: - a rejection. Fixed by tracking IndentEngine.lastToken explicitly, kept in sync with tokens.last so batch behavior (which never dequeues) is unchanged. Not yet wired into PureYAMLStreaming; scanStep() and its supporting types stay internal until that integration is designed. Also broaden the real-seed mutation property suite (#6) with the three mutation classes it named but didn't yet cover: invalidControlCharacter, mixedTabSpaceIndentation, and truncatedFlowCollection. The test now asserts diagnostic quality, not just pass/fail: line must fall within the mutated document, and column, when present, must be 1-based (some ParseError cases, like tabIndentation, are line-only by design).
99 lines
4.5 KiB
Swift
99 lines
4.5 KiB
Swift
extension PureYAML.Parsing.LibYAML {
|
|
/// Port of the indentation stack from `yaml_parser_t` plus
|
|
/// `yaml_parser_roll_indent` and `yaml_parser_unroll_indent`. libyaml tracks a
|
|
/// single `indent` column (the current block collection's indentation, -1 at
|
|
/// the document root) and a stack of the enclosing levels. A block collection
|
|
/// is opened only when content is strictly more indented than the current
|
|
/// level, which is exactly why `- a:` followed by a same-column `b:` keeps `b`
|
|
/// a sibling and leaves `a` an empty value.
|
|
struct IndentEngine {
|
|
/// `parser->indent`. -1 at the document root, matching libyaml.
|
|
private(set) var indent = -1
|
|
/// `parser->indents`, the stack of enclosing indentation levels.
|
|
private(set) var indents: [Int] = []
|
|
/// `parser->flow_level`; indentation is inert inside flow collections.
|
|
var flowLevel = 0
|
|
/// The token queue (`parser->tokens`) the engine appends to or, for
|
|
/// `roll_indent`, inserts a start token into at a recorded position.
|
|
private(set) var tokens: [Token] = []
|
|
/// `parser->tokens_parsed`, the count already dequeued, used to turn an
|
|
/// absolute token number into an index into the live queue.
|
|
var tokensParsed = 0
|
|
/// The most recently enqueued or spliced token, kept in sync with
|
|
/// `tokens.last` after every mutation below. Batch scanning never dequeues
|
|
/// (`tokensParsed` stays 0, `dequeue()` unused), so `tokens.last` alone
|
|
/// would suffice there; this field exists for a resumable scanner
|
|
/// (`Scanner.scanStep()`), which releases and discards tokens as soon as
|
|
/// they are safe to hand out, so `tokens` can be empty exactly when a check
|
|
/// like `lastTokenForbidsInlineSequence()` needs to know what the last
|
|
/// token was.
|
|
private(set) var lastToken: Token?
|
|
|
|
/// Port of `yaml_parser_roll_indent`. In flow context it does nothing.
|
|
/// When the column is greater than the current indent, it pushes the
|
|
/// current indent, adopts the new column, and emits the block start token.
|
|
/// `number == -1` appends; otherwise the token is inserted at the recorded
|
|
/// queue position so a `BLOCK-MAPPING-START` precedes a simple key that was
|
|
/// already queued.
|
|
mutating func rollIndent(
|
|
column: Int,
|
|
number: Int,
|
|
type: TokenType,
|
|
mark: PureYAML.Parsing.Mark,
|
|
) {
|
|
if flowLevel != 0 {
|
|
return
|
|
}
|
|
if indent < column {
|
|
indents.append(indent)
|
|
indent = column
|
|
let token = Token(type: type, startMark: mark, endMark: mark)
|
|
if number == -1 {
|
|
tokens.append(token)
|
|
} else {
|
|
tokens.insert(token, at: number - tokensParsed)
|
|
}
|
|
lastToken = tokens.last
|
|
}
|
|
}
|
|
|
|
/// Port of `yaml_parser_unroll_indent`. In flow context it does nothing.
|
|
/// While the current indent is deeper than the column, it emits a
|
|
/// `BLOCK-END` and pops the enclosing level, closing every block collection
|
|
/// that the dedent ends.
|
|
mutating func unrollIndent(column: Int, mark: PureYAML.Parsing.Mark) {
|
|
if flowLevel != 0 {
|
|
return
|
|
}
|
|
while indent > column {
|
|
tokens.append(Token(type: .blockEnd, startMark: mark, endMark: mark))
|
|
indent = indents.removeLast()
|
|
}
|
|
lastToken = tokens.last ?? lastToken
|
|
}
|
|
|
|
/// `ENQUEUE`: append a token to the queue.
|
|
mutating func enqueue(_ token: Token) {
|
|
tokens.append(token)
|
|
lastToken = token
|
|
}
|
|
|
|
/// `QUEUE_INSERT`: insert a token at an absolute token number, used to
|
|
/// splice a `KEY` token in front of a node already queued.
|
|
mutating func insert(_ token: Token, atTokenNumber number: Int) {
|
|
tokens.insert(token, at: number - tokensParsed)
|
|
lastToken = tokens.last
|
|
}
|
|
|
|
/// Dequeue the front token, advancing `tokensParsed` like libyaml's
|
|
/// `yaml_parser_scan` does when it hands a token to the parser.
|
|
mutating func dequeue() -> Token? {
|
|
guard !tokens.isEmpty else {
|
|
return nil
|
|
}
|
|
tokensParsed += 1
|
|
return tokens.removeFirst()
|
|
}
|
|
}
|
|
}
|