Add Scanner.scanStep(), the first increment of true chunked input streaming (#4): Reader gains isFinal/append(_:)/markFinal(), and peek(_:) flags ranDry instead of asserting end of input when the buffer runs out mid-fetch and more chunks may still arrive. scanStep() snapshots the scanner before each fetchNextToken() and restores it whenever a fetch runs dry, so a construct spanning a chunk boundary is retried after more input lands rather than misread as an early end of input. Every existing call site keeps the default isFinal: true, so scanAll() and batch parsing are unchanged; ranDry can never be set when isFinal is true, by construction. A token already queued is not always safe to release, since libyaml's simple-key mechanism can still splice a KEY token in front of it, so scanStep() ports libyaml's yaml_parser_fetch_more_tokens release condition. Proof: StreamingScannerDifferentialTests.swift drives scanStep() at several chunk sizes over representative cases always, and over the entire yaml-test-suite manifest (402 cases) behind PUREYAML_RUN_YAML_TEST_SUITE=1, asserting the token sequence matches scanAll() exactly. That proof caught a real bug: lastTokenForbidsInlineSequence() read indentEngine.tokens.last, which goes empty once tokens are released incrementally, silently breaking the key: - a rejection. Fixed by tracking IndentEngine.lastToken explicitly, kept in sync with tokens.last so batch behavior (which never dequeues) is unchanged. Not yet wired into PureYAMLStreaming; scanStep() and its supporting types stay internal until that integration is designed. Also broaden the real-seed mutation property suite (#6) with the three mutation classes it named but didn't yet cover: invalidControlCharacter, mixedTabSpaceIndentation, and truncatedFlowCollection. The test now asserts diagnostic quality, not just pass/fail: line must fall within the mutated document, and column, when present, must be 1-based (some ParseError cases, like tabIndentation, are line-only by design).
140 lines
6.6 KiB
Swift
140 lines
6.6 KiB
Swift
extension PureYAML.Parsing.LibYAML {
|
|
/// Aggregate scanner state, mirroring the fields of `yaml_parser_t` that the
|
|
/// ported functions read and write. It is built up as the port proceeds;
|
|
/// currently it holds the indentation engine and the simple-key mechanism.
|
|
/// Column values follow libyaml's 0-based convention internally; conversion to
|
|
/// the package's 1-based `Mark` happens at token emission during integration.
|
|
struct Scanner {
|
|
/// The input buffer and position (`parser->buffer` / `parser->mark`).
|
|
var reader: Reader
|
|
/// The indentation stack and `roll_indent` / `unroll_indent` (also the
|
|
/// owner of `flow_level` and the token queue).
|
|
var indentEngine = IndentEngine()
|
|
/// `parser->simple_keys`. Always holds at least the block-context level.
|
|
var simpleKeys: [SimpleKey] = [.empty]
|
|
/// `parser->simple_key_allowed`. True at stream start.
|
|
var simpleKeyAllowed = true
|
|
/// Whether a tab appeared in the indentation before the pending token. A
|
|
/// tab is valid in-line separation but never indentation, so a block
|
|
/// indicator or implicit mapping key reached through one is an error.
|
|
var tabbedSeparation = false
|
|
/// Whether reaching the pending token crossed at least one line break. In
|
|
/// flow context a continuation line must be indented past the parent block,
|
|
/// so a token on a new line at or below that indent is an error.
|
|
var crossedLineBreak = false
|
|
/// Whether a tab appeared in the leading whitespace of a flow continuation
|
|
/// line. A tab is never indentation, so flow content reached through one is
|
|
/// an error even though the tab inflates the column.
|
|
var flowLeadingTab = false
|
|
/// Whether the most recent `:` value indicator sat on the same line as its
|
|
/// key (an inline implicit-key value). Such a value may not open a block
|
|
/// sequence on that line (`key: - a`), whereas an explicit key whose `:` is
|
|
/// on a later line may (`? k`\n`: - a`).
|
|
var lastValueWasInline = false
|
|
/// For each open flow level, whether it is a sequence (`[`) rather than a
|
|
/// mapping (`{`). A flow-sequence single-pair implicit key is single-line
|
|
/// (`c-s-implicit-json-key`), so its `:` may not cross a line break, unlike
|
|
/// a flow-mapping adjacent value (`{ "a"`\n`: b }` is allowed).
|
|
var flowIsSequence: [Bool] = []
|
|
/// `parser->mark`, kept in sync with the reader at each fetch step.
|
|
var mark: PureYAML.Parsing.Mark = .start
|
|
/// `parser->stream_start_produced`.
|
|
var streamStartProduced = false
|
|
/// `parser->stream_end_produced`.
|
|
var streamEndProduced = false
|
|
|
|
init(_ input: String = "", isFinal: Bool = true) {
|
|
reader = Reader(input, isFinal: isFinal)
|
|
}
|
|
|
|
/// `parser->flow_level`, kept on the indent engine.
|
|
var flowLevel: Int {
|
|
indentEngine.flowLevel
|
|
}
|
|
|
|
/// Port of `yaml_parser_stale_simple_keys`. A potential simple key is
|
|
/// invalid once it is on an earlier line or more than 1024 characters back
|
|
/// (a simple key is single-line and bounded); a required one going stale is
|
|
/// an error.
|
|
mutating func staleSimpleKeys() throws {
|
|
for index in simpleKeys.indices {
|
|
let key = simpleKeys[index]
|
|
guard key.possible else {
|
|
continue
|
|
}
|
|
// A block-context key (index 0) is single-line. A flow-context key
|
|
// (index >= 1) may be separated from its ':' by line breaks under
|
|
// YAML 1.2, so a new line alone does not invalidate it; both remain
|
|
// bounded to 1024 characters. libyaml invalidates every key on a new
|
|
// line, which wrongly rejects multi-line flow keys.
|
|
let tooFarBack = key.mark.index + 1024 < mark.index
|
|
let stale = index == 0 ? (key.mark.line < mark.line || tooFarBack) : tooFarBack
|
|
guard stale else {
|
|
continue
|
|
}
|
|
if key.required {
|
|
throw ScannerError(
|
|
context: "while scanning a simple key",
|
|
problem: "could not find expected ':'",
|
|
mark: key.mark,
|
|
)
|
|
}
|
|
simpleKeys[index].possible = false
|
|
}
|
|
}
|
|
|
|
/// Port of `yaml_parser_save_simple_key`. Record the current position as a
|
|
/// potential simple key for this flow level when one is allowed. A key is
|
|
/// required (a `:` must follow) when it sits exactly at the block indent.
|
|
mutating func saveSimpleKey() throws {
|
|
let required = flowLevel == 0 && indentEngine.indent == mark.column
|
|
guard simpleKeyAllowed else {
|
|
return
|
|
}
|
|
let key = SimpleKey(
|
|
possible: true,
|
|
required: required,
|
|
tokenNumber: indentEngine.tokensParsed + indentEngine.tokens.count,
|
|
mark: mark,
|
|
tabbed: tabbedSeparation,
|
|
)
|
|
try removeSimpleKey()
|
|
simpleKeys[simpleKeys.count - 1] = key
|
|
}
|
|
|
|
/// Port of `yaml_parser_remove_simple_key`. Drop the potential key at the
|
|
/// current flow level; removing a required key is an error.
|
|
mutating func removeSimpleKey() throws {
|
|
let top = simpleKeys.count - 1
|
|
if simpleKeys[top].possible, simpleKeys[top].required {
|
|
throw ScannerError(
|
|
context: "while scanning a simple key",
|
|
problem: "could not find expected ':'",
|
|
mark: simpleKeys[top].mark,
|
|
)
|
|
}
|
|
simpleKeys[top].possible = false
|
|
}
|
|
|
|
/// Port of `yaml_parser_increase_flow_level`. Push an empty simple key for
|
|
/// the new flow level, recording whether the level is a sequence.
|
|
mutating func increaseFlowLevel(isSequence: Bool) {
|
|
simpleKeys.append(.empty)
|
|
flowIsSequence.append(isSequence)
|
|
indentEngine.flowLevel += 1
|
|
}
|
|
|
|
/// Port of `yaml_parser_decrease_flow_level`. Pop the flow level's simple
|
|
/// key.
|
|
mutating func decreaseFlowLevel() {
|
|
if indentEngine.flowLevel > 0 {
|
|
indentEngine.flowLevel -= 1
|
|
simpleKeys.removeLast()
|
|
if !flowIsSequence.isEmpty {
|
|
flowIsSequence.removeLast()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|