Add Scanner.scanStep(), the first increment of true chunked input streaming (#4): Reader gains isFinal/append(_:)/markFinal(), and peek(_:) flags ranDry instead of asserting end of input when the buffer runs out mid-fetch and more chunks may still arrive. scanStep() snapshots the scanner before each fetchNextToken() and restores it whenever a fetch runs dry, so a construct spanning a chunk boundary is retried after more input lands rather than misread as an early end of input. Every existing call site keeps the default isFinal: true, so scanAll() and batch parsing are unchanged; ranDry can never be set when isFinal is true, by construction. A token already queued is not always safe to release, since libyaml's simple-key mechanism can still splice a KEY token in front of it, so scanStep() ports libyaml's yaml_parser_fetch_more_tokens release condition. Proof: StreamingScannerDifferentialTests.swift drives scanStep() at several chunk sizes over representative cases always, and over the entire yaml-test-suite manifest (402 cases) behind PUREYAML_RUN_YAML_TEST_SUITE=1, asserting the token sequence matches scanAll() exactly. That proof caught a real bug: lastTokenForbidsInlineSequence() read indentEngine.tokens.last, which goes empty once tokens are released incrementally, silently breaking the key: - a rejection. Fixed by tracking IndentEngine.lastToken explicitly, kept in sync with tokens.last so batch behavior (which never dequeues) is unchanged. Not yet wired into PureYAMLStreaming; scanStep() and its supporting types stay internal until that integration is designed. Also broaden the real-seed mutation property suite (#6) with the three mutation classes it named but didn't yet cover: invalidControlCharacter, mixedTabSpaceIndentation, and truncatedFlowCollection. The test now asserts diagnostic quality, not just pass/fail: line must fall within the mutated document, and column, when present, must be 1-based (some ParseError cases, like tabIndentation, are line-only by design).
89 lines
3.8 KiB
Swift
89 lines
3.8 KiB
Swift
extension PureYAML.Parsing.LibYAML.Scanner {
|
|
private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
|
|
private typealias ScannerError = PureYAML.Parsing.LibYAML.ScannerError
|
|
|
|
/// Port of `yaml_parser_scan_to_next_token`: skip a leading BOM, spaces and
|
|
/// tabs, a comment, and a single line break, which re-allows a simple key in
|
|
/// the block context. A tab is valid in-line separation, so it is skipped
|
|
/// here; `tabbedSeparation` records when one preceded the token so a block
|
|
/// indicator or implicit key (where the tab would be indentation) can reject.
|
|
mutating func scanToNextToken() throws {
|
|
crossedLineBreak = false
|
|
flowLeadingTab = false
|
|
while true {
|
|
tabbedSeparation = false
|
|
if reader.column == 0, reader.peek() == "\u{FEFF}" {
|
|
reader.skip()
|
|
}
|
|
while reader.peek() == " " || Classify.isTab(reader.peek()) {
|
|
if Classify.isTab(reader.peek()) {
|
|
noteLeadingTab()
|
|
}
|
|
reader.skip()
|
|
}
|
|
// YAML 1.2: a line containing only whitespace is blank even when that
|
|
// whitespace includes tabs. libyaml leaves a leading tab in place and
|
|
// later rejects it; instead consume the trailing blanks so the line
|
|
// break below is reached and the blank line is skipped.
|
|
if blankLineAhead() {
|
|
while reader.peek() == " " || Classify.isTab(reader.peek()) {
|
|
reader.skip()
|
|
}
|
|
}
|
|
if reader.peek() == "#" {
|
|
try consumeComment()
|
|
}
|
|
if Classify.isBreak(reader.peek()) {
|
|
reader.skipLine()
|
|
crossedLineBreak = true
|
|
if flowLevel == 0 {
|
|
simpleKeyAllowed = true
|
|
}
|
|
} else {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Record what a tab in the leading whitespace implies. In block context with a
|
|
/// simple key allowed it is potential indentation before an indicator. In flow
|
|
/// context, below the continuation indent on a non-blank line it is indentation
|
|
/// abuse (a tab is never indentation); at or above the indent it is separation.
|
|
private mutating func noteLeadingTab() {
|
|
if flowLevel == 0, simpleKeyAllowed {
|
|
tabbedSeparation = true
|
|
}
|
|
if flowLevel > 0, crossedLineBreak, reader.column < indentEngine.indent + 1, !blankLineAhead() {
|
|
flowLeadingTab = true
|
|
}
|
|
}
|
|
|
|
/// Consume a comment to the end of the line. A comment must be preceded by
|
|
/// whitespace or start the line; a `#` adjacent to the previous token (e.g.
|
|
/// `"x"#`, `]#`, `,#`) is not a comment. The plain-scalar scanner consumes its
|
|
/// own trailing blanks, so the character just before the `#` is the reliable
|
|
/// signal. libyaml accepts the adjacent form.
|
|
private mutating func consumeComment() throws {
|
|
if reader.column > 0, let previous = reader.peek(-1), !Classify.isBlank(previous) {
|
|
throw ScannerError(
|
|
context: "while scanning for the next token",
|
|
problem: "found a comment without a preceding space",
|
|
mark: reader.mark,
|
|
)
|
|
}
|
|
while !Classify.isBreakZ(reader.peek()) {
|
|
reader.skip()
|
|
}
|
|
}
|
|
|
|
/// Whether the rest of the current line, from the current position, is only
|
|
/// spaces and tabs up to a line break or end of stream (a blank line).
|
|
mutating func blankLineAhead() -> Bool {
|
|
var offset = 0
|
|
while let character = reader.peek(offset), character == " " || Classify.isTab(character) {
|
|
offset += 1
|
|
}
|
|
return Classify.isBreakZ(reader.peek(offset))
|
|
}
|
|
}
|