Add Scanner.scanStep(), the first increment of true chunked input streaming (#4): Reader gains isFinal/append(_:)/markFinal(), and peek(_:) flags ranDry instead of asserting end of input when the buffer runs out mid-fetch and more chunks may still arrive. scanStep() snapshots the scanner before each fetchNextToken() and restores it whenever a fetch runs dry, so a construct spanning a chunk boundary is retried after more input lands rather than misread as an early end of input. Every existing call site keeps the default isFinal: true, so scanAll() and batch parsing are unchanged; ranDry can never be set when isFinal is true, by construction. A token already queued is not always safe to release, since libyaml's simple-key mechanism can still splice a KEY token in front of it, so scanStep() ports libyaml's yaml_parser_fetch_more_tokens release condition. Proof: StreamingScannerDifferentialTests.swift drives scanStep() at several chunk sizes over representative cases always, and over the entire yaml-test-suite manifest (402 cases) behind PUREYAML_RUN_YAML_TEST_SUITE=1, asserting the token sequence matches scanAll() exactly. That proof caught a real bug: lastTokenForbidsInlineSequence() read indentEngine.tokens.last, which goes empty once tokens are released incrementally, silently breaking the key: - a rejection. Fixed by tracking IndentEngine.lastToken explicitly, kept in sync with tokens.last so batch behavior (which never dequeues) is unchanged. Not yet wired into PureYAMLStreaming; scanStep() and its supporting types stay internal until that integration is designed. Also broaden the real-seed mutation property suite (#6) with the three mutation classes it named but didn't yet cover: invalidControlCharacter, mixedTabSpaceIndentation, and truncatedFlowCollection. The test now asserts diagnostic quality, not just pass/fail: line must fall within the mutated document, and column, when present, must be 1-based (some ParseError cases, like tabIndentation, are line-only by design).
143 lines
5.9 KiB
Swift
143 lines
5.9 KiB
Swift
extension PureYAML.Parsing.LibYAML.Scanner {
|
|
private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
|
|
|
|
/// The folding state carried through `scan_plain_scalar`: the accumulated
|
|
/// value plus the pending leading break, trailing breaks, and whitespace that
|
|
/// are folded in before the next content character.
|
|
private struct PlainState {
|
|
var string = ""
|
|
var leadingBreak = ""
|
|
var trailingBreaks = ""
|
|
var whitespaces = ""
|
|
var leadingBlanks = false
|
|
var endMark: PureYAML.Parsing.Mark
|
|
}
|
|
|
|
/// Port of `yaml_parser_scan_plain_scalar`. Consumes a plain scalar, folding
|
|
/// line breaks: a single break between non-empty lines becomes a space, blank
|
|
/// lines become literal newlines, and leading/trailing whitespace on folded
|
|
/// lines is dropped. Ends at a `: ` value indicator, a comment, a less-indented
|
|
/// line (block context), a flow indicator (flow context), or a document marker.
|
|
mutating func scanPlainScalar() throws -> PureYAML.Parsing.LibYAML.Token {
|
|
let indent = indentEngine.indent + 1
|
|
let startMark = reader.mark
|
|
var state = PlainState(endMark: reader.mark)
|
|
|
|
while true {
|
|
if reader.column == 0, documentMarkerAhead() {
|
|
break
|
|
}
|
|
if reader.peek() == "#" {
|
|
break
|
|
}
|
|
try consumePlainContent(&state, startMark: startMark)
|
|
guard Classify.isBlank(reader.peek()) || Classify.isBreak(reader.peek()) else {
|
|
break
|
|
}
|
|
try consumePlainBreaks(&state, indent: indent, startMark: startMark)
|
|
if flowLevel == 0, reader.column < indent {
|
|
break
|
|
}
|
|
}
|
|
|
|
if state.leadingBlanks {
|
|
simpleKeyAllowed = true
|
|
}
|
|
return PureYAML.Parsing.LibYAML.Token(
|
|
type: .scalar(value: state.string, style: .plain),
|
|
startMark: startMark,
|
|
endMark: state.endMark,
|
|
)
|
|
}
|
|
|
|
/// Consume the run of non-blank characters, folding any pending breaks or
|
|
/// whitespace ahead of each one.
|
|
private mutating func consumePlainContent(_ state: inout PlainState, startMark _: PureYAML.Parsing.Mark) throws {
|
|
while let character = reader.peek(), !Classify.isBlankZ(character) {
|
|
if character == ":", Classify.isBlankZ(reader.peek(1)) {
|
|
break
|
|
}
|
|
if flowLevel > 0, character == ":", Classify.isFlowIndicator(reader.peek(1)) {
|
|
// A `:` before a flow indicator ends the key; the value is empty.
|
|
// libyaml rejects this as an unexpected ':', but it is a valid
|
|
// value indicator in flow context.
|
|
break
|
|
}
|
|
if flowLevel > 0, Classify.isFlowIndicator(character) {
|
|
break
|
|
}
|
|
foldPending(&state)
|
|
state.string.unicodeScalars.append(character)
|
|
reader.skip()
|
|
state.endMark = reader.mark
|
|
}
|
|
}
|
|
|
|
/// Fold the pending leading break, trailing breaks, and whitespace into the
|
|
/// accumulated value (the join logic from libyaml's inner loop).
|
|
private func foldPending(_ state: inout PlainState) {
|
|
guard state.leadingBlanks || !state.whitespaces.isEmpty else {
|
|
return
|
|
}
|
|
guard state.leadingBlanks else {
|
|
state.string += state.whitespaces
|
|
state.whitespaces = ""
|
|
return
|
|
}
|
|
if state.leadingBreak.first == "\n" {
|
|
if state.trailingBreaks.isEmpty {
|
|
state.string += " "
|
|
} else {
|
|
state.string += state.trailingBreaks
|
|
state.trailingBreaks = ""
|
|
}
|
|
state.leadingBreak = ""
|
|
} else {
|
|
state.string += state.leadingBreak
|
|
state.string += state.trailingBreaks
|
|
state.leadingBreak = ""
|
|
state.trailingBreaks = ""
|
|
}
|
|
state.leadingBlanks = false
|
|
}
|
|
|
|
/// Consume the run of blanks and line breaks between content, recording them as
|
|
/// pending whitespace or folded breaks.
|
|
private mutating func consumePlainBreaks(_ state: inout PlainState, indent: Int, startMark: PureYAML.Parsing.Mark) throws {
|
|
while Classify.isBlank(reader.peek()) || Classify.isBreak(reader.peek()) {
|
|
if Classify.isBlank(reader.peek()) {
|
|
if state.leadingBlanks, reader.column < indent, Classify.isTab(reader.peek()), !blankLineAhead() {
|
|
// A tab below the indentation is illegal only when it indents
|
|
// real content; on an otherwise-blank line it is just a blank
|
|
// line the scalar folds over. libyaml rejects it either way.
|
|
throw scannerError("found a tab character that violates indentation", at: startMark)
|
|
}
|
|
if !state.leadingBlanks, let blank = reader.peek() {
|
|
state.whitespaces.unicodeScalars.append(blank)
|
|
}
|
|
reader.skip()
|
|
} else if !state.leadingBlanks {
|
|
state.whitespaces = ""
|
|
reader.skipLine()
|
|
state.leadingBreak = "\n"
|
|
state.leadingBlanks = true
|
|
} else {
|
|
reader.skipLine()
|
|
state.trailingBreaks += "\n"
|
|
}
|
|
}
|
|
}
|
|
|
|
private func scannerError(_ problem: String, at mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError {
|
|
PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a plain scalar", problem: problem, mark: mark)
|
|
}
|
|
|
|
private mutating func documentMarkerAhead() -> Bool {
|
|
let first = reader.peek()
|
|
guard first == "-" || first == "." else {
|
|
return false
|
|
}
|
|
return reader.peek(1) == first && reader.peek(2) == first && Classify.isBlankZ(reader.peek(3))
|
|
}
|
|
}
|