PureYAML/Sources/Parsing/LibYAML.Scanner.PlainScalar.swift
Mihaela Mihaljevic 18fdf9b69a feat(parsing): add a resumable scanner increment and broaden the mutation suite
Add Scanner.scanStep(), the first increment of true chunked input streaming
(#4): Reader gains isFinal/append(_:)/markFinal(), and peek(_:) flags ranDry
instead of asserting end of input when the buffer runs out mid-fetch and more
chunks may still arrive. scanStep() snapshots the scanner before each
fetchNextToken() and restores it whenever a fetch runs dry, so a construct
spanning a chunk boundary is retried after more input lands rather than
misread as an early end of input. Every existing call site keeps the default
isFinal: true, so scanAll() and batch parsing are unchanged; ranDry can never
be set when isFinal is true, by construction. A token already queued is not
always safe to release, since libyaml's simple-key mechanism can still splice
a KEY token in front of it, so scanStep() ports libyaml's
yaml_parser_fetch_more_tokens release condition.

Proof: StreamingScannerDifferentialTests.swift drives scanStep() at several
chunk sizes over representative cases always, and over the entire
yaml-test-suite manifest (402 cases) behind PUREYAML_RUN_YAML_TEST_SUITE=1,
asserting the token sequence matches scanAll() exactly. That proof caught a
real bug: lastTokenForbidsInlineSequence() read indentEngine.tokens.last,
which goes empty once tokens are released incrementally, silently breaking
the key: - a rejection. Fixed by tracking IndentEngine.lastToken explicitly,
kept in sync with tokens.last so batch behavior (which never dequeues) is
unchanged.

Not yet wired into PureYAMLStreaming; scanStep() and its supporting types stay
internal until that integration is designed.

Also broaden the real-seed mutation property suite (#6) with the three
mutation classes it named but didn't yet cover: invalidControlCharacter,
mixedTabSpaceIndentation, and truncatedFlowCollection. The test now asserts
diagnostic quality, not just pass/fail: line must fall within the mutated
document, and column, when present, must be 1-based (some ParseError cases,
like tabIndentation, are line-only by design).
2026-07-09 23:38:39 +02:00

143 lines
5.9 KiB
Swift

extension PureYAML.Parsing.LibYAML.Scanner {
private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
/// The folding state carried through `scan_plain_scalar`: the accumulated
/// value plus the pending leading break, trailing breaks, and whitespace that
/// are folded in before the next content character.
private struct PlainState {
var string = ""
var leadingBreak = ""
var trailingBreaks = ""
var whitespaces = ""
var leadingBlanks = false
var endMark: PureYAML.Parsing.Mark
}
/// Port of `yaml_parser_scan_plain_scalar`. Consumes a plain scalar, folding
/// line breaks: a single break between non-empty lines becomes a space, blank
/// lines become literal newlines, and leading/trailing whitespace on folded
/// lines is dropped. Ends at a `: ` value indicator, a comment, a less-indented
/// line (block context), a flow indicator (flow context), or a document marker.
mutating func scanPlainScalar() throws -> PureYAML.Parsing.LibYAML.Token {
let indent = indentEngine.indent + 1
let startMark = reader.mark
var state = PlainState(endMark: reader.mark)
while true {
if reader.column == 0, documentMarkerAhead() {
break
}
if reader.peek() == "#" {
break
}
try consumePlainContent(&state, startMark: startMark)
guard Classify.isBlank(reader.peek()) || Classify.isBreak(reader.peek()) else {
break
}
try consumePlainBreaks(&state, indent: indent, startMark: startMark)
if flowLevel == 0, reader.column < indent {
break
}
}
if state.leadingBlanks {
simpleKeyAllowed = true
}
return PureYAML.Parsing.LibYAML.Token(
type: .scalar(value: state.string, style: .plain),
startMark: startMark,
endMark: state.endMark,
)
}
/// Consume the run of non-blank characters, folding any pending breaks or
/// whitespace ahead of each one.
private mutating func consumePlainContent(_ state: inout PlainState, startMark _: PureYAML.Parsing.Mark) throws {
while let character = reader.peek(), !Classify.isBlankZ(character) {
if character == ":", Classify.isBlankZ(reader.peek(1)) {
break
}
if flowLevel > 0, character == ":", Classify.isFlowIndicator(reader.peek(1)) {
// A `:` before a flow indicator ends the key; the value is empty.
// libyaml rejects this as an unexpected ':', but it is a valid
// value indicator in flow context.
break
}
if flowLevel > 0, Classify.isFlowIndicator(character) {
break
}
foldPending(&state)
state.string.unicodeScalars.append(character)
reader.skip()
state.endMark = reader.mark
}
}
/// Fold the pending leading break, trailing breaks, and whitespace into the
/// accumulated value (the join logic from libyaml's inner loop).
private func foldPending(_ state: inout PlainState) {
guard state.leadingBlanks || !state.whitespaces.isEmpty else {
return
}
guard state.leadingBlanks else {
state.string += state.whitespaces
state.whitespaces = ""
return
}
if state.leadingBreak.first == "\n" {
if state.trailingBreaks.isEmpty {
state.string += " "
} else {
state.string += state.trailingBreaks
state.trailingBreaks = ""
}
state.leadingBreak = ""
} else {
state.string += state.leadingBreak
state.string += state.trailingBreaks
state.leadingBreak = ""
state.trailingBreaks = ""
}
state.leadingBlanks = false
}
/// Consume the run of blanks and line breaks between content, recording them as
/// pending whitespace or folded breaks.
private mutating func consumePlainBreaks(_ state: inout PlainState, indent: Int, startMark: PureYAML.Parsing.Mark) throws {
while Classify.isBlank(reader.peek()) || Classify.isBreak(reader.peek()) {
if Classify.isBlank(reader.peek()) {
if state.leadingBlanks, reader.column < indent, Classify.isTab(reader.peek()), !blankLineAhead() {
// A tab below the indentation is illegal only when it indents
// real content; on an otherwise-blank line it is just a blank
// line the scalar folds over. libyaml rejects it either way.
throw scannerError("found a tab character that violates indentation", at: startMark)
}
if !state.leadingBlanks, let blank = reader.peek() {
state.whitespaces.unicodeScalars.append(blank)
}
reader.skip()
} else if !state.leadingBlanks {
state.whitespaces = ""
reader.skipLine()
state.leadingBreak = "\n"
state.leadingBlanks = true
} else {
reader.skipLine()
state.trailingBreaks += "\n"
}
}
}
private func scannerError(_ problem: String, at mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError {
PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a plain scalar", problem: problem, mark: mark)
}
private mutating func documentMarkerAhead() -> Bool {
let first = reader.peek()
guard first == "-" || first == "." else {
return false
}
return reader.peek(1) == first && reader.peek(2) == first && Classify.isBlankZ(reader.peek(3))
}
}