PureYAML/Sources/Parsing/LibYAML.Scanner.BlockScalar.swift
Mihaela Mihaljevic 18fdf9b69a feat(parsing): add a resumable scanner increment and broaden the mutation suite
Add Scanner.scanStep(), the first increment of true chunked input streaming
(#4): Reader gains isFinal/append(_:)/markFinal(), and peek(_:) flags ranDry
instead of asserting end of input when the buffer runs out mid-fetch and more
chunks may still arrive. scanStep() snapshots the scanner before each
fetchNextToken() and restores it whenever a fetch runs dry, so a construct
spanning a chunk boundary is retried after more input lands rather than
misread as an early end of input. Every existing call site keeps the default
isFinal: true, so scanAll() and batch parsing are unchanged; ranDry can never
be set when isFinal is true, by construction. A token already queued is not
always safe to release, since libyaml's simple-key mechanism can still splice
a KEY token in front of it, so scanStep() ports libyaml's
yaml_parser_fetch_more_tokens release condition.

Proof: StreamingScannerDifferentialTests.swift drives scanStep() at several
chunk sizes over representative cases always, and over the entire
yaml-test-suite manifest (402 cases) behind PUREYAML_RUN_YAML_TEST_SUITE=1,
asserting the token sequence matches scanAll() exactly. That proof caught a
real bug: lastTokenForbidsInlineSequence() read indentEngine.tokens.last,
which goes empty once tokens are released incrementally, silently breaking
the key: - a rejection. Fixed by tracking IndentEngine.lastToken explicitly,
kept in sync with tokens.last so batch behavior (which never dequeues) is
unchanged.

Not yet wired into PureYAMLStreaming; scanStep() and its supporting types stay
internal until that integration is designed.

Also broaden the real-seed mutation property suite (#6) with the three
mutation classes it named but didn't yet cover: invalidControlCharacter,
mixedTabSpaceIndentation, and truncatedFlowCollection. The test now asserts
diagnostic quality, not just pass/fail: line must fall within the mutated
document, and column, when present, must be 1-based (some ParseError cases,
like tabIndentation, are line-only by design).
2026-07-09 23:38:39 +02:00

206 lines
9.1 KiB
Swift

extension PureYAML.Parsing.LibYAML.Scanner {
private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
/// The accumulation state for `scan_block_scalar`.
private struct BlockState {
var string = ""
var leadingBreak = ""
var trailingBreaks = ""
var leadingBlank = false
}
/// Port of `yaml_parser_scan_block_scalar`. Scans a literal (`|`) or folded
/// (`>`) block scalar: parses the chomping (`+`/`-`) and indentation indicators
/// from the header, auto-detects the indentation if not given, then folds the
/// content (literal keeps breaks, folded joins equally-indented non-empty lines
/// with a space) and applies clip/strip/keep chomping to the trailing breaks.
mutating func scanBlockScalar(literal: Bool) throws -> PureYAML.Parsing.LibYAML.Token {
let startMark = reader.mark
reader.skip()
let header = try readBlockScalarHeader(startMark: startMark)
try consumeBlockHeaderTail(startMark: startMark)
var endMark = reader.mark
// -1 is the "indentation not yet auto-detected" sentinel. libyaml uses 0,
// but a root-level block scalar legitimately detects indentation 0, so 0
// must be a representable detected value rather than the sentinel.
var indent = -1
if header.increment != 0 {
indent = indentEngine.indent >= 0 ? indentEngine.indent + header.increment : header.increment
}
var state = BlockState()
try scanBlockScalarBreaks(indent: &indent, state: &state, startMark: startMark, endMark: &endMark)
while reader.column == indent, reader.peek() != nil {
// A root block scalar (indent 0) ends at a `---`/`...` document marker
// in column 0. libyaml never needs this because it floors the indent
// at 1, but indent 0 lets the loop reach a column-0 marker line.
if reader.column == 0, documentMarkerAhead() {
break
}
foldBlockLine(&state, literal: literal)
while let character = reader.peek(), !Classify.isBreakZ(character) {
state.string.unicodeScalars.append(character)
reader.skip()
}
state.leadingBreak += readBlockLine()
try scanBlockScalarBreaks(indent: &indent, state: &state, startMark: startMark, endMark: &endMark)
}
if header.chomping != -1 {
state.string += state.leadingBreak
}
if header.chomping == 1 {
state.string += state.trailingBreaks
}
return PureYAML.Parsing.LibYAML.Token(
type: .scalar(value: state.string, style: literal ? .literal : .folded),
startMark: startMark,
endMark: endMark,
)
}
/// Parse the chomping and indentation indicators in either order.
private mutating func readBlockScalarHeader(startMark: PureYAML.Parsing.Mark) throws -> (chomping: Int, increment: Int) {
var chomping = 0
var increment = 0
if reader.peek() == "+" || reader.peek() == "-" {
chomping = reader.peek() == "+" ? 1 : -1
reader.skip()
if Classify.isDigit(reader.peek()) {
increment = try blockIncrement(startMark: startMark)
}
} else if Classify.isDigit(reader.peek()) {
increment = try blockIncrement(startMark: startMark)
if reader.peek() == "+" || reader.peek() == "-" {
chomping = reader.peek() == "+" ? 1 : -1
reader.skip()
}
}
return (chomping, increment)
}
private mutating func blockIncrement(startMark: PureYAML.Parsing.Mark) throws -> Int {
if reader.peek() == "0" {
throw blockError("found an indentation indicator equal to 0", startMark)
}
let value = digitValue(reader.peek())
reader.skip()
return value
}
/// Eat trailing blanks and a comment to the end of the header line, then the
/// line break.
private mutating func consumeBlockHeaderTail(startMark: PureYAML.Parsing.Mark) throws {
while reader.peek() == " " {
reader.skip()
}
if reader.peek() == "#" {
// A comment needs a preceding space, so `>#` (a `#` adjacent to the
// block scalar indicator) is malformed, not a comment. libyaml accepts it.
if reader.column > 0, let previous = reader.peek(-1), !Classify.isBlank(previous) {
throw blockError("found a comment without a preceding space", startMark)
}
while !Classify.isBreakZ(reader.peek()) {
reader.skip()
}
}
guard Classify.isBreakZ(reader.peek()) else {
throw blockError("did not find expected comment or line break", startMark)
}
if Classify.isBreak(reader.peek()) {
reader.skipLine()
}
}
/// Port of `yaml_parser_scan_block_scalar_breaks`: eat indentation spaces and
/// blank lines, auto-detecting the indentation level when none was given.
private mutating func scanBlockScalarBreaks(indent: inout Int, state: inout BlockState, startMark: PureYAML.Parsing.Mark, endMark: inout PureYAML.Parsing.Mark) throws {
var maxIndent = 0
endMark = reader.mark
// A tab is content (or separation) once the column reaches the content
// indentation; it is illegal only while still inside the required
// indentation. During auto-detection the floor is the parent indent + 1.
let tabFloor = indent >= 0 ? indent : max(0, indentEngine.indent + 1)
while true {
let lineStart = reader.column
while indent < 0 || reader.column < indent, reader.peek() == " " {
reader.skip()
}
if reader.column > maxIndent {
maxIndent = reader.column
}
if reader.column < tabFloor, Classify.isTab(reader.peek()) {
throw blockError("found a tab character where an indentation space is expected", startMark)
}
if !Classify.isBreak(reader.peek()) {
// A trailing whitespace-only line that ends at end of input without
// a break still counts as one line break for keep chomping (as the
// test-suite reference does), even with no break byte present.
if reader.peek() == nil, reader.column > lineStart {
state.trailingBreaks += "\n"
}
break
}
state.trailingBreaks += readBlockLine()
endMark = reader.mark
}
// A leading empty line may not be more indented than the first content
// line; libyaml silently detects the deeper indent and drops the content.
if indent < 0, !Classify.isBreakZ(reader.peek()), reader.column < maxIndent {
throw blockError("found a leading empty line more indented than the content", startMark)
}
if indent < 0 {
// Auto-detected indentation. A root-level block scalar has a parent
// indent of -1, so the floor is 0 (allowing column-0 content), not
// libyaml's 1. maxIndent >= 0 keeps the result non-negative.
indent = max(maxIndent, indentEngine.indent + 1)
}
}
/// Fold the pending breaks at the start of a content line (folded style joins a
/// single break between non-blank lines into a space; otherwise breaks are
/// kept).
private mutating func foldBlockLine(_ state: inout BlockState, literal: Bool) {
let trailingBlank = Classify.isBlank(reader.peek())
if !literal, state.leadingBreak.first == "\n", !state.leadingBlank, !trailingBlank {
if state.trailingBreaks.isEmpty {
state.string += " "
}
state.leadingBreak = ""
} else {
state.string += state.leadingBreak
state.leadingBreak = ""
}
state.string += state.trailingBreaks
state.trailingBreaks = ""
state.leadingBlank = Classify.isBlank(reader.peek())
}
private mutating func readBlockLine() -> String {
reader.skipLine()
return "\n"
}
/// Whether a `---` or `...` document marker (three repeats then a blank, break,
/// or end of input) starts at the current position.
private mutating func documentMarkerAhead() -> Bool {
let first = reader.peek()
guard first == "-" || first == "." else {
return false
}
return reader.peek(1) == first && reader.peek(2) == first && Classify.isBlankZ(reader.peek(3))
}
private func digitValue(_ scalar: Unicode.Scalar?) -> Int {
guard let scalar, scalar >= "0", scalar <= "9" else {
return 0
}
return Int(scalar.value - Unicode.Scalar("0").value)
}
private func blockError(_ problem: String, _ mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError {
PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a block scalar", problem: problem, mark: mark)
}
}