Add Scanner.scanStep(), the first increment of true chunked input streaming (#4): Reader gains isFinal/append(_:)/markFinal(), and peek(_:) flags ranDry instead of asserting end of input when the buffer runs out mid-fetch and more chunks may still arrive. scanStep() snapshots the scanner before each fetchNextToken() and restores it whenever a fetch runs dry, so a construct spanning a chunk boundary is retried after more input lands rather than misread as an early end of input. Every existing call site keeps the default isFinal: true, so scanAll() and batch parsing are unchanged; ranDry can never be set when isFinal is true, by construction. A token already queued is not always safe to release, since libyaml's simple-key mechanism can still splice a KEY token in front of it, so scanStep() ports libyaml's yaml_parser_fetch_more_tokens release condition. Proof: StreamingScannerDifferentialTests.swift drives scanStep() at several chunk sizes over representative cases always, and over the entire yaml-test-suite manifest (402 cases) behind PUREYAML_RUN_YAML_TEST_SUITE=1, asserting the token sequence matches scanAll() exactly. That proof caught a real bug: lastTokenForbidsInlineSequence() read indentEngine.tokens.last, which goes empty once tokens are released incrementally, silently breaking the key: - a rejection. Fixed by tracking IndentEngine.lastToken explicitly, kept in sync with tokens.last so batch behavior (which never dequeues) is unchanged. Not yet wired into PureYAMLStreaming; scanStep() and its supporting types stay internal until that integration is designed. Also broaden the real-seed mutation property suite (#6) with the three mutation classes it named but didn't yet cover: invalidControlCharacter, mixedTabSpaceIndentation, and truncatedFlowCollection. The test now asserts diagnostic quality, not just pass/fail: line must fall within the mutated document, and column, when present, must be 1-based (some ParseError cases, like tabIndentation, are line-only by design).
206 lines
9.1 KiB
Swift
206 lines
9.1 KiB
Swift
extension PureYAML.Parsing.LibYAML.Scanner {
|
|
private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
|
|
|
|
/// The accumulation state for `scan_block_scalar`.
|
|
private struct BlockState {
|
|
var string = ""
|
|
var leadingBreak = ""
|
|
var trailingBreaks = ""
|
|
var leadingBlank = false
|
|
}
|
|
|
|
/// Port of `yaml_parser_scan_block_scalar`. Scans a literal (`|`) or folded
|
|
/// (`>`) block scalar: parses the chomping (`+`/`-`) and indentation indicators
|
|
/// from the header, auto-detects the indentation if not given, then folds the
|
|
/// content (literal keeps breaks, folded joins equally-indented non-empty lines
|
|
/// with a space) and applies clip/strip/keep chomping to the trailing breaks.
|
|
mutating func scanBlockScalar(literal: Bool) throws -> PureYAML.Parsing.LibYAML.Token {
|
|
let startMark = reader.mark
|
|
reader.skip()
|
|
let header = try readBlockScalarHeader(startMark: startMark)
|
|
try consumeBlockHeaderTail(startMark: startMark)
|
|
|
|
var endMark = reader.mark
|
|
// -1 is the "indentation not yet auto-detected" sentinel. libyaml uses 0,
|
|
// but a root-level block scalar legitimately detects indentation 0, so 0
|
|
// must be a representable detected value rather than the sentinel.
|
|
var indent = -1
|
|
if header.increment != 0 {
|
|
indent = indentEngine.indent >= 0 ? indentEngine.indent + header.increment : header.increment
|
|
}
|
|
var state = BlockState()
|
|
try scanBlockScalarBreaks(indent: &indent, state: &state, startMark: startMark, endMark: &endMark)
|
|
|
|
while reader.column == indent, reader.peek() != nil {
|
|
// A root block scalar (indent 0) ends at a `---`/`...` document marker
|
|
// in column 0. libyaml never needs this because it floors the indent
|
|
// at 1, but indent 0 lets the loop reach a column-0 marker line.
|
|
if reader.column == 0, documentMarkerAhead() {
|
|
break
|
|
}
|
|
foldBlockLine(&state, literal: literal)
|
|
while let character = reader.peek(), !Classify.isBreakZ(character) {
|
|
state.string.unicodeScalars.append(character)
|
|
reader.skip()
|
|
}
|
|
state.leadingBreak += readBlockLine()
|
|
try scanBlockScalarBreaks(indent: &indent, state: &state, startMark: startMark, endMark: &endMark)
|
|
}
|
|
|
|
if header.chomping != -1 {
|
|
state.string += state.leadingBreak
|
|
}
|
|
if header.chomping == 1 {
|
|
state.string += state.trailingBreaks
|
|
}
|
|
return PureYAML.Parsing.LibYAML.Token(
|
|
type: .scalar(value: state.string, style: literal ? .literal : .folded),
|
|
startMark: startMark,
|
|
endMark: endMark,
|
|
)
|
|
}
|
|
|
|
/// Parse the chomping and indentation indicators in either order.
|
|
private mutating func readBlockScalarHeader(startMark: PureYAML.Parsing.Mark) throws -> (chomping: Int, increment: Int) {
|
|
var chomping = 0
|
|
var increment = 0
|
|
if reader.peek() == "+" || reader.peek() == "-" {
|
|
chomping = reader.peek() == "+" ? 1 : -1
|
|
reader.skip()
|
|
if Classify.isDigit(reader.peek()) {
|
|
increment = try blockIncrement(startMark: startMark)
|
|
}
|
|
} else if Classify.isDigit(reader.peek()) {
|
|
increment = try blockIncrement(startMark: startMark)
|
|
if reader.peek() == "+" || reader.peek() == "-" {
|
|
chomping = reader.peek() == "+" ? 1 : -1
|
|
reader.skip()
|
|
}
|
|
}
|
|
return (chomping, increment)
|
|
}
|
|
|
|
private mutating func blockIncrement(startMark: PureYAML.Parsing.Mark) throws -> Int {
|
|
if reader.peek() == "0" {
|
|
throw blockError("found an indentation indicator equal to 0", startMark)
|
|
}
|
|
let value = digitValue(reader.peek())
|
|
reader.skip()
|
|
return value
|
|
}
|
|
|
|
/// Eat trailing blanks and a comment to the end of the header line, then the
|
|
/// line break.
|
|
private mutating func consumeBlockHeaderTail(startMark: PureYAML.Parsing.Mark) throws {
|
|
while reader.peek() == " " {
|
|
reader.skip()
|
|
}
|
|
if reader.peek() == "#" {
|
|
// A comment needs a preceding space, so `>#` (a `#` adjacent to the
|
|
// block scalar indicator) is malformed, not a comment. libyaml accepts it.
|
|
if reader.column > 0, let previous = reader.peek(-1), !Classify.isBlank(previous) {
|
|
throw blockError("found a comment without a preceding space", startMark)
|
|
}
|
|
while !Classify.isBreakZ(reader.peek()) {
|
|
reader.skip()
|
|
}
|
|
}
|
|
guard Classify.isBreakZ(reader.peek()) else {
|
|
throw blockError("did not find expected comment or line break", startMark)
|
|
}
|
|
if Classify.isBreak(reader.peek()) {
|
|
reader.skipLine()
|
|
}
|
|
}
|
|
|
|
/// Port of `yaml_parser_scan_block_scalar_breaks`: eat indentation spaces and
|
|
/// blank lines, auto-detecting the indentation level when none was given.
|
|
private mutating func scanBlockScalarBreaks(indent: inout Int, state: inout BlockState, startMark: PureYAML.Parsing.Mark, endMark: inout PureYAML.Parsing.Mark) throws {
|
|
var maxIndent = 0
|
|
endMark = reader.mark
|
|
// A tab is content (or separation) once the column reaches the content
|
|
// indentation; it is illegal only while still inside the required
|
|
// indentation. During auto-detection the floor is the parent indent + 1.
|
|
let tabFloor = indent >= 0 ? indent : max(0, indentEngine.indent + 1)
|
|
while true {
|
|
let lineStart = reader.column
|
|
while indent < 0 || reader.column < indent, reader.peek() == " " {
|
|
reader.skip()
|
|
}
|
|
if reader.column > maxIndent {
|
|
maxIndent = reader.column
|
|
}
|
|
if reader.column < tabFloor, Classify.isTab(reader.peek()) {
|
|
throw blockError("found a tab character where an indentation space is expected", startMark)
|
|
}
|
|
if !Classify.isBreak(reader.peek()) {
|
|
// A trailing whitespace-only line that ends at end of input without
|
|
// a break still counts as one line break for keep chomping (as the
|
|
// test-suite reference does), even with no break byte present.
|
|
if reader.peek() == nil, reader.column > lineStart {
|
|
state.trailingBreaks += "\n"
|
|
}
|
|
break
|
|
}
|
|
state.trailingBreaks += readBlockLine()
|
|
endMark = reader.mark
|
|
}
|
|
// A leading empty line may not be more indented than the first content
|
|
// line; libyaml silently detects the deeper indent and drops the content.
|
|
if indent < 0, !Classify.isBreakZ(reader.peek()), reader.column < maxIndent {
|
|
throw blockError("found a leading empty line more indented than the content", startMark)
|
|
}
|
|
if indent < 0 {
|
|
// Auto-detected indentation. A root-level block scalar has a parent
|
|
// indent of -1, so the floor is 0 (allowing column-0 content), not
|
|
// libyaml's 1. maxIndent >= 0 keeps the result non-negative.
|
|
indent = max(maxIndent, indentEngine.indent + 1)
|
|
}
|
|
}
|
|
|
|
/// Fold the pending breaks at the start of a content line (folded style joins a
|
|
/// single break between non-blank lines into a space; otherwise breaks are
|
|
/// kept).
|
|
private mutating func foldBlockLine(_ state: inout BlockState, literal: Bool) {
|
|
let trailingBlank = Classify.isBlank(reader.peek())
|
|
if !literal, state.leadingBreak.first == "\n", !state.leadingBlank, !trailingBlank {
|
|
if state.trailingBreaks.isEmpty {
|
|
state.string += " "
|
|
}
|
|
state.leadingBreak = ""
|
|
} else {
|
|
state.string += state.leadingBreak
|
|
state.leadingBreak = ""
|
|
}
|
|
state.string += state.trailingBreaks
|
|
state.trailingBreaks = ""
|
|
state.leadingBlank = Classify.isBlank(reader.peek())
|
|
}
|
|
|
|
private mutating func readBlockLine() -> String {
|
|
reader.skipLine()
|
|
return "\n"
|
|
}
|
|
|
|
/// Whether a `---` or `...` document marker (three repeats then a blank, break,
|
|
/// or end of input) starts at the current position.
|
|
private mutating func documentMarkerAhead() -> Bool {
|
|
let first = reader.peek()
|
|
guard first == "-" || first == "." else {
|
|
return false
|
|
}
|
|
return reader.peek(1) == first && reader.peek(2) == first && Classify.isBlankZ(reader.peek(3))
|
|
}
|
|
|
|
private func digitValue(_ scalar: Unicode.Scalar?) -> Int {
|
|
guard let scalar, scalar >= "0", scalar <= "9" else {
|
|
return 0
|
|
}
|
|
return Int(scalar.value - Unicode.Scalar("0").value)
|
|
}
|
|
|
|
private func blockError(_ problem: String, _ mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError {
|
|
PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a block scalar", problem: problem, mark: mark)
|
|
}
|
|
}
|