PureYAML/Sources/Parsing/Scanner.swift
Mihaela Mihaljevic 1875d66d46 fix(parsing): parse multi-line and block scalars at the document root
Model libyaml's parser->indent (the enclosing block-collection column, -1 at the bare root) separately from per-line indentation, via a blockIndent stack that rolls on block-sequence entries and mapping keys, unrolls at each block-line start, and resets at document boundaries. Plain-scalar continuation folding now compares against blockIndent, so a root scalar folds its column-0 continuations (single break to space, blank line to newline, per-line whitespace stripped) while a scalar inside a collection still ends at a sibling line. The ':' value indicator deliberately does not roll the block indent.

A '---' document-start marker may now be followed by the document's root node on the same line ('--- foo', '--- [1, 2]', '--- |', '--- >'), and a block scalar whose body sits at the document root's column 0 is captured and ends at the next '---'/'...' marker.

Flips 33 official YAML test suite cases (338/402): 18 false-rejects and 12 value mismatches now match, and 3 invalid inputs (9KBC, ZVH3, 9MQT/01) are now correctly rejected. M7A3 and NB6Z now parse but with mismatched values (separate multi-doc and tab-folding gaps) and stay tracked. Adds root multi-line plain scalar regression tests; updates manifest, fixtures, counts, and roadmap.
2026-06-23 22:56:33 +02:00

374 lines
14 KiB
Swift

extension PureYAML.Parsing {
struct Scanner {
init() {}
func scan(_ yaml: String) throws -> [Token] {
var state = State(reader: Reader(yaml))
state.append(.streamStart)
while !state.reader.isAtEnd {
if isLineBreak(state.reader.peek()) {
state.reader.advance()
state.isAtLineStart = true
if state.flowDepth == 0 {
state.simpleKeyAllowed = true
}
continue
}
if state.isAtLineStart {
try scanIndentation(&state)
if state.reader.isAtEnd {
break
}
if isLineBreak(state.reader.peek()) {
continue
}
if try scanDirectiveOrDocumentMarker(&state) {
continue
}
state.isAtLineStart = false
}
try scanToken(&state)
}
if state.hasPendingDirective {
throw PureYAML.Parsing.ParseError.directiveWithoutDocument(line: state.reader.mark.line)
}
closeIndentation(&state)
state.append(.streamEnd)
return state.tokens
}
}
}
extension PureYAML.Parsing.Scanner {
struct State {
var reader: PureYAML.Parsing.Reader
var tokens: [PureYAML.Parsing.Token] = []
var indentation: [Int] = [0]
var isAtLineStart = true
var hasDocumentContent = false
var hasDocumentStartMarker = false
var isDocumentClosed = false
/// True once a `%YAML` directive has been scanned for the document that
/// the next `---` will start. A second `%YAML` before that `---` is a
/// duplicate and is rejected; the flag resets at every document boundary.
var sawYAMLDirective = false
/// True once any directive has been scanned in the current directive
/// section. A directive section must be terminated by a `---` document
/// start; reaching a `...` end marker or the end of the stream with a
/// directive still pending is an error.
var hasPendingDirective = false
var flowDepth = 0
/// Mirrors libyaml's `parser->indent`: the 0-based column of the innermost
/// open block collection (sequence or mapping), or -1 at the bare document
/// root. Plain-scalar continuation folding compares a continuation line's
/// indentation against this value, so a root scalar (block indent -1) folds
/// column-0 continuations while a scalar inside a collection only folds lines
/// indented past the collection. Tracked separately from ``indentation`` --
/// which records every line-indentation level -- because libyaml rolls this
/// only on block-collection indicators, not on every indentation increase.
var blockIndent = -1
/// Saved ``blockIndent`` values, mirroring libyaml's `parser->indents` stack,
/// so a dedent restores the enclosing collection's indent.
var blockIndents: [Int] = []
/// Mirrors libyaml's `simple_key_allowed`. In block context a tab may only
/// be consumed as whitespace when a simple key is NOT allowed (i.e. mid-line,
/// after a node has been emitted). It is true at the start of every block
/// line and immediately after a `-`, `?`, or `:` indicator, so a tab in those
/// positions provides block indentation and is rejected. See scanner.c:1948.
var simpleKeyAllowed = true
var flowContexts: [PureYAML.Parsing.ScannerFlowContext] = []
var validImplicitIndentation: Set<Int> = []
var tagHandles = [
"!": "!",
"!!": "tag:yaml.org,2002:",
]
mutating func append(_ kind: PureYAML.Parsing.TokenKind) {
let mark = reader.mark
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: mark))
markDocumentContentIfNeeded(kind)
}
mutating func append(
_ kind: PureYAML.Parsing.TokenKind,
mark: PureYAML.Parsing.Mark,
endMark: PureYAML.Parsing.Mark,
) {
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: endMark))
markDocumentContentIfNeeded(kind)
}
mutating func markDocumentContentIfNeeded(_ kind: PureYAML.Parsing.TokenKind) {
switch kind {
case .alias, .anchor, .blockScalarHeader, .scalar, .tag:
// A node (or one of its properties) has appeared on this line, so a
// simple key is no longer allowed and a following tab is ordinary
// content rather than indentation. Mirrors libyaml setting
// simple_key_allowed = 0 after scanning these tokens.
hasDocumentContent = true
simpleKeyAllowed = false
case .comment, .dedent, .documentEnd, .documentStart, .indent, .streamEnd, .streamStart:
return
default:
hasDocumentContent = true
}
}
var isExpectingFlowMappingKey: Bool {
flowContexts.last?.kind == .mapping
&& flowContexts.last?.isExpectingKey == true
}
}
}
extension PureYAML.Parsing.Scanner {
func scanToken(_ state: inout State) throws {
skipSeparation(&state)
try rejectTabSeparation(&state)
guard let character = state.reader.peek(), !isLineBreak(character) else {
return
}
if try scanLineIndicatorToken(character, state: &state) {
return
}
if scanFlowToken(character, state: &state) {
return
}
if try scanQuotedOrBlockScalarToken(character, state: &state) {
return
}
if try scanNodePropertyToken(character, state: &state) {
return
}
scanPlainScalar(&state)
}
func scanLineIndicatorToken(
_ character: Character,
state: inout State,
) throws -> Bool {
switch character {
case "#":
scanComment(&state)
return true
case "-" where isBlockEntry(state.reader):
scanFixedToken(.blockEntry, state: &state)
if state.flowDepth == 0 {
state.simpleKeyAllowed = true
}
return true
case "?" where isIndicatorBoundary(state.reader.peek(offset: 1)):
scanFixedToken(.mappingKey, state: &state)
if state.flowDepth == 0 {
state.simpleKeyAllowed = true
}
return true
case ":" where isAtMappingValueIndicator(state: state):
scanFixedToken(.mappingValue, state: &state)
if state.flowDepth == 0 {
state.simpleKeyAllowed = true
}
return true
default:
return false
}
}
func scanFlowToken(
_ character: Character,
state: inout State,
) -> Bool {
switch character {
case "[":
state.flowDepth += 1
state.flowContexts.append(.init(kind: .sequence, isExpectingKey: false))
scanFixedToken(.flowSequenceStart, state: &state)
return true
case "]":
state.flowDepth = max(0, state.flowDepth - 1)
scanFixedToken(.flowSequenceEnd, state: &state)
_ = state.flowContexts.popLast()
return true
case "{":
state.flowDepth += 1
state.flowContexts.append(.init(kind: .mapping, isExpectingKey: true))
scanFixedToken(.flowMappingStart, state: &state)
return true
case "}":
state.flowDepth = max(0, state.flowDepth - 1)
scanFixedToken(.flowMappingEnd, state: &state)
_ = state.flowContexts.popLast()
return true
case ",":
scanFixedToken(.flowEntry, state: &state)
if state.flowContexts.last?.kind == .mapping {
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = true
}
return true
default:
return false
}
}
func scanNodePropertyToken(
_ character: Character,
state: inout State,
) throws -> Bool {
switch character {
case "&":
try scanNamedToken(
kind: PureYAML.Parsing.TokenKind.anchor,
missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName,
state: &state,
)
return true
case "*":
try scanNamedToken(
kind: PureYAML.Parsing.TokenKind.alias,
missingNameError: PureYAML.Parsing.ParseError.expectedAliasName,
state: &state,
)
return true
case "!":
try scanTag(&state)
return true
default:
return false
}
}
func scanQuotedOrBlockScalarToken(
_ character: Character,
state: inout State,
) throws -> Bool {
switch character {
case "\"":
try scanQuotedScalar(.doubleQuoted, state: &state)
return true
case "'":
try scanQuotedScalar(.singleQuoted, state: &state)
return true
case "|":
try scanBlockScalarHeader(.literal, state: &state)
return true
case ">":
try scanBlockScalarHeader(.folded, state: &state)
return true
default:
return false
}
}
func skipSeparation(_ state: inout State) {
while state.reader.peek() == " " {
state.reader.advance()
}
}
func scanFixedToken(
_ kind: PureYAML.Parsing.TokenKind,
state: inout State,
) {
let start = state.reader.mark
// Mirror libyaml rolling the block indent when a block sequence entry (`-`)
// or an explicit `?` mapping key opens a collection at this column. The
// implicit-key case rolls in `appendMappingKeyIfNeeded` at the key column;
// the `:` value indicator does not roll again, because for a simple key the
// mapping indent was already set at the key's (shallower) column and rolling
// to the deeper `:` column would wrongly raise the continuation threshold.
switch kind {
case .blockEntry, .mappingKey:
rollBlockIndent(toColumn: start.column - 1, state: &state)
default:
break
}
state.reader.advance()
state.append(kind, mark: start, endMark: state.reader.mark)
if case .blockEntry = kind, hasNodeOnSameLine(afterCurrentPositionIn: state.reader) {
state.validImplicitIndentation.insert(start.column + 1)
}
if case .mappingValue = kind, state.flowContexts.last?.kind == .mapping {
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = false
}
}
func scanComment(_ state: inout State) {
let start = state.reader.mark
state.reader.advance()
if state.reader.peek() == " " {
state.reader.advance()
}
let value = state.reader.consume { character in
!isLineBreak(character)
}
state.append(.comment(value), mark: start, endMark: state.reader.mark)
}
func scanQuotedScalar(
_ style: PureYAML.Parsing.ScalarStyle,
state: inout State,
) throws {
let start = state.reader.mark
let quote: Character = style == .doubleQuoted ? "\"" : "'"
state.reader.advance()
var value = ""
var previousWasEscape = false
while let character = state.reader.peek() {
// A `---` or `...` document marker at column 0 cannot appear inside an
// open quoted scalar: it means the scalar was never closed before a
// document boundary (scanner.c:3058-3070). Reject it as unterminated.
if state.reader.mark.column == 1, isDocumentMarkerHere(state) {
throw PureYAML.Parsing.ParseError.unterminatedQuotedString(line: start.line)
}
if style == .singleQuoted, character == quote, state.reader.peek(offset: 1) == quote {
value.append(character)
state.reader.advance()
value.append(character)
state.reader.advance()
continue
}
if character == quote, !previousWasEscape {
break
}
value.append(character)
previousWasEscape = character == "\\" && style == .doubleQuoted && !previousWasEscape
state.reader.advance()
if character != "\\" {
previousWasEscape = false
}
}
guard state.reader.peek() == quote else {
throw PureYAML.Parsing.ParseError.unterminatedQuotedString(line: start.line)
}
state.reader.advance()
appendMappingKeyIfNeeded(start: start, state: &state)
state.append(.scalar(value: value, style: style), mark: start, endMark: state.reader.mark)
}
func scanPlainScalar(_ state: inout State) {
let start = state.reader.mark
var value = ""
while let character = state.reader.peek() {
if shouldEndPlainScalar(character, state: state, accumulated: value) {
break
}
value.append(character)
state.reader.advance()
}
var trimmed = trimTrailingSpaces(value)
let isMappingKey = appendMappingKeyIfNeeded(start: start, state: &state)
if !isMappingKey {
trimmed = scanPlainScalarContinuations(
after: trimmed,
state: &state,
)
}
state.append(.scalar(value: trimmed, style: .plain), mark: start, endMark: state.reader.mark)
}
}