PureYAML/Sources/Parsing/Scanner.Helpers.swift
Mihaela M 6c322fc5ed fix(parsing): end a plain scalar at a tab before a comment
A comment opens after white space, a tab as well as a space, but the plain
scalar terminator checked only for a space before `#`. So `c: d\t#X` kept the
tab and hash as content, giving the value `d\t#X` rather than `d` with a
trailing comment.

Treat a tab before `#` as a space there, in both the lookahead from a white
space character and the check on the accumulated value's last character. A tab
inside a plain scalar with no `#` after it stays content, and a `#` glued to
content with no white space is still content.

Raises yaml-test-suite value-level conformance to 187/194 (96.4%) with the
parse-level and should-error rejection rates unchanged. 448 tests pass.
2026-06-28 21:24:59 +02:00

343 lines
13 KiB
Swift

extension PureYAML.Parsing.Scanner {
/// A blank line inside a block scalar is preserved as empty content so that
/// chomping and folding can apply to it. The guard keeps blank lines outside
/// block scalars unaffected; `isAtLineStart` is true here only when the
/// previous line already ended, so a content line's own terminating break
/// does not emit a blank. Blank lines between the header and the first
/// content line precede the indent token, so they are buffered and emitted
/// once the content indentation is established.
func captureBlockScalarBlankLine(_ state: inout State) {
guard state.isAtLineStart else {
return
}
if state.blockScalarIndentation != nil {
// The prefix holds any surplus indentation past the block's content level for an
// otherwise-empty line (see `preserveBlockScalarSurplusOnBlankLine`); empty otherwise.
state.append(.scalar(value: state.blockScalarContentPrefix, style: .plain))
state.blockScalarContentPrefix = ""
} else if state.pendingBlockScalarIndentation {
state.pendingBlockScalarLeadingBlankLines += 1
}
}
func startsWith(
_ prefix: String,
_ reader: PureYAML.Parsing.Reader,
) -> Bool {
for (offset, character) in prefix.enumerated() {
guard reader.peek(offset: offset) == character else {
return false
}
}
return true
}
func isLineBreak(_ character: Character?) -> Bool {
character == "\n" || character == "\r" || character == "\r\n"
}
func isBlockEntry(_ reader: PureYAML.Parsing.Reader) -> Bool {
reader.peek() == "-" && isIndicatorBoundary(reader.peek(offset: 1))
}
func isIndicatorBoundary(_ character: Character?) -> Bool {
guard let character else {
return true
}
// A tab is separation white space in YAML (only indentation forbids it), so it ends an
// indicator just as a space does: `---\tscalar`, `-\titem`, `?\tkey`.
return character == " " || character == "\t" || isLineBreak(character)
}
func isMappingValueBoundary(_ character: Character?) -> Bool {
guard let character else {
return true
}
// A tab is separation white space, so it follows a `:` value indicator as a space does.
return character == " "
|| character == "\t"
|| isLineBreak(character)
|| character == ","
|| character == "]"
|| character == "}"
}
func isBlockMappingValueBoundary(_ character: Character?) -> Bool {
guard let character else {
return true
}
return character == " " || character == "\t" || isLineBreak(character)
}
func isTokenTerminator(_ character: Character) -> Bool {
character == " "
|| isLineBreak(character)
|| character == ","
|| character == "["
|| character == "]"
|| character == "{"
|| character == "}"
|| character == "#"
}
func canStartAnchorName(_ character: Character?) -> Bool {
guard let character else {
return false
}
return !isTokenTerminator(character)
&& character != "*"
&& character != "&"
&& character != "!"
&& character != "\""
&& character != "'"
}
func hasNodeOnSameLine(afterCurrentPositionIn reader: PureYAML.Parsing.Reader) -> Bool {
var offset = 0
while reader.peek(offset: offset) == " " {
offset += 1
}
guard let character = reader.peek(offset: offset) else {
return false
}
return !isLineBreak(character) && character != "#"
}
func isFlowDelimiter(_ character: Character) -> Bool {
character == ","
|| character == "["
|| character == "]"
|| character == "{"
|| character == "}"
}
func isBlockScalarHeaderCharacter(_ character: Character) -> Bool {
character == "+"
|| character == "-"
|| character.isNumber
}
func trimTrailingSpaces(_ value: String) -> String {
var end = value.endIndex
while end > value.startIndex {
let previous = value.index(before: end)
guard value[previous] == " " else {
break
}
end = previous
}
return String(value[..<end])
}
func scanPlainScalarContinuations(
after firstLine: String,
firstLineIndent: Int,
state: inout State,
) -> String {
var value = firstLine
while true {
var probe = state.reader
guard isLineBreak(probe.peek()) else {
return value
}
probe.advance()
let blankLineCount = consumeBlankLines(&probe)
_ = probe.consume { character in
character == " "
}
let indentationWidth = probe.mark.column - 1
guard !probe.isAtEnd, !isLineBreak(probe.peek()), probe.peek() != "#" else {
return value
}
// A plain scalar never spans a document marker (`---` / `...`); stop the continuation at
// one so it begins (or ends) the document rather than folding into the scalar's value.
if indentationWidth == 0, isAtDocumentMarker(probe) {
return value
}
let baseIndentation = plainScalarContinuationBaseIndentation(state: state, firstLineIndent: firstLineIndent)
guard indentationWidth > baseIndentation else {
return value
}
if isBlockEntry(probe), state.validImplicitIndentation.contains(indentationWidth) {
return value
}
var continuationState = state
continuationState.reader = probe
let continuation = scanPlainScalarContinuationLine(&continuationState)
guard !isAtMappingValueIndicator(state: continuationState) else {
return value
}
state = continuationState
if blankLineCount == 0 {
value += " "
} else {
value += String(repeating: "\n", count: blankLineCount)
}
value += continuation
}
}
/// Advance `probe` over a run of blank lines (lines that are empty or only spaces), returning
/// how many were consumed; the probe is left at the start of the next non-blank line.
func consumeBlankLines(_ probe: inout PureYAML.Parsing.Reader) -> Int {
var count = 0
while true {
var blankProbe = probe
// A line of only white space is empty for folding, including one that holds a tab: a tab
// is not indentation, but a whitespace-only line is still blank, so consume it here.
_ = blankProbe.consume { character in
character == " " || character == "\t"
}
guard isLineBreak(blankProbe.peek()) else {
break
}
blankProbe.advance()
probe = blankProbe
count += 1
}
return count
}
/// Whether the reader is positioned at a document marker (`---` or `...`) followed by an
/// indicator boundary, i.e. the marker that starts or ends a document.
func isAtDocumentMarker(_ reader: PureYAML.Parsing.Reader) -> Bool {
(startsWith("---", reader) || startsWith("...", reader)) && isIndicatorBoundary(reader.peek(offset: 3))
}
func plainScalarContinuationBaseIndentation(state: State, firstLineIndent: Int) -> Int {
// When the scalar begins exactly at the current indentation top, it is a block value on its
// own line, so its own indent was pushed onto the stack; its continuation lines sit at that
// same indent, and the level they must exceed is the enclosing block's: the indent below on
// the stack, or -1 at the document root, where nothing encloses the scalar and a
// continuation may sit at column 1. For an inline value the top of the stack already is the
// enclosing block's indent, so it is used directly.
if state.indentation.last == firstLineIndent {
return state.indentation.count >= 2 ? state.indentation[state.indentation.count - 2] : -1
}
return state.indentation.last ?? 0
}
func scanPlainScalarContinuationLine(_ state: inout State) -> String {
// Leading white space beyond the line's indentation folds away in a plain scalar. A tab is
// not indentation, so the indentation scan leaves it here; skip it (and any spaces after it)
// so a continuation line like `\t3rd` joins as `3rd`, not `\t3rd`.
while let character = state.reader.peek(), character == " " || character == "\t" {
state.reader.advance()
}
var value = ""
while let character = state.reader.peek(), !isLineBreak(character) {
if character == " ", state.reader.peek(offset: 1) == "#" {
break
}
if character == "#", value.isEmpty || value.last == " " {
break
}
if character == ":", shouldEndPlainScalarAtMappingValue(state: state) {
break
}
if state.flowDepth > 0, isFlowDelimiter(character) {
break
}
value.append(character)
state.reader.advance()
}
return trimTrailingSpaces(value)
}
func skipLine(_ state: inout State) {
_ = state.reader.consume { character in
!isLineBreak(character)
}
}
func expandTag(
_ tag: String,
state: State,
) -> String {
let handles = state.tagHandles.keys.sorted { first, second in
first.count > second.count
}
for handle in handles where tag.hasPrefix(handle) {
let suffix = tag.dropFirst(handle.count)
return (state.tagHandles[handle] ?? handle) + suffix
}
return tag
}
func shouldEndPlainScalar(
_ character: Character,
state: State,
accumulated: String,
) -> Bool {
if isLineBreak(character) {
return true
}
// A comment opens after white space, a tab as well as a space, so a `#` preceded by either
// ends the plain scalar: `d\t#X` is the value `d` with a trailing comment.
if character == " " || character == "\t", state.reader.peek(offset: 1) == "#" {
return true
}
if character == "#", accumulated.isEmpty || accumulated.last == " " || accumulated.last == "\t" {
return true
}
if character == ":", shouldEndPlainScalarAtMappingValue(state: state) {
return true
}
if state.flowDepth > 0, isFlowDelimiter(character) {
return true
}
return false
}
func shouldEndPlainScalarAtMappingValue(state: State) -> Bool {
if state.flowDepth > 0 {
return state.isExpectingFlowMappingKey
|| isMappingValueBoundary(state.reader.peek(offset: 1))
}
return isBlockMappingValueBoundary(state.reader.peek(offset: 1))
}
func isAtMappingValueIndicator(state: State) -> Bool {
state.reader.peek() == ":" && shouldEndPlainScalarAtMappingValue(state: state)
}
/// Whether a mapping-value indicator (`: ` or `:` at line end) follows, allowing spaces before
/// the colon. Used after a quoted scalar, where the reader stops at the closing quote and any
/// separating spaces remain, unlike a plain scalar that ends at the colon itself.
func isAtMappingValueIndicatorAfterSpaces(state: State) -> Bool {
var offset = 0
while state.reader.peek(offset: offset) == " " {
offset += 1
}
return state.reader.peek(offset: offset) == ":"
&& isBlockMappingValueBoundary(state.reader.peek(offset: offset + 1))
}
@discardableResult
func appendMappingKeyIfNeeded(
start: PureYAML.Parsing.Mark,
state: inout State,
) -> Bool {
guard isAtMappingValueIndicator(state: state), canSynthesizeImplicitBlockKey(state) else {
return false
}
state.append(.mappingKey, mark: start, endMark: start)
return true
}
/// Whether a scalar followed by `:` may start an implicit mapping key here. In block style a
/// line holds at most one mapping pair, so once a block-level `:` has appeared on the line a
/// further key would form a forbidden inline nested mapping; in flow style several pairs per
/// line are allowed, so the restriction does not apply.
func canSynthesizeImplicitBlockKey(_ state: State) -> Bool {
state.flowDepth > 0 || !state.sawMappingValueOnLine
}
}