PureYAML/Sources/Parsing/Scanner.Indentation.swift
Mihaela M 4b0dc9375e fix(parsing): keep tabs in block scalar content verbatim
A block scalar is indented with spaces only, so a tab past the content
indentation is literal content. The indentation scan measured the leading
whitespace as a tab-expanded width and rebuilt the surplus beyond the
indentation as spaces, so a line like `␣\ttext` under a one-space indent came
out `   text` rather than `\ttext`: the tab became three spaces.

Record the literal leading-whitespace characters during the indentation scan
and reconstruct the surplus from them, walking columns the way the width scan
does and keeping the remainder verbatim. A tab stays a tab; surplus spaces
stay spaces; the blank-line surplus path uses the same reconstruction.

Raises yaml-test-suite value-level conformance to 180/194 (92.8%) with the
parse-level and should-error rejection rates unchanged. 438 tests pass.
2026-06-28 20:53:40 +02:00

265 lines
12 KiB
Swift

extension PureYAML.Parsing.Scanner {
func scanIndentation(_ state: inout State) throws {
let start = state.reader.mark
let width = try scanIndentationWidth(&state)
guard shouldProcessIndentation(state) else {
preserveBlockScalarSurplusOnBlankLine(width: width, state: &state)
recordPendingBlockScalarBlankIndent(width: width, state: &state)
return
}
let end = state.reader.mark
let current = state.indentation.last ?? 0
let wasInBlockScalar = state.blockScalarIndentation != nil
if consumeBlockScalarContentIndentation(width: width, state: &state) {
return
}
// An active block scalar that just ended (its content was less indented this line) onto a
// line beginning with '#' is leaving an ordinary comment. A comment's own indentation does
// not participate in block structure, so leave it for the main loop to scan rather than
// forcing its column to match an indentation level, which would dedent onto a width no stack
// level holds and raise a spurious error. The guard on `wasInBlockScalar` keeps a block
// scalar's own first content line that begins with '#' (e.g. `|-` then `# text`) as content.
if wasInBlockScalar, state.reader.peek() == "#" {
return
}
if let indicator = state.pendingBlockScalarIndentationIndicator {
try applyExplicitBlockScalarIndentation(
indicator: indicator,
width: width,
current: current,
marks: (start: start, end: end),
state: &state,
)
} else if width > current {
try applyIncreasedIndentation(width: width, start: start, end: end, state: &state)
} else if state.pendingBlockScalarIndentation, state.pendingBlockScalarIsRootNode, width == current {
// A root block scalar (`>`/`|` as the whole document) has the document node as its
// parent, whose indentation is -1, so content at column 0 (width == current == 0) is
// still more indented than its parent and must establish the block scalar's content
// indentation here, where `width > current` (with the base level read as 0) cannot fire.
// A non-root block scalar (a sequence item or mapping value) keeps its parent's indent as
// the floor, so its under-indented content still falls through to an error, unchanged.
try applyIncreasedIndentation(width: width, start: start, end: end, state: &state)
} else if width < current {
try applyDedentation(width: width, start: start, end: end, state: &state)
}
pruneImplicitIndentation(&state)
clearPendingBlockScalarIndentation(&state)
}
/// Records a leading blank line's indentation while a block scalar's content indentation is
/// still pending (before its first content line), so an over-indented leading line can be
/// caught once the content indentation is established.
func recordPendingBlockScalarBlankIndent(width _: Int, state: inout State) {
guard state.pendingBlockScalarIndentation, state.blockScalarIndentation == nil else {
return
}
// Compare leading spaces, not the tab-expanded width: a tab on a blank line is not
// indentation, so `␣\t` counts as one space, not a jump to the next tab stop.
state.pendingBlockScalarMaxBlankIndent = max(state.pendingBlockScalarMaxBlankIndent, state.lastLeadingSpaceCount)
}
/// A block scalar's leading empty lines must not be more indented than its content, so reject
/// the scalar if the widest leading blank exceeds the content indentation.
func enforceLeadingBlankIndentation(contentIndent: Int, line: Int, state: State) throws {
guard state.pendingBlockScalarMaxBlankIndent > contentIndent else {
return
}
throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: line)
}
/// A whitespace-only line inside an active block scalar that is indented past the block's
/// content indentation carries the surplus spaces as content (e.g. a 3-space line in a 2-space
/// block contributes `" "`). The width is consumed before the line is recognized as blank, so
/// stash the surplus in the content prefix for the blank-line emission to pick up; otherwise the
/// line collapses to empty and a space is dropped from the scalar's value.
func preserveBlockScalarSurplusOnBlankLine(width: Int, state: inout State) {
guard let blockScalarIndentation = state.blockScalarIndentation, width > blockScalarIndentation else {
return
}
state.blockScalarContentPrefix = blockScalarSurplusPrefix(beyond: blockScalarIndentation, in: state)
}
func scanIndentationWidth(_ state: inout State) throws -> Int {
var width = 0
var leadingSpaces = 0
var sawTab = false
var whitespace = ""
while let character = state.reader.peek() {
if character == " " {
width += 1
if !sawTab {
leadingSpaces += 1
}
whitespace.append(character)
state.reader.advance()
} else if character == "\t" {
sawTab = true
width = nextTabStop(after: width)
whitespace.append(character)
state.reader.advance()
} else {
break
}
}
state.lastLeadingSpaceCount = leadingSpaces
state.lastLeadingWhitespace = whitespace
return width
}
func nextTabStop(after width: Int) -> Int {
((width / 4) + 1) * 4
}
/// The leading-whitespace characters that lie beyond a block scalar's space-only `indent`, kept
/// verbatim so a tab in the content (` \ttext` under a 2-space indent) stays a tab rather than
/// being re-expanded to spaces. Walks the recorded whitespace, counting columns the way the width
/// scan does, and returns the remainder once the indentation columns are covered.
func blockScalarSurplusPrefix(beyond indent: Int, in state: State) -> String {
var column = 0
var index = state.lastLeadingWhitespace.startIndex
while index < state.lastLeadingWhitespace.endIndex, column < indent {
let character = state.lastLeadingWhitespace[index]
column = character == "\t" ? nextTabStop(after: column) : column + 1
index = state.lastLeadingWhitespace.index(after: index)
}
return String(state.lastLeadingWhitespace[index...])
}
func shouldProcessIndentation(_ state: State) -> Bool {
guard !state.reader.isAtEnd, !isLineBreak(state.reader.peek()) else {
return false
}
// A leading '#' normally starts a comment line, which must not drive
// indent/dedent processing. Inside a block scalar it is different: when
// the header is still pending or the content is active, a leading '#' is
// literal scalar content, so indentation must be processed so the block
// scalar's indentation resolves and its content lines are consumed.
// Bailing here would leave `blockScalarIndentation` unset and the main
// loop would mis-scan the line as a comment, dropping the content.
if state.reader.peek() == "#", !state.pendingBlockScalarIndentation, state.blockScalarIndentation == nil {
return false
}
return state.flowDepth == 0
}
func consumeBlockScalarContentIndentation(
width: Int,
state: inout State,
) -> Bool {
guard let blockScalarIndentation = state.blockScalarIndentation else {
return false
}
guard width < blockScalarIndentation else {
state.blockScalarContentPrefix = blockScalarSurplusPrefix(beyond: blockScalarIndentation, in: state)
return true
}
state.blockScalarIndentation = nil
state.blockScalarContentPrefix = ""
return false
}
func applyExplicitBlockScalarIndentation(
indicator: Int,
width: Int,
current: Int,
marks: (start: PureYAML.Parsing.Mark, end: PureYAML.Parsing.Mark),
state: inout State,
) throws {
let blockScalarIndentation = current + indicator
guard width >= blockScalarIndentation else {
throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: marks.end.line)
}
try enforceLeadingBlankIndentation(contentIndent: blockScalarIndentation, line: marks.end.line, state: state)
state.indentation.append(blockScalarIndentation)
state.append(.indent(width: blockScalarIndentation), mark: marks.start, endMark: marks.end)
state.blockScalarIndentation = blockScalarIndentation
state.blockScalarContentPrefix = String(repeating: " ", count: width - blockScalarIndentation)
emitBufferedBlockScalarLeadingBlanks(&state)
clearPendingBlockScalarIndentation(&state)
}
/// Leading blank lines (between the header and the first content line) are
/// buffered until the indent token is emitted, then flushed as empty content
/// so they land after the indent and before the first content line.
func emitBufferedBlockScalarLeadingBlanks(_ state: inout State) {
for _ in 0 ..< state.pendingBlockScalarLeadingBlankLines {
state.append(.scalar(value: "", style: .plain))
}
state.pendingBlockScalarLeadingBlankLines = 0
}
func applyIncreasedIndentation(
width: Int,
start: PureYAML.Parsing.Mark,
end: PureYAML.Parsing.Mark,
state: inout State,
) throws {
guard state.pendingBlockScalarIndentation else {
state.indentation.append(width)
state.append(.indent(width: width), mark: start, endMark: end)
return
}
try enforceLeadingBlankIndentation(contentIndent: width, line: end.line, state: state)
state.indentation.append(width)
state.append(.indent(width: width), mark: start, endMark: end)
state.blockScalarIndentation = width
state.blockScalarContentPrefix = ""
emitBufferedBlockScalarLeadingBlanks(&state)
clearPendingBlockScalarIndentation(&state)
}
func applyDedentation(
width: Int,
start: PureYAML.Parsing.Mark,
end: PureYAML.Parsing.Mark,
state: inout State,
) throws {
while state.indentation.count > 1, width < (state.indentation.last ?? 0) {
state.indentation.removeLast()
state.append(.dedent(width: state.indentation.last ?? 0), mark: end, endMark: end)
}
if canUseImplicitIndentation(width: width, state: state) {
state.indentation.append(width)
state.append(.indent(width: width), mark: start, endMark: end)
}
guard width == state.indentation.last else {
throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: end.line)
}
}
func canUseImplicitIndentation(
width: Int,
state: State,
) -> Bool {
width != state.indentation.last
&& width > (state.indentation.last ?? 0)
&& state.validImplicitIndentation.contains(width)
}
func pruneImplicitIndentation(_ state: inout State) {
let current = state.indentation.last ?? 0
state.validImplicitIndentation = state.validImplicitIndentation.filter { $0 <= current }
}
func clearPendingBlockScalarIndentation(_ state: inout State) {
state.pendingBlockScalarIndentation = false
state.pendingBlockScalarIndentationIndicator = nil
state.pendingBlockScalarLeadingBlankLines = 0
state.pendingBlockScalarMaxBlankIndent = 0
state.pendingBlockScalarIsRootNode = false
}
func closeIndentation(_ state: inout State) {
let mark = state.reader.mark
while state.indentation.count > 1 {
state.indentation.removeLast()
state.append(.dedent(width: state.indentation.last ?? 0), mark: mark, endMark: mark)
}
}
}