feat(parsing): port libyaml block-scalar scanner to Swift

Stage 3 of the faithful libyaml scanner port, continued. Translates fetch_block_scalar, scan_block_scalar, and scan_block_scalar_breaks: literal (|) and folded (>) block scalars with the chomping (+/-) and indentation indicators in either order, indentation auto-detection from the first non-empty line (clamped to parser indent + 1), folded joining of equally-indented non-empty lines into a space, and clip/strip/keep chomping of the trailing breaks. Unit-tested for literal clip, folded join, and strip chomping. Only tags and directives remain unported in the scanner. 314 tests, no regression.
This commit is contained in:
Mihaela Mihaljevic 2026-06-24 16:40:18 +02:00
parent 526abe9998
commit 7918eefe14
3 changed files with 236 additions and 5 deletions

View File

@ -0,0 +1,165 @@
extension PureYAML.Parsing.LibYAML.Scanner {
private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
/// The accumulation state for `scan_block_scalar`.
private struct BlockState {
var string = ""
var leadingBreak = ""
var trailingBreaks = ""
var leadingBlank = false
}
/// Port of `yaml_parser_scan_block_scalar`. Scans a literal (`|`) or folded
/// (`>`) block scalar: parses the chomping (`+`/`-`) and indentation indicators
/// from the header, auto-detects the indentation if not given, then folds the
/// content (literal keeps breaks, folded joins equally-indented non-empty lines
/// with a space) and applies clip/strip/keep chomping to the trailing breaks.
mutating func scanBlockScalar(literal: Bool) throws -> PureYAML.Parsing.LibYAML.Token {
let startMark = reader.mark
reader.skip()
let header = try readBlockScalarHeader(startMark: startMark)
try consumeBlockHeaderTail(startMark: startMark)
var endMark = reader.mark
var indent = 0
if header.increment != 0 {
indent = indentEngine.indent >= 0 ? indentEngine.indent + header.increment : header.increment
}
var state = BlockState()
try scanBlockScalarBreaks(indent: &indent, state: &state, startMark: startMark, endMark: &endMark)
while reader.column == indent, reader.peek() != nil {
foldBlockLine(&state, literal: literal)
while let character = reader.peek(), !Classify.isBreakZ(character) {
state.string.unicodeScalars.append(character)
reader.skip()
}
state.leadingBreak += readBlockLine()
try scanBlockScalarBreaks(indent: &indent, state: &state, startMark: startMark, endMark: &endMark)
}
if header.chomping != -1 {
state.string += state.leadingBreak
}
if header.chomping == 1 {
state.string += state.trailingBreaks
}
return PureYAML.Parsing.LibYAML.Token(
type: .scalar(value: state.string, style: literal ? .literal : .folded),
startMark: startMark,
endMark: endMark,
)
}
/// Parse the chomping and indentation indicators in either order.
private mutating func readBlockScalarHeader(startMark: PureYAML.Parsing.Mark) throws -> (chomping: Int, increment: Int) {
var chomping = 0
var increment = 0
if reader.peek() == "+" || reader.peek() == "-" {
chomping = reader.peek() == "+" ? 1 : -1
reader.skip()
if Classify.isDigit(reader.peek()) {
increment = try blockIncrement(startMark: startMark)
}
} else if Classify.isDigit(reader.peek()) {
increment = try blockIncrement(startMark: startMark)
if reader.peek() == "+" || reader.peek() == "-" {
chomping = reader.peek() == "+" ? 1 : -1
reader.skip()
}
}
return (chomping, increment)
}
private mutating func blockIncrement(startMark: PureYAML.Parsing.Mark) throws -> Int {
if reader.peek() == "0" {
throw blockError("found an indentation indicator equal to 0", startMark)
}
let value = digitValue(reader.peek())
reader.skip()
return value
}
/// Eat trailing blanks and a comment to the end of the header line, then the
/// line break.
private mutating func consumeBlockHeaderTail(startMark: PureYAML.Parsing.Mark) throws {
while reader.peek() == " " {
reader.skip()
}
if reader.peek() == "#" {
while !Classify.isBreakZ(reader.peek()) {
reader.skip()
}
}
guard Classify.isBreakZ(reader.peek()) else {
throw blockError("did not find expected comment or line break", startMark)
}
if Classify.isBreak(reader.peek()) {
reader.skipLine()
}
}
/// Port of `yaml_parser_scan_block_scalar_breaks`: eat indentation spaces and
/// blank lines, auto-detecting the indentation level when none was given.
private mutating func scanBlockScalarBreaks(indent: inout Int, state: inout BlockState, startMark: PureYAML.Parsing.Mark, endMark: inout PureYAML.Parsing.Mark) throws {
var maxIndent = 0
endMark = reader.mark
while true {
while indent == 0 || reader.column < indent, reader.peek() == " " {
reader.skip()
}
if reader.column > maxIndent {
maxIndent = reader.column
}
if indent == 0 || reader.column < indent, Classify.isTab(reader.peek()) {
throw blockError("found a tab character where an indentation space is expected", startMark)
}
if !Classify.isBreak(reader.peek()) {
break
}
state.trailingBreaks += readBlockLine()
endMark = reader.mark
}
if indent == 0 {
indent = max(maxIndent, indentEngine.indent + 1)
if indent < 1 {
indent = 1
}
}
}
/// Fold the pending breaks at the start of a content line (folded style joins a
/// single break between non-blank lines into a space; otherwise breaks are
/// kept).
private func foldBlockLine(_ state: inout BlockState, literal: Bool) {
let trailingBlank = Classify.isBlank(reader.peek())
if !literal, state.leadingBreak.first == "\n", !state.leadingBlank, !trailingBlank {
if state.trailingBreaks.isEmpty {
state.string += " "
}
state.leadingBreak = ""
} else {
state.string += state.leadingBreak
state.leadingBreak = ""
}
state.string += state.trailingBreaks
state.trailingBreaks = ""
state.leadingBlank = Classify.isBlank(reader.peek())
}
private mutating func readBlockLine() -> String {
reader.skipLine()
return "\n"
}
private func digitValue(_ scalar: Unicode.Scalar?) -> Int {
guard let scalar, scalar >= "0", scalar <= "9" else {
return 0
}
return Int(scalar.value - Unicode.Scalar("0").value)
}
private func blockError(_ problem: String, _ mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError {
PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a block scalar", problem: problem, mark: mark)
}
}

View File

@ -90,7 +90,10 @@ extension PureYAML.Parsing.LibYAML.Scanner {
if let unported = unportedConstruct(character) {
throw notPorted(unported)
}
if try fetchIndicatorToken(character) {
if try fetchFlowToken(character) {
return
}
if try fetchNodeToken(character) {
return
}
switch character {
@ -105,24 +108,43 @@ extension PureYAML.Parsing.LibYAML.Scanner {
}
}
/// Fetch a flow-collection indicator, a quoted scalar, or an anchor/alias;
/// returns false if the character starts none of these.
private mutating func fetchIndicatorToken(_ character: Unicode.Scalar) throws -> Bool {
/// Fetch a flow-collection indicator (`[` `]` `{` `}` `,`); returns false
/// otherwise.
private mutating func fetchFlowToken(_ character: Unicode.Scalar) throws -> Bool {
switch character {
case "[": fetchFlowCollectionStart(.flowSequenceStart)
case "{": fetchFlowCollectionStart(.flowMappingStart)
case "]": try fetchFlowCollectionEnd(.flowSequenceEnd)
case "}": try fetchFlowCollectionEnd(.flowMappingEnd)
case ",": try fetchFlowEntry()
default: return false
}
return true
}
/// Fetch a quoted scalar, an anchor/alias, or a block scalar; returns false
/// otherwise.
private mutating func fetchNodeToken(_ character: Unicode.Scalar) throws -> Bool {
switch character {
case "'": try fetchFlowScalar(single: true)
case "\"": try fetchFlowScalar(single: false)
case "&": try fetchAnchor(isAnchor: true)
case "*": try fetchAnchor(isAnchor: false)
case "|" where flowLevel == 0: try fetchBlockScalar(literal: true)
case ">" where flowLevel == 0: try fetchBlockScalar(literal: false)
default: return false
}
return true
}
/// Port of `yaml_parser_fetch_block_scalar`.
private mutating func fetchBlockScalar(literal: Bool) throws {
try removeSimpleKey()
simpleKeyAllowed = true
let token = try scanBlockScalar(literal: literal)
indentEngine.enqueue(token)
}
/// Port of `yaml_parser_fetch_flow_scalar`.
private mutating func fetchFlowScalar(single: Bool) throws {
mark = reader.mark
@ -145,7 +167,6 @@ extension PureYAML.Parsing.LibYAML.Scanner {
private func unportedConstruct(_ character: Unicode.Scalar) -> String? {
switch character {
case "!": "tag"
case "|" where flowLevel == 0, ">" where flowLevel == 0: "block scalar"
default: nil
}
}

View File

@ -159,4 +159,49 @@ struct LibYAMLScannerFetchTests {
.streamEnd,
])
}
@Test("a literal block scalar keeps line breaks and clips the trailing newline")
func test_literalBlockScalar() throws {
let tokens = try tokenize("key: |\n a\n b\n")
#expect(tokens == [
.streamStart,
.blockMappingStart,
.key,
.scalar(value: "key", style: .plain),
.value,
.scalar(value: "a\nb\n", style: .literal),
.blockEnd,
.streamEnd,
])
}
@Test("a folded block scalar joins lines with a space")
func test_foldedBlockScalar() throws {
let tokens = try tokenize("key: >\n a\n b\n")
#expect(tokens == [
.streamStart,
.blockMappingStart,
.key,
.scalar(value: "key", style: .plain),
.value,
.scalar(value: "a b\n", style: .folded),
.blockEnd,
.streamEnd,
])
}
@Test("a strip chomping indicator drops the trailing newline")
func test_literalStripChomping() throws {
let tokens = try tokenize("key: |-\n a\n")
#expect(tokens == [
.streamStart,
.blockMappingStart,
.key,
.scalar(value: "key", style: .plain),
.value,
.scalar(value: "a", style: .literal),
.blockEnd,
.streamEnd,
])
}
}