feat(parsing): port libyaml block-scalar scanner to Swift
Stage 3 of the faithful libyaml scanner port, continued. Translates fetch_block_scalar, scan_block_scalar, and scan_block_scalar_breaks: literal (|) and folded (>) block scalars with the chomping (+/-) and indentation indicators in either order, indentation auto-detection from the first non-empty line (clamped to parser indent + 1), folded joining of equally-indented non-empty lines into a space, and clip/strip/keep chomping of the trailing breaks. Unit-tested for literal clip, folded join, and strip chomping. Only tags and directives remain unported in the scanner. 314 tests, no regression.
This commit is contained in:
parent
526abe9998
commit
7918eefe14
165
Sources/Parsing/LibYAML.Scanner.BlockScalar.swift
Normal file
165
Sources/Parsing/LibYAML.Scanner.BlockScalar.swift
Normal file
@ -0,0 +1,165 @@
|
||||
extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
|
||||
|
||||
/// The accumulation state for `scan_block_scalar`.
|
||||
private struct BlockState {
|
||||
var string = ""
|
||||
var leadingBreak = ""
|
||||
var trailingBreaks = ""
|
||||
var leadingBlank = false
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_scan_block_scalar`. Scans a literal (`|`) or folded
|
||||
/// (`>`) block scalar: parses the chomping (`+`/`-`) and indentation indicators
|
||||
/// from the header, auto-detects the indentation if not given, then folds the
|
||||
/// content (literal keeps breaks, folded joins equally-indented non-empty lines
|
||||
/// with a space) and applies clip/strip/keep chomping to the trailing breaks.
|
||||
mutating func scanBlockScalar(literal: Bool) throws -> PureYAML.Parsing.LibYAML.Token {
|
||||
let startMark = reader.mark
|
||||
reader.skip()
|
||||
let header = try readBlockScalarHeader(startMark: startMark)
|
||||
try consumeBlockHeaderTail(startMark: startMark)
|
||||
|
||||
var endMark = reader.mark
|
||||
var indent = 0
|
||||
if header.increment != 0 {
|
||||
indent = indentEngine.indent >= 0 ? indentEngine.indent + header.increment : header.increment
|
||||
}
|
||||
var state = BlockState()
|
||||
try scanBlockScalarBreaks(indent: &indent, state: &state, startMark: startMark, endMark: &endMark)
|
||||
|
||||
while reader.column == indent, reader.peek() != nil {
|
||||
foldBlockLine(&state, literal: literal)
|
||||
while let character = reader.peek(), !Classify.isBreakZ(character) {
|
||||
state.string.unicodeScalars.append(character)
|
||||
reader.skip()
|
||||
}
|
||||
state.leadingBreak += readBlockLine()
|
||||
try scanBlockScalarBreaks(indent: &indent, state: &state, startMark: startMark, endMark: &endMark)
|
||||
}
|
||||
|
||||
if header.chomping != -1 {
|
||||
state.string += state.leadingBreak
|
||||
}
|
||||
if header.chomping == 1 {
|
||||
state.string += state.trailingBreaks
|
||||
}
|
||||
return PureYAML.Parsing.LibYAML.Token(
|
||||
type: .scalar(value: state.string, style: literal ? .literal : .folded),
|
||||
startMark: startMark,
|
||||
endMark: endMark,
|
||||
)
|
||||
}
|
||||
|
||||
/// Parse the chomping and indentation indicators in either order.
|
||||
private mutating func readBlockScalarHeader(startMark: PureYAML.Parsing.Mark) throws -> (chomping: Int, increment: Int) {
|
||||
var chomping = 0
|
||||
var increment = 0
|
||||
if reader.peek() == "+" || reader.peek() == "-" {
|
||||
chomping = reader.peek() == "+" ? 1 : -1
|
||||
reader.skip()
|
||||
if Classify.isDigit(reader.peek()) {
|
||||
increment = try blockIncrement(startMark: startMark)
|
||||
}
|
||||
} else if Classify.isDigit(reader.peek()) {
|
||||
increment = try blockIncrement(startMark: startMark)
|
||||
if reader.peek() == "+" || reader.peek() == "-" {
|
||||
chomping = reader.peek() == "+" ? 1 : -1
|
||||
reader.skip()
|
||||
}
|
||||
}
|
||||
return (chomping, increment)
|
||||
}
|
||||
|
||||
private mutating func blockIncrement(startMark: PureYAML.Parsing.Mark) throws -> Int {
|
||||
if reader.peek() == "0" {
|
||||
throw blockError("found an indentation indicator equal to 0", startMark)
|
||||
}
|
||||
let value = digitValue(reader.peek())
|
||||
reader.skip()
|
||||
return value
|
||||
}
|
||||
|
||||
/// Eat trailing blanks and a comment to the end of the header line, then the
|
||||
/// line break.
|
||||
private mutating func consumeBlockHeaderTail(startMark: PureYAML.Parsing.Mark) throws {
|
||||
while reader.peek() == " " {
|
||||
reader.skip()
|
||||
}
|
||||
if reader.peek() == "#" {
|
||||
while !Classify.isBreakZ(reader.peek()) {
|
||||
reader.skip()
|
||||
}
|
||||
}
|
||||
guard Classify.isBreakZ(reader.peek()) else {
|
||||
throw blockError("did not find expected comment or line break", startMark)
|
||||
}
|
||||
if Classify.isBreak(reader.peek()) {
|
||||
reader.skipLine()
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_scan_block_scalar_breaks`: eat indentation spaces and
|
||||
/// blank lines, auto-detecting the indentation level when none was given.
|
||||
private mutating func scanBlockScalarBreaks(indent: inout Int, state: inout BlockState, startMark: PureYAML.Parsing.Mark, endMark: inout PureYAML.Parsing.Mark) throws {
|
||||
var maxIndent = 0
|
||||
endMark = reader.mark
|
||||
while true {
|
||||
while indent == 0 || reader.column < indent, reader.peek() == " " {
|
||||
reader.skip()
|
||||
}
|
||||
if reader.column > maxIndent {
|
||||
maxIndent = reader.column
|
||||
}
|
||||
if indent == 0 || reader.column < indent, Classify.isTab(reader.peek()) {
|
||||
throw blockError("found a tab character where an indentation space is expected", startMark)
|
||||
}
|
||||
if !Classify.isBreak(reader.peek()) {
|
||||
break
|
||||
}
|
||||
state.trailingBreaks += readBlockLine()
|
||||
endMark = reader.mark
|
||||
}
|
||||
if indent == 0 {
|
||||
indent = max(maxIndent, indentEngine.indent + 1)
|
||||
if indent < 1 {
|
||||
indent = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fold the pending breaks at the start of a content line (folded style joins a
|
||||
/// single break between non-blank lines into a space; otherwise breaks are
|
||||
/// kept).
|
||||
private func foldBlockLine(_ state: inout BlockState, literal: Bool) {
|
||||
let trailingBlank = Classify.isBlank(reader.peek())
|
||||
if !literal, state.leadingBreak.first == "\n", !state.leadingBlank, !trailingBlank {
|
||||
if state.trailingBreaks.isEmpty {
|
||||
state.string += " "
|
||||
}
|
||||
state.leadingBreak = ""
|
||||
} else {
|
||||
state.string += state.leadingBreak
|
||||
state.leadingBreak = ""
|
||||
}
|
||||
state.string += state.trailingBreaks
|
||||
state.trailingBreaks = ""
|
||||
state.leadingBlank = Classify.isBlank(reader.peek())
|
||||
}
|
||||
|
||||
private mutating func readBlockLine() -> String {
|
||||
reader.skipLine()
|
||||
return "\n"
|
||||
}
|
||||
|
||||
private func digitValue(_ scalar: Unicode.Scalar?) -> Int {
|
||||
guard let scalar, scalar >= "0", scalar <= "9" else {
|
||||
return 0
|
||||
}
|
||||
return Int(scalar.value - Unicode.Scalar("0").value)
|
||||
}
|
||||
|
||||
private func blockError(_ problem: String, _ mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError {
|
||||
PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a block scalar", problem: problem, mark: mark)
|
||||
}
|
||||
}
|
||||
@ -90,7 +90,10 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
if let unported = unportedConstruct(character) {
|
||||
throw notPorted(unported)
|
||||
}
|
||||
if try fetchIndicatorToken(character) {
|
||||
if try fetchFlowToken(character) {
|
||||
return
|
||||
}
|
||||
if try fetchNodeToken(character) {
|
||||
return
|
||||
}
|
||||
switch character {
|
||||
@ -105,24 +108,43 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch a flow-collection indicator, a quoted scalar, or an anchor/alias;
|
||||
/// returns false if the character starts none of these.
|
||||
private mutating func fetchIndicatorToken(_ character: Unicode.Scalar) throws -> Bool {
|
||||
/// Fetch a flow-collection indicator (`[` `]` `{` `}` `,`); returns false
|
||||
/// otherwise.
|
||||
private mutating func fetchFlowToken(_ character: Unicode.Scalar) throws -> Bool {
|
||||
switch character {
|
||||
case "[": fetchFlowCollectionStart(.flowSequenceStart)
|
||||
case "{": fetchFlowCollectionStart(.flowMappingStart)
|
||||
case "]": try fetchFlowCollectionEnd(.flowSequenceEnd)
|
||||
case "}": try fetchFlowCollectionEnd(.flowMappingEnd)
|
||||
case ",": try fetchFlowEntry()
|
||||
default: return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/// Fetch a quoted scalar, an anchor/alias, or a block scalar; returns false
|
||||
/// otherwise.
|
||||
private mutating func fetchNodeToken(_ character: Unicode.Scalar) throws -> Bool {
|
||||
switch character {
|
||||
case "'": try fetchFlowScalar(single: true)
|
||||
case "\"": try fetchFlowScalar(single: false)
|
||||
case "&": try fetchAnchor(isAnchor: true)
|
||||
case "*": try fetchAnchor(isAnchor: false)
|
||||
case "|" where flowLevel == 0: try fetchBlockScalar(literal: true)
|
||||
case ">" where flowLevel == 0: try fetchBlockScalar(literal: false)
|
||||
default: return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_fetch_block_scalar`.
|
||||
private mutating func fetchBlockScalar(literal: Bool) throws {
|
||||
try removeSimpleKey()
|
||||
simpleKeyAllowed = true
|
||||
let token = try scanBlockScalar(literal: literal)
|
||||
indentEngine.enqueue(token)
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_fetch_flow_scalar`.
|
||||
private mutating func fetchFlowScalar(single: Bool) throws {
|
||||
mark = reader.mark
|
||||
@ -145,7 +167,6 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
private func unportedConstruct(_ character: Unicode.Scalar) -> String? {
|
||||
switch character {
|
||||
case "!": "tag"
|
||||
case "|" where flowLevel == 0, ">" where flowLevel == 0: "block scalar"
|
||||
default: nil
|
||||
}
|
||||
}
|
||||
|
||||
@ -159,4 +159,49 @@ struct LibYAMLScannerFetchTests {
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
|
||||
@Test("a literal block scalar keeps line breaks and clips the trailing newline")
|
||||
func test_literalBlockScalar() throws {
|
||||
let tokens = try tokenize("key: |\n a\n b\n")
|
||||
#expect(tokens == [
|
||||
.streamStart,
|
||||
.blockMappingStart,
|
||||
.key,
|
||||
.scalar(value: "key", style: .plain),
|
||||
.value,
|
||||
.scalar(value: "a\nb\n", style: .literal),
|
||||
.blockEnd,
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
|
||||
@Test("a folded block scalar joins lines with a space")
|
||||
func test_foldedBlockScalar() throws {
|
||||
let tokens = try tokenize("key: >\n a\n b\n")
|
||||
#expect(tokens == [
|
||||
.streamStart,
|
||||
.blockMappingStart,
|
||||
.key,
|
||||
.scalar(value: "key", style: .plain),
|
||||
.value,
|
||||
.scalar(value: "a b\n", style: .folded),
|
||||
.blockEnd,
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
|
||||
@Test("a strip chomping indicator drops the trailing newline")
|
||||
func test_literalStripChomping() throws {
|
||||
let tokens = try tokenize("key: |-\n a\n")
|
||||
#expect(tokens == [
|
||||
.streamStart,
|
||||
.blockMappingStart,
|
||||
.key,
|
||||
.scalar(value: "key", style: .plain),
|
||||
.value,
|
||||
.scalar(value: "a", style: .literal),
|
||||
.blockEnd,
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user