diff --git a/Sources/Parsing/LibYAML.Scanner.BlockScalar.swift b/Sources/Parsing/LibYAML.Scanner.BlockScalar.swift new file mode 100644 index 0000000..9685c19 --- /dev/null +++ b/Sources/Parsing/LibYAML.Scanner.BlockScalar.swift @@ -0,0 +1,165 @@ +extension PureYAML.Parsing.LibYAML.Scanner { + private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass + + /// The accumulation state for `scan_block_scalar`. + private struct BlockState { + var string = "" + var leadingBreak = "" + var trailingBreaks = "" + var leadingBlank = false + } + + /// Port of `yaml_parser_scan_block_scalar`. Scans a literal (`|`) or folded + /// (`>`) block scalar: parses the chomping (`+`/`-`) and indentation indicators + /// from the header, auto-detects the indentation if not given, then folds the + /// content (literal keeps breaks, folded joins equally-indented non-empty lines + /// with a space) and applies clip/strip/keep chomping to the trailing breaks. + mutating func scanBlockScalar(literal: Bool) throws -> PureYAML.Parsing.LibYAML.Token { + let startMark = reader.mark + reader.skip() + let header = try readBlockScalarHeader(startMark: startMark) + try consumeBlockHeaderTail(startMark: startMark) + + var endMark = reader.mark + var indent = 0 + if header.increment != 0 { + indent = indentEngine.indent >= 0 ? indentEngine.indent + header.increment : header.increment + } + var state = BlockState() + try scanBlockScalarBreaks(indent: &indent, state: &state, startMark: startMark, endMark: &endMark) + + while reader.column == indent, reader.peek() != nil { + foldBlockLine(&state, literal: literal) + while let character = reader.peek(), !Classify.isBreakZ(character) { + state.string.unicodeScalars.append(character) + reader.skip() + } + state.leadingBreak += readBlockLine() + try scanBlockScalarBreaks(indent: &indent, state: &state, startMark: startMark, endMark: &endMark) + } + + if header.chomping != -1 { + state.string += state.leadingBreak + } + if header.chomping == 1 { + state.string += state.trailingBreaks + } + return PureYAML.Parsing.LibYAML.Token( + type: .scalar(value: state.string, style: literal ? .literal : .folded), + startMark: startMark, + endMark: endMark, + ) + } + + /// Parse the chomping and indentation indicators in either order. + private mutating func readBlockScalarHeader(startMark: PureYAML.Parsing.Mark) throws -> (chomping: Int, increment: Int) { + var chomping = 0 + var increment = 0 + if reader.peek() == "+" || reader.peek() == "-" { + chomping = reader.peek() == "+" ? 1 : -1 + reader.skip() + if Classify.isDigit(reader.peek()) { + increment = try blockIncrement(startMark: startMark) + } + } else if Classify.isDigit(reader.peek()) { + increment = try blockIncrement(startMark: startMark) + if reader.peek() == "+" || reader.peek() == "-" { + chomping = reader.peek() == "+" ? 1 : -1 + reader.skip() + } + } + return (chomping, increment) + } + + private mutating func blockIncrement(startMark: PureYAML.Parsing.Mark) throws -> Int { + if reader.peek() == "0" { + throw blockError("found an indentation indicator equal to 0", startMark) + } + let value = digitValue(reader.peek()) + reader.skip() + return value + } + + /// Eat trailing blanks and a comment to the end of the header line, then the + /// line break. + private mutating func consumeBlockHeaderTail(startMark: PureYAML.Parsing.Mark) throws { + while reader.peek() == " " { + reader.skip() + } + if reader.peek() == "#" { + while !Classify.isBreakZ(reader.peek()) { + reader.skip() + } + } + guard Classify.isBreakZ(reader.peek()) else { + throw blockError("did not find expected comment or line break", startMark) + } + if Classify.isBreak(reader.peek()) { + reader.skipLine() + } + } + + /// Port of `yaml_parser_scan_block_scalar_breaks`: eat indentation spaces and + /// blank lines, auto-detecting the indentation level when none was given. + private mutating func scanBlockScalarBreaks(indent: inout Int, state: inout BlockState, startMark: PureYAML.Parsing.Mark, endMark: inout PureYAML.Parsing.Mark) throws { + var maxIndent = 0 + endMark = reader.mark + while true { + while indent == 0 || reader.column < indent, reader.peek() == " " { + reader.skip() + } + if reader.column > maxIndent { + maxIndent = reader.column + } + if indent == 0 || reader.column < indent, Classify.isTab(reader.peek()) { + throw blockError("found a tab character where an indentation space is expected", startMark) + } + if !Classify.isBreak(reader.peek()) { + break + } + state.trailingBreaks += readBlockLine() + endMark = reader.mark + } + if indent == 0 { + indent = max(maxIndent, indentEngine.indent + 1) + if indent < 1 { + indent = 1 + } + } + } + + /// Fold the pending breaks at the start of a content line (folded style joins a + /// single break between non-blank lines into a space; otherwise breaks are + /// kept). + private func foldBlockLine(_ state: inout BlockState, literal: Bool) { + let trailingBlank = Classify.isBlank(reader.peek()) + if !literal, state.leadingBreak.first == "\n", !state.leadingBlank, !trailingBlank { + if state.trailingBreaks.isEmpty { + state.string += " " + } + state.leadingBreak = "" + } else { + state.string += state.leadingBreak + state.leadingBreak = "" + } + state.string += state.trailingBreaks + state.trailingBreaks = "" + state.leadingBlank = Classify.isBlank(reader.peek()) + } + + private mutating func readBlockLine() -> String { + reader.skipLine() + return "\n" + } + + private func digitValue(_ scalar: Unicode.Scalar?) -> Int { + guard let scalar, scalar >= "0", scalar <= "9" else { + return 0 + } + return Int(scalar.value - Unicode.Scalar("0").value) + } + + private func blockError(_ problem: String, _ mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError { + PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a block scalar", problem: problem, mark: mark) + } +} diff --git a/Sources/Parsing/LibYAML.Scanner.Fetch.swift b/Sources/Parsing/LibYAML.Scanner.Fetch.swift index 34c0cbc..3818b77 100644 --- a/Sources/Parsing/LibYAML.Scanner.Fetch.swift +++ b/Sources/Parsing/LibYAML.Scanner.Fetch.swift @@ -90,7 +90,10 @@ extension PureYAML.Parsing.LibYAML.Scanner { if let unported = unportedConstruct(character) { throw notPorted(unported) } - if try fetchIndicatorToken(character) { + if try fetchFlowToken(character) { + return + } + if try fetchNodeToken(character) { return } switch character { @@ -105,24 +108,43 @@ extension PureYAML.Parsing.LibYAML.Scanner { } } - /// Fetch a flow-collection indicator, a quoted scalar, or an anchor/alias; - /// returns false if the character starts none of these. - private mutating func fetchIndicatorToken(_ character: Unicode.Scalar) throws -> Bool { + /// Fetch a flow-collection indicator (`[` `]` `{` `}` `,`); returns false + /// otherwise. + private mutating func fetchFlowToken(_ character: Unicode.Scalar) throws -> Bool { switch character { case "[": fetchFlowCollectionStart(.flowSequenceStart) case "{": fetchFlowCollectionStart(.flowMappingStart) case "]": try fetchFlowCollectionEnd(.flowSequenceEnd) case "}": try fetchFlowCollectionEnd(.flowMappingEnd) case ",": try fetchFlowEntry() + default: return false + } + return true + } + + /// Fetch a quoted scalar, an anchor/alias, or a block scalar; returns false + /// otherwise. + private mutating func fetchNodeToken(_ character: Unicode.Scalar) throws -> Bool { + switch character { case "'": try fetchFlowScalar(single: true) case "\"": try fetchFlowScalar(single: false) case "&": try fetchAnchor(isAnchor: true) case "*": try fetchAnchor(isAnchor: false) + case "|" where flowLevel == 0: try fetchBlockScalar(literal: true) + case ">" where flowLevel == 0: try fetchBlockScalar(literal: false) default: return false } return true } + /// Port of `yaml_parser_fetch_block_scalar`. + private mutating func fetchBlockScalar(literal: Bool) throws { + try removeSimpleKey() + simpleKeyAllowed = true + let token = try scanBlockScalar(literal: literal) + indentEngine.enqueue(token) + } + /// Port of `yaml_parser_fetch_flow_scalar`. private mutating func fetchFlowScalar(single: Bool) throws { mark = reader.mark @@ -145,7 +167,6 @@ extension PureYAML.Parsing.LibYAML.Scanner { private func unportedConstruct(_ character: Unicode.Scalar) -> String? { switch character { case "!": "tag" - case "|" where flowLevel == 0, ">" where flowLevel == 0: "block scalar" default: nil } } diff --git a/Tests/LibYAMLScannerFetchTests.swift b/Tests/LibYAMLScannerFetchTests.swift index 524e606..f2f4678 100644 --- a/Tests/LibYAMLScannerFetchTests.swift +++ b/Tests/LibYAMLScannerFetchTests.swift @@ -159,4 +159,49 @@ struct LibYAMLScannerFetchTests { .streamEnd, ]) } + + @Test("a literal block scalar keeps line breaks and clips the trailing newline") + func test_literalBlockScalar() throws { + let tokens = try tokenize("key: |\n a\n b\n") + #expect(tokens == [ + .streamStart, + .blockMappingStart, + .key, + .scalar(value: "key", style: .plain), + .value, + .scalar(value: "a\nb\n", style: .literal), + .blockEnd, + .streamEnd, + ]) + } + + @Test("a folded block scalar joins lines with a space") + func test_foldedBlockScalar() throws { + let tokens = try tokenize("key: >\n a\n b\n") + #expect(tokens == [ + .streamStart, + .blockMappingStart, + .key, + .scalar(value: "key", style: .plain), + .value, + .scalar(value: "a b\n", style: .folded), + .blockEnd, + .streamEnd, + ]) + } + + @Test("a strip chomping indicator drops the trailing newline") + func test_literalStripChomping() throws { + let tokens = try tokenize("key: |-\n a\n") + #expect(tokens == [ + .streamStart, + .blockMappingStart, + .key, + .scalar(value: "key", style: .plain), + .value, + .scalar(value: "a", style: .literal), + .blockEnd, + .streamEnd, + ]) + } }