feat(parsing): port libyaml quoted-scalar scanner to Swift
Stage 3 of the faithful libyaml scanner port, continued. Translates fetch_flow_scalar and scan_flow_scalar: single- and double-quoted scalars with line-break folding (a single break folds to a space, blank lines to newlines), the single-quote '' escape, double-quoted backslash escapes (the full \0 \a \b \t \n \v \f \r \e space \" / \ \N \_ \L \P table plus \xXX/\uXXXX/\UXXXXXXXX hex), and the escaped end-of-line continuation that suppresses the break. Escapes are decoded to Unicode scalars directly rather than UTF-8 bytes. Unit-tested for the escaped single quote, the escape table including a hex unicode escape, and multi-line folding. The remaining unported fetchers are block scalars, anchors, tags, and directives. 310 tests, no regression.
This commit is contained in:
parent
27514466d0
commit
4e7ca8fcb2
@ -93,6 +93,9 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
if try fetchFlowToken(character) {
|
||||
return
|
||||
}
|
||||
if try fetchQuotedScalar(character) {
|
||||
return
|
||||
}
|
||||
switch character {
|
||||
case "-" where Classify.isBlankZ(reader.peek(1)):
|
||||
fetchBlockEntry()
|
||||
@ -105,6 +108,26 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch a single- or double-quoted scalar; returns false if the character is
|
||||
/// not a quote.
|
||||
private mutating func fetchQuotedScalar(_ character: Unicode.Scalar) throws -> Bool {
|
||||
switch character {
|
||||
case "'": try fetchFlowScalar(single: true)
|
||||
case "\"": try fetchFlowScalar(single: false)
|
||||
default: return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_fetch_flow_scalar`.
|
||||
private mutating func fetchFlowScalar(single: Bool) throws {
|
||||
mark = reader.mark
|
||||
try saveSimpleKey()
|
||||
simpleKeyAllowed = false
|
||||
let token = try scanFlowScalar(single: single)
|
||||
indentEngine.enqueue(token)
|
||||
}
|
||||
|
||||
/// Fetch a flow-collection indicator (`[` `]` `{` `}` `,`); returns false if
|
||||
/// the character is not one.
|
||||
private mutating func fetchFlowToken(_ character: Unicode.Scalar) throws -> Bool {
|
||||
@ -125,7 +148,6 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
case "*", "&": "anchor or alias"
|
||||
case "!": "tag"
|
||||
case "|" where flowLevel == 0, ">" where flowLevel == 0: "block scalar"
|
||||
case "'", "\"": "flow scalar"
|
||||
default: nil
|
||||
}
|
||||
}
|
||||
|
||||
176
Sources/Parsing/LibYAML.Scanner.FlowScalar.swift
Normal file
176
Sources/Parsing/LibYAML.Scanner.FlowScalar.swift
Normal file
@ -0,0 +1,176 @@
|
||||
extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
|
||||
|
||||
/// The folding state carried through `scan_flow_scalar`, the same shape as the
|
||||
/// plain-scalar state.
|
||||
private struct FlowState {
|
||||
var string = ""
|
||||
var leadingBreak = ""
|
||||
var trailingBreaks = ""
|
||||
var whitespaces = ""
|
||||
var leadingBlanks = false
|
||||
}
|
||||
|
||||
/// Decoded single-character escapes for double-quoted scalars (libyaml's escape
|
||||
/// switch). libyaml emits UTF-8 bytes; here each maps to its Unicode scalar.
|
||||
private static let doubleQuotedEscapes: [Unicode.Scalar: Unicode.Scalar] = [
|
||||
"0": "\u{0}", "a": "\u{07}", "b": "\u{08}", "t": "\u{09}", "\t": "\u{09}",
|
||||
"n": "\u{0A}", "v": "\u{0B}", "f": "\u{0C}", "r": "\u{0D}", "e": "\u{1B}",
|
||||
" ": "\u{20}", "\"": "\"", "/": "/", "\\": "\\",
|
||||
"N": "\u{85}", "_": "\u{A0}", "L": "\u{2028}", "P": "\u{2029}",
|
||||
]
|
||||
|
||||
/// Port of `yaml_parser_scan_flow_scalar`. Scans a single- or double-quoted
|
||||
/// scalar, folding line breaks like a plain scalar and, for double quotes,
|
||||
/// decoding escape sequences.
|
||||
mutating func scanFlowScalar(single: Bool) throws -> PureYAML.Parsing.LibYAML.Token {
|
||||
let startMark = reader.mark
|
||||
reader.skip()
|
||||
var state = FlowState()
|
||||
let quote: Unicode.Scalar = single ? "'" : "\""
|
||||
while true {
|
||||
if reader.column == 0, documentMarkerAhead() {
|
||||
throw flowError("found unexpected document indicator", startMark)
|
||||
}
|
||||
if reader.peek() == nil {
|
||||
throw flowError("found unexpected end of stream", startMark)
|
||||
}
|
||||
try consumeFlowContent(single: single, &state, startMark: startMark)
|
||||
if reader.peek() == quote {
|
||||
break
|
||||
}
|
||||
consumeFlowBreaks(&state)
|
||||
foldFlow(&state)
|
||||
}
|
||||
reader.skip()
|
||||
return PureYAML.Parsing.LibYAML.Token(
|
||||
type: .scalar(value: state.string, style: single ? .singleQuoted : .doubleQuoted),
|
||||
startMark: startMark,
|
||||
endMark: reader.mark,
|
||||
)
|
||||
}
|
||||
|
||||
/// Consume the run of non-blank characters, decoding escapes and stopping at
|
||||
/// the closing quote or an escaped line break.
|
||||
private mutating func consumeFlowContent(single: Bool, _ state: inout FlowState, startMark: PureYAML.Parsing.Mark) throws {
|
||||
state.leadingBlanks = false
|
||||
let quote: Unicode.Scalar = single ? "'" : "\""
|
||||
while let character = reader.peek(), !Classify.isBlankZ(character) {
|
||||
if single, character == "'", reader.peek(1) == "'" {
|
||||
state.string.unicodeScalars.append("'")
|
||||
reader.skip()
|
||||
reader.skip()
|
||||
} else if character == quote {
|
||||
return
|
||||
} else if !single, character == "\\", Classify.isBreak(reader.peek(1)) {
|
||||
reader.skip()
|
||||
reader.skipLine()
|
||||
state.leadingBlanks = true
|
||||
return
|
||||
} else if !single, character == "\\" {
|
||||
try appendEscape(&state, startMark: startMark)
|
||||
} else {
|
||||
state.string.unicodeScalars.append(character)
|
||||
reader.skip()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode a double-quoted escape sequence (`\` already at the reader).
|
||||
private mutating func appendEscape(_ state: inout FlowState, startMark: PureYAML.Parsing.Mark) throws {
|
||||
let escape = reader.peek(1)
|
||||
if let escape, let decoded = Self.doubleQuotedEscapes[escape] {
|
||||
state.string.unicodeScalars.append(decoded)
|
||||
reader.skip()
|
||||
reader.skip()
|
||||
return
|
||||
}
|
||||
let codeLength: Int
|
||||
switch escape {
|
||||
case "x": codeLength = 2
|
||||
case "u": codeLength = 4
|
||||
case "U": codeLength = 8
|
||||
default: throw flowError("found unknown escape character", startMark)
|
||||
}
|
||||
reader.skip()
|
||||
reader.skip()
|
||||
var value: UInt32 = 0
|
||||
for offset in 0 ..< codeLength {
|
||||
guard let digit = reader.peek(offset), Classify.isHex(digit) else {
|
||||
throw flowError("did not find expected hexdecimal number", startMark)
|
||||
}
|
||||
value = (value << 4) + hexValue(digit)
|
||||
}
|
||||
guard let scalar = Unicode.Scalar(value) else {
|
||||
throw flowError("found invalid Unicode character escape code", startMark)
|
||||
}
|
||||
state.string.unicodeScalars.append(scalar)
|
||||
for _ in 0 ..< codeLength {
|
||||
reader.skip()
|
||||
}
|
||||
}
|
||||
|
||||
/// Consume blanks and line breaks between content, recording pending
|
||||
/// whitespace and folded breaks.
|
||||
private mutating func consumeFlowBreaks(_ state: inout FlowState) {
|
||||
while Classify.isBlank(reader.peek()) || Classify.isBreak(reader.peek()) {
|
||||
if Classify.isBlank(reader.peek()) {
|
||||
if !state.leadingBlanks, let blank = reader.peek() {
|
||||
state.whitespaces.unicodeScalars.append(blank)
|
||||
}
|
||||
reader.skip()
|
||||
} else if !state.leadingBlanks {
|
||||
state.whitespaces = ""
|
||||
reader.skipLine()
|
||||
state.leadingBreak = "\n"
|
||||
state.leadingBlanks = true
|
||||
} else {
|
||||
reader.skipLine()
|
||||
state.trailingBreaks += "\n"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fold the pending breaks or whitespace into the value.
|
||||
private func foldFlow(_ state: inout FlowState) {
|
||||
guard state.leadingBlanks else {
|
||||
state.string += state.whitespaces
|
||||
state.whitespaces = ""
|
||||
return
|
||||
}
|
||||
if state.leadingBreak.first == "\n" {
|
||||
if state.trailingBreaks.isEmpty {
|
||||
state.string += " "
|
||||
} else {
|
||||
state.string += state.trailingBreaks
|
||||
state.trailingBreaks = ""
|
||||
}
|
||||
state.leadingBreak = ""
|
||||
} else {
|
||||
state.string += state.leadingBreak
|
||||
state.string += state.trailingBreaks
|
||||
state.leadingBreak = ""
|
||||
state.trailingBreaks = ""
|
||||
}
|
||||
}
|
||||
|
||||
private func hexValue(_ scalar: Unicode.Scalar) -> UInt32 {
|
||||
switch scalar {
|
||||
case "0" ... "9": scalar.value - Unicode.Scalar("0").value
|
||||
case "A" ... "F": scalar.value - Unicode.Scalar("A").value + 10
|
||||
default: scalar.value - Unicode.Scalar("a").value + 10
|
||||
}
|
||||
}
|
||||
|
||||
private func flowError(_ problem: String, _ mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError {
|
||||
PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a quoted scalar", problem: problem, mark: mark)
|
||||
}
|
||||
|
||||
private func documentMarkerAhead() -> Bool {
|
||||
let first = reader.peek()
|
||||
guard first == "-" || first == "." else {
|
||||
return false
|
||||
}
|
||||
return reader.peek(1) == first && reader.peek(2) == first && Classify.isBlankZ(reader.peek(3))
|
||||
}
|
||||
}
|
||||
@ -109,4 +109,34 @@ struct LibYAMLScannerFetchTests {
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
|
||||
@Test("single-quoted scalar with an escaped quote")
|
||||
func test_singleQuoted() throws {
|
||||
let tokens = try tokenize("'it''s here'\n")
|
||||
#expect(tokens == [
|
||||
.streamStart,
|
||||
.scalar(value: "it's here", style: .singleQuoted),
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
|
||||
@Test("double-quoted scalar decodes escapes")
|
||||
func test_doubleQuotedEscapes() throws {
|
||||
let tokens = try tokenize("\"a\\nb\\tc\\u263A\"\n")
|
||||
#expect(tokens == [
|
||||
.streamStart,
|
||||
.scalar(value: "a\nb\tc\u{263A}", style: .doubleQuoted),
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
|
||||
@Test("a multi-line quoted scalar folds line breaks")
|
||||
func test_quotedFolding() throws {
|
||||
let tokens = try tokenize("\"a\nb\"\n")
|
||||
#expect(tokens == [
|
||||
.streamStart,
|
||||
.scalar(value: "a b", style: .doubleQuoted),
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user