From 4e7ca8fcb2206ef86fa76af5748791e000f10ffd Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Wed, 24 Jun 2026 15:22:41 +0200 Subject: [PATCH] feat(parsing): port libyaml quoted-scalar scanner to Swift Stage 3 of the faithful libyaml scanner port, continued. Translates fetch_flow_scalar and scan_flow_scalar: single- and double-quoted scalars with line-break folding (a single break folds to a space, blank lines to newlines), the single-quote '' escape, double-quoted backslash escapes (the full \0 \a \b \t \n \v \f \r \e space \" / \ \N \_ \L \P table plus \xXX/\uXXXX/\UXXXXXXXX hex), and the escaped end-of-line continuation that suppresses the break. Escapes are decoded to Unicode scalars directly rather than UTF-8 bytes. Unit-tested for the escaped single quote, the escape table including a hex unicode escape, and multi-line folding. The remaining unported fetchers are block scalars, anchors, tags, and directives. 310 tests, no regression. --- Sources/Parsing/LibYAML.Scanner.Fetch.swift | 24 ++- .../Parsing/LibYAML.Scanner.FlowScalar.swift | 176 ++++++++++++++++++ Tests/LibYAMLScannerFetchTests.swift | 30 +++ 3 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 Sources/Parsing/LibYAML.Scanner.FlowScalar.swift diff --git a/Sources/Parsing/LibYAML.Scanner.Fetch.swift b/Sources/Parsing/LibYAML.Scanner.Fetch.swift index 33a8799..4d54a6e 100644 --- a/Sources/Parsing/LibYAML.Scanner.Fetch.swift +++ b/Sources/Parsing/LibYAML.Scanner.Fetch.swift @@ -93,6 +93,9 @@ extension PureYAML.Parsing.LibYAML.Scanner { if try fetchFlowToken(character) { return } + if try fetchQuotedScalar(character) { + return + } switch character { case "-" where Classify.isBlankZ(reader.peek(1)): fetchBlockEntry() @@ -105,6 +108,26 @@ extension PureYAML.Parsing.LibYAML.Scanner { } } + /// Fetch a single- or double-quoted scalar; returns false if the character is + /// not a quote. + private mutating func fetchQuotedScalar(_ character: Unicode.Scalar) throws -> Bool { + switch character { + case "'": try fetchFlowScalar(single: true) + case "\"": try fetchFlowScalar(single: false) + default: return false + } + return true + } + + /// Port of `yaml_parser_fetch_flow_scalar`. + private mutating func fetchFlowScalar(single: Bool) throws { + mark = reader.mark + try saveSimpleKey() + simpleKeyAllowed = false + let token = try scanFlowScalar(single: single) + indentEngine.enqueue(token) + } + /// Fetch a flow-collection indicator (`[` `]` `{` `}` `,`); returns false if /// the character is not one. private mutating func fetchFlowToken(_ character: Unicode.Scalar) throws -> Bool { @@ -125,7 +148,6 @@ extension PureYAML.Parsing.LibYAML.Scanner { case "*", "&": "anchor or alias" case "!": "tag" case "|" where flowLevel == 0, ">" where flowLevel == 0: "block scalar" - case "'", "\"": "flow scalar" default: nil } } diff --git a/Sources/Parsing/LibYAML.Scanner.FlowScalar.swift b/Sources/Parsing/LibYAML.Scanner.FlowScalar.swift new file mode 100644 index 0000000..0ae318c --- /dev/null +++ b/Sources/Parsing/LibYAML.Scanner.FlowScalar.swift @@ -0,0 +1,176 @@ +extension PureYAML.Parsing.LibYAML.Scanner { + private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass + + /// The folding state carried through `scan_flow_scalar`, the same shape as the + /// plain-scalar state. + private struct FlowState { + var string = "" + var leadingBreak = "" + var trailingBreaks = "" + var whitespaces = "" + var leadingBlanks = false + } + + /// Decoded single-character escapes for double-quoted scalars (libyaml's escape + /// switch). libyaml emits UTF-8 bytes; here each maps to its Unicode scalar. + private static let doubleQuotedEscapes: [Unicode.Scalar: Unicode.Scalar] = [ + "0": "\u{0}", "a": "\u{07}", "b": "\u{08}", "t": "\u{09}", "\t": "\u{09}", + "n": "\u{0A}", "v": "\u{0B}", "f": "\u{0C}", "r": "\u{0D}", "e": "\u{1B}", + " ": "\u{20}", "\"": "\"", "/": "/", "\\": "\\", + "N": "\u{85}", "_": "\u{A0}", "L": "\u{2028}", "P": "\u{2029}", + ] + + /// Port of `yaml_parser_scan_flow_scalar`. Scans a single- or double-quoted + /// scalar, folding line breaks like a plain scalar and, for double quotes, + /// decoding escape sequences. + mutating func scanFlowScalar(single: Bool) throws -> PureYAML.Parsing.LibYAML.Token { + let startMark = reader.mark + reader.skip() + var state = FlowState() + let quote: Unicode.Scalar = single ? "'" : "\"" + while true { + if reader.column == 0, documentMarkerAhead() { + throw flowError("found unexpected document indicator", startMark) + } + if reader.peek() == nil { + throw flowError("found unexpected end of stream", startMark) + } + try consumeFlowContent(single: single, &state, startMark: startMark) + if reader.peek() == quote { + break + } + consumeFlowBreaks(&state) + foldFlow(&state) + } + reader.skip() + return PureYAML.Parsing.LibYAML.Token( + type: .scalar(value: state.string, style: single ? .singleQuoted : .doubleQuoted), + startMark: startMark, + endMark: reader.mark, + ) + } + + /// Consume the run of non-blank characters, decoding escapes and stopping at + /// the closing quote or an escaped line break. + private mutating func consumeFlowContent(single: Bool, _ state: inout FlowState, startMark: PureYAML.Parsing.Mark) throws { + state.leadingBlanks = false + let quote: Unicode.Scalar = single ? "'" : "\"" + while let character = reader.peek(), !Classify.isBlankZ(character) { + if single, character == "'", reader.peek(1) == "'" { + state.string.unicodeScalars.append("'") + reader.skip() + reader.skip() + } else if character == quote { + return + } else if !single, character == "\\", Classify.isBreak(reader.peek(1)) { + reader.skip() + reader.skipLine() + state.leadingBlanks = true + return + } else if !single, character == "\\" { + try appendEscape(&state, startMark: startMark) + } else { + state.string.unicodeScalars.append(character) + reader.skip() + } + } + } + + /// Decode a double-quoted escape sequence (`\` already at the reader). + private mutating func appendEscape(_ state: inout FlowState, startMark: PureYAML.Parsing.Mark) throws { + let escape = reader.peek(1) + if let escape, let decoded = Self.doubleQuotedEscapes[escape] { + state.string.unicodeScalars.append(decoded) + reader.skip() + reader.skip() + return + } + let codeLength: Int + switch escape { + case "x": codeLength = 2 + case "u": codeLength = 4 + case "U": codeLength = 8 + default: throw flowError("found unknown escape character", startMark) + } + reader.skip() + reader.skip() + var value: UInt32 = 0 + for offset in 0 ..< codeLength { + guard let digit = reader.peek(offset), Classify.isHex(digit) else { + throw flowError("did not find expected hexdecimal number", startMark) + } + value = (value << 4) + hexValue(digit) + } + guard let scalar = Unicode.Scalar(value) else { + throw flowError("found invalid Unicode character escape code", startMark) + } + state.string.unicodeScalars.append(scalar) + for _ in 0 ..< codeLength { + reader.skip() + } + } + + /// Consume blanks and line breaks between content, recording pending + /// whitespace and folded breaks. + private mutating func consumeFlowBreaks(_ state: inout FlowState) { + while Classify.isBlank(reader.peek()) || Classify.isBreak(reader.peek()) { + if Classify.isBlank(reader.peek()) { + if !state.leadingBlanks, let blank = reader.peek() { + state.whitespaces.unicodeScalars.append(blank) + } + reader.skip() + } else if !state.leadingBlanks { + state.whitespaces = "" + reader.skipLine() + state.leadingBreak = "\n" + state.leadingBlanks = true + } else { + reader.skipLine() + state.trailingBreaks += "\n" + } + } + } + + /// Fold the pending breaks or whitespace into the value. + private func foldFlow(_ state: inout FlowState) { + guard state.leadingBlanks else { + state.string += state.whitespaces + state.whitespaces = "" + return + } + if state.leadingBreak.first == "\n" { + if state.trailingBreaks.isEmpty { + state.string += " " + } else { + state.string += state.trailingBreaks + state.trailingBreaks = "" + } + state.leadingBreak = "" + } else { + state.string += state.leadingBreak + state.string += state.trailingBreaks + state.leadingBreak = "" + state.trailingBreaks = "" + } + } + + private func hexValue(_ scalar: Unicode.Scalar) -> UInt32 { + switch scalar { + case "0" ... "9": scalar.value - Unicode.Scalar("0").value + case "A" ... "F": scalar.value - Unicode.Scalar("A").value + 10 + default: scalar.value - Unicode.Scalar("a").value + 10 + } + } + + private func flowError(_ problem: String, _ mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError { + PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a quoted scalar", problem: problem, mark: mark) + } + + private func documentMarkerAhead() -> Bool { + let first = reader.peek() + guard first == "-" || first == "." else { + return false + } + return reader.peek(1) == first && reader.peek(2) == first && Classify.isBlankZ(reader.peek(3)) + } +} diff --git a/Tests/LibYAMLScannerFetchTests.swift b/Tests/LibYAMLScannerFetchTests.swift index 90a2cd0..ff69126 100644 --- a/Tests/LibYAMLScannerFetchTests.swift +++ b/Tests/LibYAMLScannerFetchTests.swift @@ -109,4 +109,34 @@ struct LibYAMLScannerFetchTests { .streamEnd, ]) } + + @Test("single-quoted scalar with an escaped quote") + func test_singleQuoted() throws { + let tokens = try tokenize("'it''s here'\n") + #expect(tokens == [ + .streamStart, + .scalar(value: "it's here", style: .singleQuoted), + .streamEnd, + ]) + } + + @Test("double-quoted scalar decodes escapes") + func test_doubleQuotedEscapes() throws { + let tokens = try tokenize("\"a\\nb\\tc\\u263A\"\n") + #expect(tokens == [ + .streamStart, + .scalar(value: "a\nb\tc\u{263A}", style: .doubleQuoted), + .streamEnd, + ]) + } + + @Test("a multi-line quoted scalar folds line breaks") + func test_quotedFolding() throws { + let tokens = try tokenize("\"a\nb\"\n") + #expect(tokens == [ + .streamStart, + .scalar(value: "a b", style: .doubleQuoted), + .streamEnd, + ]) + } }