NodetteHQ/PureYAML
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(parsing): enforce the full YAML 1.2 double-quoted escape table

Browse Source 
The double-quoted escape decoder recognized only a subset of escapes
(" \ \/ \b \f \n \r \t \u) and silently passed any other character
through. That accepted invalid escapes such as \. and \', and mangled
valid ones the subset omitted: \x and \U hex escapes, \v \e \0, the named
\N \_ \L \P, and a backslash before a space, were emitted as their literal
letters (\x0d became the text x0d).

The decoder now maps every escape in the YAML 1.2 table to its character,
reads \x (8-bit) and \U (32-bit) hex escapes alongside \u, treats a
backslash before a line break as an escaped line break that joins the
lines with no separator, and rejects an unknown or truncated escape with
a diagnostic. Single-character escapes move to a lookup table so the
decoder stays within the complexity limit.

Measured against the official YAML test-suite, parse-level conformance
rises from 247/333 (74.2%) to 249/333 (74.8%) as two invalid documents
are now rejected, and value-level from 166 to 170 of comparable cases
(90.4%) as the previously-mangled escapes decode correctly, with no
regression.
This commit is contained in:
Mihaela M 2026-06-27 16:19:25 +02:00
parent 1d07fce635
commit b7bcfe1a01
4 changed files with 152 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CHANGELOG.md
View File

@ -23,6 +23,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- Double-quoted escape sequences now follow the full YAML 1.2 escape table. Previously the
decoder recognized only a subset (`\" \\ \/ \b \f \n \r \t \u`) and silently passed any other
character through, so invalid escapes such as `\.` and `\'` were accepted while valid ones such
as `\x`, `\v`, `\e`, `\0`, `\N`, `\_`, `\L`, `\P`, `\U`, and backslash-space were mangled
(`\x0d` became the text `x0d`). The decoder now maps every defined escape to its character,
handles `\x` (8-bit) and `\U` (32-bit) hex escapes alongside `\u`, treats a backslash before a
line break as an escaped line break that joins the lines with no separator, and rejects an
unknown or truncated escape. Lifts official test-suite parse-level conformance from 247/333
(74.2%) to 249/333 (74.8%) and value-level from 166 to 170 of comparable cases (90.4%), with
no regression.
- A `#` glued to earlier content on its line (e.g. `"value"# c`, `[ a, b ]#c`, `>#c`) is now
rejected: the spec requires a comment to be preceded by whitespace or to open a line, and such
input was previously accepted as a comment. A `#` that opens a line, follows a space, or sits

11
Sources/Parsing/ParseError.swift
View File

@ -19,6 +19,7 @@ public extension PureYAML.Parsing {
case unexpectedToken(expected: String, actual: String, line: Int, column: Int)
case nestingTooDeep(line: Int, column: Int)
case commentNotPrecededByWhitespace(line: Int, column: Int)
case invalidEscape(sequence: String, line: Int)
public var description: String {
switch self {
@ -58,6 +59,8 @@ public extension PureYAML.Parsing {
"nesting is too deep at line \(line), column \(column)"
case let .commentNotPrecededByWhitespace(line, column):
"a comment must be preceded by whitespace at line \(line), column \(column)"
case let .invalidEscape(sequence, line):
"invalid escape sequence '\(sequence)' at line \(line)"
}
}
@ -99,6 +102,8 @@ public extension PureYAML.Parsing {
"nestingTooDeep"
case .commentNotPrecededByWhitespace:
"commentNotPrecededByWhitespace"
case .invalidEscape:
"invalidEscape"
}
}
@ -112,7 +117,8 @@ public extension PureYAML.Parsing {
let .unterminatedTag(line),
let .unterminatedQuotedString(line),
let .unsupportedDirective(_, line),
let .unsupportedMultiDocumentStream(line):
let .unsupportedMultiDocumentStream(line),
let .invalidEscape(_, line):
line
case let .expectedNode(line, _),
let .expectedScalarKey(line, _),
@ -137,7 +143,8 @@ public extension PureYAML.Parsing {
.unterminatedTag,
.unterminatedQuotedString,
.unsupportedDirective,
.unsupportedMultiDocumentStream:
.unsupportedMultiDocumentStream,
.invalidEscape:
nil
case let .expectedNode(_, column),
let .expectedScalarKey(_, column),

108
Sources/Parsing/Scalar.swift
View File

@ -151,7 +151,7 @@ extension PureYAML.Parsing.Parser {
output.append("\\")
break
}
appendDoubleQuotedEscape(in: inner, index: &index, to: &output)
try appendDoubleQuotedEscape(in: inner, index: &index, to: &output, line: line)
} else if character == "\n" || character == "\r" {
foldFlowScalarLineBreaks(in: inner, index: &index, to: &output)
} else {
@ -196,45 +196,99 @@ extension PureYAML.Parsing.Parser {
output.append(breaks == 1 ? " " : String(repeating: "\n", count: breaks - 1))
}
/// Decodes one double-quoted escape, the character after a `\`. The set is exactly YAML 1.2's
/// escape table; anything else is rejected rather than silently passed through. The index enters
/// at the escape character and is left on the last consumed character, so the caller's trailing
/// advance steps past it.
func appendDoubleQuotedEscape(
in text: Substring,
index: inout Substring.Index,
to output: inout String,
) {
line: Int,
) throws {
let character = text[index]
switch character {
case "\"": output.append("\"")
case "\\": output.append("\\")
case "/": output.append("/")
case "b": output.append("\u{08}")
case "f": output.append("\u{0C}")
case "n": output.append("\n")
case "r": output.append("\r")
case "t": output.append("\t")
case "u":
appendUnicodeEscape(in: text, index: &index, to: &output)
default: output.append(character)
if let replacement = Self.simpleDoubleQuotedEscapes[character] {
output.append(replacement)
} else {
switch character {
case "x": try appendHexEscape(in: text, index: &index, to: &output, digits: 2, line: line)
case "u": try appendUnicodeEscape(in: text, index: &index, to: &output, line: line)
case "U": try appendHexEscape(in: text, index: &index, to: &output, digits: 8, line: line)
case "\n", "\r": consumeEscapedLineBreak(in: text, index: &index)
default:
throw PureYAML.Parsing.ParseError.invalidEscape(sequence: "\\\(character)", line: line)
}
}
text.formIndex(after: &index)
}
/// The YAML 1.2 escapes that map a single character after `\` to one character. Escapes that
/// consume following characters (`\x`, `\u`, `\U`, an escaped line break) are handled separately.
static let simpleDoubleQuotedEscapes: [Character: Character] = [
"0": "\u{00}", "a": "\u{07}", "b": "\u{08}", "t": "\t", "\t": "\t",
"n": "\n", "v": "\u{0B}", "f": "\u{0C}", "r": "\r", "e": "\u{1B}",
" ": " ", "\"": "\"", "/": "/", "\\": "\\",
"N": "\u{85}", "_": "\u{A0}", "L": "\u{2028}", "P": "\u{2029}",
]
/// Handles a `\` immediately before a line break: an escaped line break, where the two lines
/// join with no separator. Consumes the break (LF, CR, or CRLF) and the continuation line's
/// leading spaces and tabs, emitting nothing, and leaves the index on the last consumed
/// character so the caller's advance steps to the continuation's first content character.
func consumeEscapedLineBreak(
in text: Substring,
index: inout Substring.Index,
) {
if text[index] == "\r" {
var next = index
text.formIndex(after: &next)
if next < text.endIndex, text[next] == "\n" {
index = next
}
}
while true {
var next = index
text.formIndex(after: &next)
guard next < text.endIndex, text[next] == " " || text[next] == "\t" else {
break
}
index = next
}
}
/// Reads a fixed-width hex escape (`\xHH`, `\UHHHHHHHH`) and appends the scalar it denotes,
/// rejecting a truncated sequence or a value that is not a valid Unicode scalar.
func appendHexEscape(
in text: Substring,
index: inout Substring.Index,
to output: inout String,
digits: Int,
line: Int,
) throws {
let typeCharacter = text[index]
guard let value = readHexEscape(in: text, index: &index, digits: digits),
let scalar = UnicodeScalar(value)
else {
throw PureYAML.Parsing.ParseError.invalidEscape(sequence: "\\\(typeCharacter)", line: line)
}
output.unicodeScalars.append(scalar)
}
func appendUnicodeEscape(
in text: Substring,
index: inout Substring.Index,
to output: inout String,
) {
let escapeStart = index
guard let codeUnit = readUnicodeEscapeCodeUnit(in: text, index: &index) else {
output.append("u")
return
line: Int,
) throws {
guard let codeUnit = readHexEscape(in: text, index: &index, digits: 4) else {
throw PureYAML.Parsing.ParseError.invalidEscape(sequence: "\\u", line: line)
}
let surrogatePair = readSurrogatePairIfNeeded(high: codeUnit, in: text, index: &index)
if let scalar = surrogatePair {
if let scalar = readSurrogatePairIfNeeded(high: codeUnit, in: text, index: &index) {
output.unicodeScalars.append(scalar)
} else if let scalar = UnicodeScalar(codeUnit) {
output.unicodeScalars.append(scalar)
} else {
output += text[escapeStart ... index]
throw PureYAML.Parsing.ParseError.invalidEscape(sequence: "\\u", line: line)
}
}
@ -261,7 +315,7 @@ extension PureYAML.Parsing.Parser {
}
text.formIndex(after: &probe)
guard probe < text.endIndex, text[probe] == "u",
let low = readUnicodeEscapeCodeUnit(in: text, index: &probe),
let low = readHexEscape(in: text, index: &probe, digits: 4),
isLowSurrogate(low)
else {
return nil
@ -271,14 +325,18 @@ extension PureYAML.Parsing.Parser {
return UnicodeScalar(value)
}
func readUnicodeEscapeCodeUnit(
/// Reads `digits` hex characters following the escape type character. The index enters at the
/// type character and is left on the final hex digit; returns nil if fewer than `digits` hex
/// characters are available.
func readHexEscape(
in text: Substring,
index: inout Substring.Index,
digits: Int,
) -> UInt32? {
var probe = index
text.formIndex(after: &probe)
var hex = ""
for _ in 0 ..< 4 {
for _ in 0 ..< digits {
guard probe < text.endIndex, text[probe].isHexDigit else {
return nil
}

50
Tests/DoubleQuotedEscapeTests.swift Normal file
View File

@ -0,0 +1,50 @@
@testable import PureYAML
import Testing
@Suite("Double-quoted escape sequences")
struct DoubleQuotedEscapeTests {
@Test("Every YAML 1.2 escape decodes to its character", arguments: [
("\"\\0\"\n", "\u{00}"),
("\"\\a\"\n", "\u{07}"),
("\"\\b\"\n", "\u{08}"),
("\"\\t\"\n", "\t"),
("\"\\n\"\n", "\n"),
("\"\\v\"\n", "\u{0B}"),
("\"\\f\"\n", "\u{0C}"),
("\"\\r\"\n", "\r"),
("\"\\e\"\n", "\u{1B}"),
("\"a\\ b\"\n", "a b"),
("\"\\\"\"\n", "\""),
("\"\\/\"\n", "/"),
("\"\\\\\"\n", "\\"),
("\"\\N\"\n", "\u{85}"),
("\"\\_\"\n", "\u{A0}"),
("\"\\L\"\n", "\u{2028}"),
("\"\\P\"\n", "\u{2029}"),
("\"\\x41\"\n", "A"),
("\"\\u0041\"\n", "A"),
("\"\\U0001F600\"\n", "\u{1F600}"),
])
func test_validEscapeDecodes(_ yaml: String, _ expected: String) throws {
#expect(try PureYAML.parse(yaml) == .string(expected))
}
@Test("An unknown or truncated escape is rejected", arguments: [
"\"\\.\"\n", // not an escape character
"\"\\'\"\n", // single quote is not escaped in a double-quoted scalar
"\"\\q\"\n", // unknown letter
"\"\\x4\"\n", // truncated hex
"\"\\U12\"\n", // truncated 32-bit hex
])
func test_invalidEscapeIsRejected(_ yaml: String) throws {
#expect(throws: PureYAML.Parsing.ParseError.self) {
try PureYAML.parse(yaml)
}
}
@Test("An escaped line break joins the lines with no separator")
func test_escapedLineBreakJoinsLines() throws {
#expect(try PureYAML.parse("\"a\\\n b\"\n") == .string("ab"))
#expect(try PureYAML.parse("\"one \\\n two\"\n") == .string("one two"))
}
}