diff --git a/CHANGELOG.md b/CHANGELOG.md index 929d3f4..b83d1fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Double-quoted escape sequences now follow the full YAML 1.2 escape table. Previously the + decoder recognized only a subset (`\" \\ \/ \b \f \n \r \t \u`) and silently passed any other + character through, so invalid escapes such as `\.` and `\'` were accepted while valid ones such + as `\x`, `\v`, `\e`, `\0`, `\N`, `\_`, `\L`, `\P`, `\U`, and backslash-space were mangled + (`\x0d` became the text `x0d`). The decoder now maps every defined escape to its character, + handles `\x` (8-bit) and `\U` (32-bit) hex escapes alongside `\u`, treats a backslash before a + line break as an escaped line break that joins the lines with no separator, and rejects an + unknown or truncated escape. Lifts official test-suite parse-level conformance from 247/333 + (74.2%) to 249/333 (74.8%) and value-level from 166 to 170 of comparable cases (90.4%), with + no regression. - A `#` glued to earlier content on its line (e.g. `"value"# c`, `[ a, b ]#c`, `>#c`) is now rejected: the spec requires a comment to be preceded by whitespace or to open a line, and such input was previously accepted as a comment. A `#` that opens a line, follows a space, or sits diff --git a/Sources/Parsing/ParseError.swift b/Sources/Parsing/ParseError.swift index fe018cd..beccdde 100644 --- a/Sources/Parsing/ParseError.swift +++ b/Sources/Parsing/ParseError.swift @@ -19,6 +19,7 @@ public extension PureYAML.Parsing { case unexpectedToken(expected: String, actual: String, line: Int, column: Int) case nestingTooDeep(line: Int, column: Int) case commentNotPrecededByWhitespace(line: Int, column: Int) + case invalidEscape(sequence: String, line: Int) public var description: String { switch self { @@ -58,6 +59,8 @@ public extension PureYAML.Parsing { "nesting is too deep at line \(line), column \(column)" case let .commentNotPrecededByWhitespace(line, column): "a comment must be preceded by whitespace at line \(line), column \(column)" + case let .invalidEscape(sequence, line): + "invalid escape sequence '\(sequence)' at line \(line)" } } @@ -99,6 +102,8 @@ public extension PureYAML.Parsing { "nestingTooDeep" case .commentNotPrecededByWhitespace: "commentNotPrecededByWhitespace" + case .invalidEscape: + "invalidEscape" } } @@ -112,7 +117,8 @@ public extension PureYAML.Parsing { let .unterminatedTag(line), let .unterminatedQuotedString(line), let .unsupportedDirective(_, line), - let .unsupportedMultiDocumentStream(line): + let .unsupportedMultiDocumentStream(line), + let .invalidEscape(_, line): line case let .expectedNode(line, _), let .expectedScalarKey(line, _), @@ -137,7 +143,8 @@ public extension PureYAML.Parsing { .unterminatedTag, .unterminatedQuotedString, .unsupportedDirective, - .unsupportedMultiDocumentStream: + .unsupportedMultiDocumentStream, + .invalidEscape: nil case let .expectedNode(_, column), let .expectedScalarKey(_, column), diff --git a/Sources/Parsing/Scalar.swift b/Sources/Parsing/Scalar.swift index 8cb55a8..e82c433 100644 --- a/Sources/Parsing/Scalar.swift +++ b/Sources/Parsing/Scalar.swift @@ -151,7 +151,7 @@ extension PureYAML.Parsing.Parser { output.append("\\") break } - appendDoubleQuotedEscape(in: inner, index: &index, to: &output) + try appendDoubleQuotedEscape(in: inner, index: &index, to: &output, line: line) } else if character == "\n" || character == "\r" { foldFlowScalarLineBreaks(in: inner, index: &index, to: &output) } else { @@ -196,45 +196,99 @@ extension PureYAML.Parsing.Parser { output.append(breaks == 1 ? " " : String(repeating: "\n", count: breaks - 1)) } + /// Decodes one double-quoted escape, the character after a `\`. The set is exactly YAML 1.2's + /// escape table; anything else is rejected rather than silently passed through. The index enters + /// at the escape character and is left on the last consumed character, so the caller's trailing + /// advance steps past it. func appendDoubleQuotedEscape( in text: Substring, index: inout Substring.Index, to output: inout String, - ) { + line: Int, + ) throws { let character = text[index] - switch character { - case "\"": output.append("\"") - case "\\": output.append("\\") - case "/": output.append("/") - case "b": output.append("\u{08}") - case "f": output.append("\u{0C}") - case "n": output.append("\n") - case "r": output.append("\r") - case "t": output.append("\t") - case "u": - appendUnicodeEscape(in: text, index: &index, to: &output) - default: output.append(character) + if let replacement = Self.simpleDoubleQuotedEscapes[character] { + output.append(replacement) + } else { + switch character { + case "x": try appendHexEscape(in: text, index: &index, to: &output, digits: 2, line: line) + case "u": try appendUnicodeEscape(in: text, index: &index, to: &output, line: line) + case "U": try appendHexEscape(in: text, index: &index, to: &output, digits: 8, line: line) + case "\n", "\r": consumeEscapedLineBreak(in: text, index: &index) + default: + throw PureYAML.Parsing.ParseError.invalidEscape(sequence: "\\\(character)", line: line) + } } text.formIndex(after: &index) } + /// The YAML 1.2 escapes that map a single character after `\` to one character. Escapes that + /// consume following characters (`\x`, `\u`, `\U`, an escaped line break) are handled separately. + static let simpleDoubleQuotedEscapes: [Character: Character] = [ + "0": "\u{00}", "a": "\u{07}", "b": "\u{08}", "t": "\t", "\t": "\t", + "n": "\n", "v": "\u{0B}", "f": "\u{0C}", "r": "\r", "e": "\u{1B}", + " ": " ", "\"": "\"", "/": "/", "\\": "\\", + "N": "\u{85}", "_": "\u{A0}", "L": "\u{2028}", "P": "\u{2029}", + ] + + /// Handles a `\` immediately before a line break: an escaped line break, where the two lines + /// join with no separator. Consumes the break (LF, CR, or CRLF) and the continuation line's + /// leading spaces and tabs, emitting nothing, and leaves the index on the last consumed + /// character so the caller's advance steps to the continuation's first content character. + func consumeEscapedLineBreak( + in text: Substring, + index: inout Substring.Index, + ) { + if text[index] == "\r" { + var next = index + text.formIndex(after: &next) + if next < text.endIndex, text[next] == "\n" { + index = next + } + } + while true { + var next = index + text.formIndex(after: &next) + guard next < text.endIndex, text[next] == " " || text[next] == "\t" else { + break + } + index = next + } + } + + /// Reads a fixed-width hex escape (`\xHH`, `\UHHHHHHHH`) and appends the scalar it denotes, + /// rejecting a truncated sequence or a value that is not a valid Unicode scalar. + func appendHexEscape( + in text: Substring, + index: inout Substring.Index, + to output: inout String, + digits: Int, + line: Int, + ) throws { + let typeCharacter = text[index] + guard let value = readHexEscape(in: text, index: &index, digits: digits), + let scalar = UnicodeScalar(value) + else { + throw PureYAML.Parsing.ParseError.invalidEscape(sequence: "\\\(typeCharacter)", line: line) + } + output.unicodeScalars.append(scalar) + } + func appendUnicodeEscape( in text: Substring, index: inout Substring.Index, to output: inout String, - ) { - let escapeStart = index - guard let codeUnit = readUnicodeEscapeCodeUnit(in: text, index: &index) else { - output.append("u") - return + line: Int, + ) throws { + guard let codeUnit = readHexEscape(in: text, index: &index, digits: 4) else { + throw PureYAML.Parsing.ParseError.invalidEscape(sequence: "\\u", line: line) } - let surrogatePair = readSurrogatePairIfNeeded(high: codeUnit, in: text, index: &index) - if let scalar = surrogatePair { + if let scalar = readSurrogatePairIfNeeded(high: codeUnit, in: text, index: &index) { output.unicodeScalars.append(scalar) } else if let scalar = UnicodeScalar(codeUnit) { output.unicodeScalars.append(scalar) } else { - output += text[escapeStart ... index] + throw PureYAML.Parsing.ParseError.invalidEscape(sequence: "\\u", line: line) } } @@ -261,7 +315,7 @@ extension PureYAML.Parsing.Parser { } text.formIndex(after: &probe) guard probe < text.endIndex, text[probe] == "u", - let low = readUnicodeEscapeCodeUnit(in: text, index: &probe), + let low = readHexEscape(in: text, index: &probe, digits: 4), isLowSurrogate(low) else { return nil @@ -271,14 +325,18 @@ extension PureYAML.Parsing.Parser { return UnicodeScalar(value) } - func readUnicodeEscapeCodeUnit( + /// Reads `digits` hex characters following the escape type character. The index enters at the + /// type character and is left on the final hex digit; returns nil if fewer than `digits` hex + /// characters are available. + func readHexEscape( in text: Substring, index: inout Substring.Index, + digits: Int, ) -> UInt32? { var probe = index text.formIndex(after: &probe) var hex = "" - for _ in 0 ..< 4 { + for _ in 0 ..< digits { guard probe < text.endIndex, text[probe].isHexDigit else { return nil } diff --git a/Tests/DoubleQuotedEscapeTests.swift b/Tests/DoubleQuotedEscapeTests.swift new file mode 100644 index 0000000..27b866d --- /dev/null +++ b/Tests/DoubleQuotedEscapeTests.swift @@ -0,0 +1,50 @@ +@testable import PureYAML +import Testing + +@Suite("Double-quoted escape sequences") +struct DoubleQuotedEscapeTests { + @Test("Every YAML 1.2 escape decodes to its character", arguments: [ + ("\"\\0\"\n", "\u{00}"), + ("\"\\a\"\n", "\u{07}"), + ("\"\\b\"\n", "\u{08}"), + ("\"\\t\"\n", "\t"), + ("\"\\n\"\n", "\n"), + ("\"\\v\"\n", "\u{0B}"), + ("\"\\f\"\n", "\u{0C}"), + ("\"\\r\"\n", "\r"), + ("\"\\e\"\n", "\u{1B}"), + ("\"a\\ b\"\n", "a b"), + ("\"\\\"\"\n", "\""), + ("\"\\/\"\n", "/"), + ("\"\\\\\"\n", "\\"), + ("\"\\N\"\n", "\u{85}"), + ("\"\\_\"\n", "\u{A0}"), + ("\"\\L\"\n", "\u{2028}"), + ("\"\\P\"\n", "\u{2029}"), + ("\"\\x41\"\n", "A"), + ("\"\\u0041\"\n", "A"), + ("\"\\U0001F600\"\n", "\u{1F600}"), + ]) + func test_validEscapeDecodes(_ yaml: String, _ expected: String) throws { + #expect(try PureYAML.parse(yaml) == .string(expected)) + } + + @Test("An unknown or truncated escape is rejected", arguments: [ + "\"\\.\"\n", // not an escape character + "\"\\'\"\n", // single quote is not escaped in a double-quoted scalar + "\"\\q\"\n", // unknown letter + "\"\\x4\"\n", // truncated hex + "\"\\U12\"\n", // truncated 32-bit hex + ]) + func test_invalidEscapeIsRejected(_ yaml: String) throws { + #expect(throws: PureYAML.Parsing.ParseError.self) { + try PureYAML.parse(yaml) + } + } + + @Test("An escaped line break joins the lines with no separator") + func test_escapedLineBreakJoinsLines() throws { + #expect(try PureYAML.parse("\"a\\\n b\"\n") == .string("ab")) + #expect(try PureYAML.parse("\"one \\\n two\"\n") == .string("one two")) + } +}