From b2d2d2f595268fba4fa4b95c4fbc2a2acbe8baed Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Wed, 24 Jun 2026 19:14:12 +0200 Subject: [PATCH] fix(parsing): accept tabs as separation, reject them as indentation libyaml refuses to skip a tab in the inter-token whitespace whenever a simple key is allowed, so a tab used as valid separation (after a '-', '?', or ':' indicator, or before a flow collection) is reported as a character that cannot start a token. But a tab is legal in-line separation; it is only illegal as indentation. The scanner now skips tabs in scan_to_next_token and records in tabbedSeparation whether one preceded the pending token. A block entry, an explicit key, or an implicit mapping key promoted by ':' is rejected when reached through a tab (there the tab would be indentation), while a scalar value or a flow collection is accepted. Tab-indented keys and '-\t-' nested entries still reject. Surpasses libyaml on 6BCT, 6CA3, A2M4, DK95/00, Q5MG, Y79Y/010. Port score 299 -> 305. --- CHANGELOG.md | 11 ++++++ Sources/Parsing/LibYAML.Scanner.Fetch.swift | 42 +++++++++++++++++---- Sources/Parsing/LibYAML.Scanner.swift | 5 +++ Sources/Parsing/LibYAML.SimpleKey.swift | 4 ++ Tests/CanonicalEventConformanceTests.swift | 6 +-- 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbc8053..5743cb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Accept a tab as in-line separation while still rejecting it as indentation. A + tab after a `-`, `?`, or `:` indicator, or before a flow collection, is valid + separation, but libyaml refuses to skip it whenever a simple key is allowed and + reports "found character that cannot start any token". The scanner now skips + tabs in the inter-token whitespace and records whether one preceded the token; + a block indicator or an implicit mapping key reached through a tab is rejected + (the tab would be indentation), while a scalar value or flow collection is + accepted. So `- foo:\tbar`, `-\t-1`, `: -\tb`, `\t[ ]`, `\t{}`, and a + space-then-tab continuation parse, while tab-indented keys and `-\t-` style + nested entries are still rejected. Surpasses libyaml on 6BCT, 6CA3, A2M4, + DK95/00, Q5MG, and Y79Y/010. - Parse a flow sequence entry that begins with `:` as a single-pair mapping with an empty key (`[ : v ]`). libyaml hands the leading VALUE token to its node parser and fails. The flow-sequence-entry parser now opens the implicit mapping diff --git a/Sources/Parsing/LibYAML.Scanner.Fetch.swift b/Sources/Parsing/LibYAML.Scanner.Fetch.swift index bcb6c1c..43232d6 100644 --- a/Sources/Parsing/LibYAML.Scanner.Fetch.swift +++ b/Sources/Parsing/LibYAML.Scanner.Fetch.swift @@ -14,15 +14,21 @@ extension PureYAML.Parsing.LibYAML.Scanner { return indentEngine.tokens } - /// Port of `yaml_parser_scan_to_next_token`: skip a leading BOM, spaces (and - /// tabs only in flow context or where a simple key is not allowed), a comment, - /// and a single line break, which re-allows a simple key in the block context. + /// Port of `yaml_parser_scan_to_next_token`: skip a leading BOM, spaces and + /// tabs, a comment, and a single line break, which re-allows a simple key in + /// the block context. A tab is valid in-line separation, so it is skipped + /// here; `tabbedSeparation` records when one preceded the token so a block + /// indicator or implicit key (where the tab would be indentation) can reject. private mutating func scanToNextToken() { while true { + tabbedSeparation = false if reader.column == 0, reader.peek() == "\u{FEFF}" { reader.skip() } - while reader.peek() == " " || ((flowLevel > 0 || !simpleKeyAllowed) && reader.peek() == "\t") { + while reader.peek() == " " || Classify.isTab(reader.peek()) { + if Classify.isTab(reader.peek()), flowLevel == 0, simpleKeyAllowed { + tabbedSeparation = true + } reader.skip() } // YAML 1.2: a line containing only whitespace is blank even when that @@ -105,12 +111,12 @@ extension PureYAML.Parsing.LibYAML.Scanner { } switch character { case "-" where Classify.isBlankZ(reader.peek(1)): - fetchBlockEntry() + try fetchBlockEntry() case "?" where Classify.isBlankZ(reader.peek(1)) || (flowLevel > 0 && Classify.isFlowIndicator(reader.peek(1))): // In flow context libyaml treats every `?` as a key indicator, but per // `ns-plain-first` a `?` followed by a flow-safe character (anything but // whitespace, end, or a flow indicator) begins a plain scalar (`?foo`). - fetchKey() + try fetchKey() case ":" where Classify.isBlankZ(reader.peek(1)) || (flowLevel > 0 && (Classify.isFlowIndicator(reader.peek(1)) || flowKeyIsJSON)): // In flow context libyaml makes every `:` a value indicator. Per the @@ -133,6 +139,17 @@ extension PureYAML.Parsing.LibYAML.Scanner { } } + /// The error raised when a tab is used where indentation is required (before a + /// block indicator or an implicit mapping key). A tab is valid separation but + /// never indentation. + private func tabIndentationError() -> PureYAML.Parsing.LibYAML.ScannerError { + ScannerError( + context: "while scanning for the next token", + problem: "found a tab character where an indentation space is expected", + mark: reader.mark, + ) + } + /// Whether this flow level's pending simple key is a JSON (quoted) node, which /// lets an adjacent `:` open its value without a separating space. private var flowKeyIsJSON: Bool { @@ -283,7 +300,10 @@ extension PureYAML.Parsing.LibYAML.Scanner { } /// Port of `yaml_parser_fetch_block_entry`. - private mutating func fetchBlockEntry() { + private mutating func fetchBlockEntry() throws { + if tabbedSeparation { + throw tabIndentationError() + } if flowLevel == 0 { indentEngine.rollIndent(column: reader.column, number: -1, type: .blockSequenceStart, mark: reader.mark) } @@ -295,7 +315,10 @@ extension PureYAML.Parsing.LibYAML.Scanner { } /// Port of `yaml_parser_fetch_key`. - private mutating func fetchKey() { + private mutating func fetchKey() throws { + if tabbedSeparation { + throw tabIndentationError() + } if flowLevel == 0 { indentEngine.rollIndent(column: reader.column, number: -1, type: .blockMappingStart, mark: reader.mark) } @@ -312,6 +335,9 @@ extension PureYAML.Parsing.LibYAML.Scanner { private mutating func fetchValue() throws { let top = simpleKeys.count - 1 if simpleKeys[top].possible { + if simpleKeys[top].tabbed { + throw tabIndentationError() + } let key = simpleKeys[top] indentEngine.insert(Token(type: .key, startMark: key.mark, endMark: key.mark), atTokenNumber: key.tokenNumber) indentEngine.rollIndent(column: key.mark.column, number: key.tokenNumber, type: .blockMappingStart, mark: key.mark) diff --git a/Sources/Parsing/LibYAML.Scanner.swift b/Sources/Parsing/LibYAML.Scanner.swift index bff0fa6..845715e 100644 --- a/Sources/Parsing/LibYAML.Scanner.swift +++ b/Sources/Parsing/LibYAML.Scanner.swift @@ -14,6 +14,10 @@ extension PureYAML.Parsing.LibYAML { var simpleKeys: [SimpleKey] = [.empty] /// `parser->simple_key_allowed`. True at stream start. var simpleKeyAllowed = true + /// Whether a tab appeared in the indentation before the pending token. A + /// tab is valid in-line separation but never indentation, so a block + /// indicator or implicit mapping key reached through one is an error. + var tabbedSeparation = false /// `parser->mark`, kept in sync with the reader at each fetch step. var mark: PureYAML.Parsing.Mark = .start /// `parser->stream_start_produced`. @@ -74,6 +78,7 @@ extension PureYAML.Parsing.LibYAML { required: required, tokenNumber: indentEngine.tokensParsed + indentEngine.tokens.count, mark: mark, + tabbed: tabbedSeparation, ) try removeSimpleKey() simpleKeys[simpleKeys.count - 1] = key diff --git a/Sources/Parsing/LibYAML.SimpleKey.swift b/Sources/Parsing/LibYAML.SimpleKey.swift index c6b6c4a..ac438e6 100644 --- a/Sources/Parsing/LibYAML.SimpleKey.swift +++ b/Sources/Parsing/LibYAML.SimpleKey.swift @@ -15,6 +15,10 @@ extension PureYAML.Parsing.LibYAML { /// adjacent to such a key is a value indicator even without a following /// space, per the YAML flow "adjacent value" rule. var isJSON = false + /// Whether a tab preceded this key in its indentation. A tab may be + /// separation but never indentation, so promoting such a key to a block + /// mapping key (a `:` follows) is an error. + var tabbed = false /// The empty entry pushed for a new flow level (`{ 0, 0, 0, {0,0,0} }`). static var empty: Self { diff --git a/Tests/CanonicalEventConformanceTests.swift b/Tests/CanonicalEventConformanceTests.swift index 263d50a..04a8a4b 100644 --- a/Tests/CanonicalEventConformanceTests.swift +++ b/Tests/CanonicalEventConformanceTests.swift @@ -12,7 +12,7 @@ struct CanonicalEventConformanceTests { /// Valid suite cases whose event stream PureYAML does NOT yet emit exactly. /// This is the authoritative gap set: a fix removes an id (and the test fails /// until it is removed), a regression adds one (and the test fails). Of these, - /// 7 currently throw on valid input and 2 emit a wrong event stream. + /// 1 currently throws on valid input and 2 emit a wrong event stream. static let knownEventGaps: Set = [ // Gap set for the ported libyaml engine. Verified against the libyaml // reference dumper: on all but four of these the port matches libyaml @@ -21,11 +21,7 @@ struct CanonicalEventConformanceTests { // itself handles. // // false-rejects (parseEvents throws on valid input, as libyaml does) - "6BCT", "6CA3", - "A2M4", "DK95/00", "DK95/04", - "Q5MG", - "Y79Y/010", // wrong event stream: the port matches libyaml but both deviate from the // canonical events (libyaml quirks to be surpassed). "JEF9/02", "W4TN",