From 1b67030ed5e6300a32ec4f8d2134c31b9cfc61ab Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Wed, 24 Jun 2026 20:06:30 +0200 Subject: [PATCH] fix(parsing): reject a bare dash before a flow indicator Per ns-plain-first a '-' starts a plain scalar in flow only when a flow-safe character follows. A '-' before a flow indicator ('[-]', '[-, -]') cannot start a node, but the port (like libyaml) read it as the scalar '-'. Reject that form in the indicator dispatch. Extract scan_to_next_token, the comment validator, and blankLineAhead into LibYAML.Scanner.SkipToToken to keep the fetch file within the length limit. Surpasses libyaml on G5U8, YJV2; whole-suite conformance 390/402, canonical events 308/308. --- CHANGELOG.md | 4 + Sources/Parsing/LibYAML.Scanner.Fetch.swift | 77 +++---------------- .../Parsing/LibYAML.Scanner.SkipToToken.swift | 72 +++++++++++++++++ Tests/Fixtures/yaml-test-suite-manifest.yaml | 8 +- 4 files changed, 89 insertions(+), 72 deletions(-) create mode 100644 Sources/Parsing/LibYAML.Scanner.SkipToToken.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 6430b5c..8eb9b36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Reject a bare `-` before a flow indicator inside a flow collection. Per + `ns-plain-first` a `-` begins a plain scalar in flow only when a flow-safe + character follows; `[-]` and `[-, -]` are malformed, not a sequence of `-` + scalars. libyaml reads the `-` as a scalar. Surpasses libyaml on G5U8 and YJV2. - Reject a comment that is not preceded by whitespace. A `#` only begins a comment after a space, a tab, or at the start of a line; a `#` adjacent to the previous token (`key: "value"# c`, `[ a ]#c`, `[ a,#c`, a `>#c` block scalar header) is diff --git a/Sources/Parsing/LibYAML.Scanner.Fetch.swift b/Sources/Parsing/LibYAML.Scanner.Fetch.swift index 5e8bcc7..b3b6d02 100644 --- a/Sources/Parsing/LibYAML.Scanner.Fetch.swift +++ b/Sources/Parsing/LibYAML.Scanner.Fetch.swift @@ -14,64 +14,6 @@ extension PureYAML.Parsing.LibYAML.Scanner { return indentEngine.tokens } - /// Port of `yaml_parser_scan_to_next_token`: skip a leading BOM, spaces and - /// tabs, a comment, and a single line break, which re-allows a simple key in - /// the block context. A tab is valid in-line separation, so it is skipped - /// here; `tabbedSeparation` records when one preceded the token so a block - /// indicator or implicit key (where the tab would be indentation) can reject. - private mutating func scanToNextToken() throws { - while true { - tabbedSeparation = false - if reader.column == 0, reader.peek() == "\u{FEFF}" { - reader.skip() - } - while reader.peek() == " " || Classify.isTab(reader.peek()) { - if Classify.isTab(reader.peek()), flowLevel == 0, simpleKeyAllowed { - tabbedSeparation = true - } - reader.skip() - } - // YAML 1.2: a line containing only whitespace is blank even when that - // whitespace includes tabs. libyaml leaves a leading tab in place and - // later rejects it; instead consume the trailing blanks so the line - // break below is reached and the blank line is skipped. - if blankLineAhead() { - while reader.peek() == " " || Classify.isTab(reader.peek()) { - reader.skip() - } - } - if reader.peek() == "#" { - try consumeComment() - } - if Classify.isBreak(reader.peek()) { - reader.skipLine() - if flowLevel == 0 { - simpleKeyAllowed = true - } - } else { - break - } - } - } - - /// Consume a comment to the end of the line. A comment must be preceded by - /// whitespace or start the line; a `#` adjacent to the previous token (e.g. - /// `"x"#`, `]#`, `,#`) is not a comment. The plain-scalar scanner consumes its - /// own trailing blanks, so the character just before the `#` is the reliable - /// signal. libyaml accepts the adjacent form. - private mutating func consumeComment() throws { - if reader.column > 0, let previous = reader.peek(-1), !Classify.isBlank(previous) { - throw ScannerError( - context: "while scanning for the next token", - problem: "found a comment without a preceding space", - mark: reader.mark, - ) - } - while !Classify.isBreakZ(reader.peek()) { - reader.skip() - } - } - /// Port of `yaml_parser_fetch_next_token`: emit STREAM-START first, otherwise /// skip to the next token, expire stale simple keys, unroll the indentation to /// the current column, and dispatch on the leading character. Constructs not @@ -128,6 +70,15 @@ extension PureYAML.Parsing.LibYAML.Scanner { switch character { case "-" where Classify.isBlankZ(reader.peek(1)): try fetchBlockEntry() + case "-" where flowLevel > 0 && Classify.isFlowIndicator(reader.peek(1)): + // Per `ns-plain-first` a `-` begins a plain scalar in flow only when a + // flow-safe character follows; before a flow indicator (`[-]`, `[-,`) + // it cannot start a node. libyaml reads it as the scalar `-`. + throw ScannerError( + context: "while scanning for the next token", + problem: "found unexpected '-' indicator", + mark: reader.mark, + ) case "?" where Classify.isBlankZ(reader.peek(1)) || (flowLevel > 0 && Classify.isFlowIndicator(reader.peek(1))): // In flow context libyaml treats every `?` as a key indicator, but per // `ns-plain-first` a `?` followed by a flow-safe character (anything but @@ -379,16 +330,6 @@ extension PureYAML.Parsing.LibYAML.Scanner { indentEngine.enqueue(token) } - /// Whether the rest of the current line, from the current position, is only - /// spaces and tabs up to a line break or end of stream (a blank line). - func blankLineAhead() -> Bool { - var offset = 0 - while let character = reader.peek(offset), character == " " || Classify.isTab(character) { - offset += 1 - } - return Classify.isBreakZ(reader.peek(offset)) - } - private func isDocumentMarker(_ character: Unicode.Scalar) -> Bool { reader.peek(0) == character && reader.peek(1) == character diff --git a/Sources/Parsing/LibYAML.Scanner.SkipToToken.swift b/Sources/Parsing/LibYAML.Scanner.SkipToToken.swift new file mode 100644 index 0000000..27aaab0 --- /dev/null +++ b/Sources/Parsing/LibYAML.Scanner.SkipToToken.swift @@ -0,0 +1,72 @@ +extension PureYAML.Parsing.LibYAML.Scanner { + private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass + private typealias ScannerError = PureYAML.Parsing.LibYAML.ScannerError + + /// Port of `yaml_parser_scan_to_next_token`: skip a leading BOM, spaces and + /// tabs, a comment, and a single line break, which re-allows a simple key in + /// the block context. A tab is valid in-line separation, so it is skipped + /// here; `tabbedSeparation` records when one preceded the token so a block + /// indicator or implicit key (where the tab would be indentation) can reject. + mutating func scanToNextToken() throws { + while true { + tabbedSeparation = false + if reader.column == 0, reader.peek() == "\u{FEFF}" { + reader.skip() + } + while reader.peek() == " " || Classify.isTab(reader.peek()) { + if Classify.isTab(reader.peek()), flowLevel == 0, simpleKeyAllowed { + tabbedSeparation = true + } + reader.skip() + } + // YAML 1.2: a line containing only whitespace is blank even when that + // whitespace includes tabs. libyaml leaves a leading tab in place and + // later rejects it; instead consume the trailing blanks so the line + // break below is reached and the blank line is skipped. + if blankLineAhead() { + while reader.peek() == " " || Classify.isTab(reader.peek()) { + reader.skip() + } + } + if reader.peek() == "#" { + try consumeComment() + } + if Classify.isBreak(reader.peek()) { + reader.skipLine() + if flowLevel == 0 { + simpleKeyAllowed = true + } + } else { + break + } + } + } + + /// Consume a comment to the end of the line. A comment must be preceded by + /// whitespace or start the line; a `#` adjacent to the previous token (e.g. + /// `"x"#`, `]#`, `,#`) is not a comment. The plain-scalar scanner consumes its + /// own trailing blanks, so the character just before the `#` is the reliable + /// signal. libyaml accepts the adjacent form. + private mutating func consumeComment() throws { + if reader.column > 0, let previous = reader.peek(-1), !Classify.isBlank(previous) { + throw ScannerError( + context: "while scanning for the next token", + problem: "found a comment without a preceding space", + mark: reader.mark, + ) + } + while !Classify.isBreakZ(reader.peek()) { + reader.skip() + } + } + + /// Whether the rest of the current line, from the current position, is only + /// spaces and tabs up to a line break or end of stream (a blank line). + func blankLineAhead() -> Bool { + var offset = 0 + while let character = reader.peek(offset), character == " " || Classify.isTab(character) { + offset += 1 + } + return Classify.isBreakZ(reader.peek(offset)) + } +} diff --git a/Tests/Fixtures/yaml-test-suite-manifest.yaml b/Tests/Fixtures/yaml-test-suite-manifest.yaml index 3fdaa42..f8ff897 100644 --- a/Tests/Fixtures/yaml-test-suite-manifest.yaml +++ b/Tests/Fixtures/yaml-test-suite-manifest.yaml @@ -10,8 +10,8 @@ source: # oracle: json if in.json present, else none # status: passing (PureYAML conformant today) | gap (not yet conformant) total: 402 -passing: 388 -gap: 14 +passing: 390 +gap: 12 cases: - id: "229Q" name: "Spec Example 2.4. Sequence of Mappings" @@ -1730,7 +1730,7 @@ cases: localPath: yaml-test-suite/G5U8/in.yaml expected: reject oracle: none - status: gap + status: passing byteCount: 13 lineCount: 2 - id: "G7JE" @@ -3146,7 +3146,7 @@ cases: localPath: yaml-test-suite/YJV2/in.yaml expected: reject oracle: none - status: gap + status: passing byteCount: 4 lineCount: 1 - id: "Z67P"