fix(parsing): accept tabs as separation, reject them as indentation
libyaml refuses to skip a tab in the inter-token whitespace whenever a simple key is allowed, so a tab used as valid separation (after a '-', '?', or ':' indicator, or before a flow collection) is reported as a character that cannot start a token. But a tab is legal in-line separation; it is only illegal as indentation. The scanner now skips tabs in scan_to_next_token and records in tabbedSeparation whether one preceded the pending token. A block entry, an explicit key, or an implicit mapping key promoted by ':' is rejected when reached through a tab (there the tab would be indentation), while a scalar value or a flow collection is accepted. Tab-indented keys and '-\t-' nested entries still reject. Surpasses libyaml on 6BCT, 6CA3, A2M4, DK95/00, Q5MG, Y79Y/010. Port score 299 -> 305.
This commit is contained in:
parent
8092897612
commit
b2d2d2f595
11
CHANGELOG.md
11
CHANGELOG.md
@ -22,6 +22,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- Accept a tab as in-line separation while still rejecting it as indentation. A
|
||||
tab after a `-`, `?`, or `:` indicator, or before a flow collection, is valid
|
||||
separation, but libyaml refuses to skip it whenever a simple key is allowed and
|
||||
reports "found character that cannot start any token". The scanner now skips
|
||||
tabs in the inter-token whitespace and records whether one preceded the token;
|
||||
a block indicator or an implicit mapping key reached through a tab is rejected
|
||||
(the tab would be indentation), while a scalar value or flow collection is
|
||||
accepted. So `- foo:\tbar`, `-\t-1`, `: -\tb`, `\t[ ]`, `\t{}`, and a
|
||||
space-then-tab continuation parse, while tab-indented keys and `-\t-` style
|
||||
nested entries are still rejected. Surpasses libyaml on 6BCT, 6CA3, A2M4,
|
||||
DK95/00, Q5MG, and Y79Y/010.
|
||||
- Parse a flow sequence entry that begins with `:` as a single-pair mapping with
|
||||
an empty key (`[ : v ]`). libyaml hands the leading VALUE token to its node
|
||||
parser and fails. The flow-sequence-entry parser now opens the implicit mapping
|
||||
|
||||
@ -14,15 +14,21 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
return indentEngine.tokens
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_scan_to_next_token`: skip a leading BOM, spaces (and
|
||||
/// tabs only in flow context or where a simple key is not allowed), a comment,
|
||||
/// and a single line break, which re-allows a simple key in the block context.
|
||||
/// Port of `yaml_parser_scan_to_next_token`: skip a leading BOM, spaces and
|
||||
/// tabs, a comment, and a single line break, which re-allows a simple key in
|
||||
/// the block context. A tab is valid in-line separation, so it is skipped
|
||||
/// here; `tabbedSeparation` records when one preceded the token so a block
|
||||
/// indicator or implicit key (where the tab would be indentation) can reject.
|
||||
private mutating func scanToNextToken() {
|
||||
while true {
|
||||
tabbedSeparation = false
|
||||
if reader.column == 0, reader.peek() == "\u{FEFF}" {
|
||||
reader.skip()
|
||||
}
|
||||
while reader.peek() == " " || ((flowLevel > 0 || !simpleKeyAllowed) && reader.peek() == "\t") {
|
||||
while reader.peek() == " " || Classify.isTab(reader.peek()) {
|
||||
if Classify.isTab(reader.peek()), flowLevel == 0, simpleKeyAllowed {
|
||||
tabbedSeparation = true
|
||||
}
|
||||
reader.skip()
|
||||
}
|
||||
// YAML 1.2: a line containing only whitespace is blank even when that
|
||||
@ -105,12 +111,12 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
}
|
||||
switch character {
|
||||
case "-" where Classify.isBlankZ(reader.peek(1)):
|
||||
fetchBlockEntry()
|
||||
try fetchBlockEntry()
|
||||
case "?" where Classify.isBlankZ(reader.peek(1)) || (flowLevel > 0 && Classify.isFlowIndicator(reader.peek(1))):
|
||||
// In flow context libyaml treats every `?` as a key indicator, but per
|
||||
// `ns-plain-first` a `?` followed by a flow-safe character (anything but
|
||||
// whitespace, end, or a flow indicator) begins a plain scalar (`?foo`).
|
||||
fetchKey()
|
||||
try fetchKey()
|
||||
case ":" where Classify.isBlankZ(reader.peek(1))
|
||||
|| (flowLevel > 0 && (Classify.isFlowIndicator(reader.peek(1)) || flowKeyIsJSON)):
|
||||
// In flow context libyaml makes every `:` a value indicator. Per the
|
||||
@ -133,6 +139,17 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
}
|
||||
}
|
||||
|
||||
/// The error raised when a tab is used where indentation is required (before a
|
||||
/// block indicator or an implicit mapping key). A tab is valid separation but
|
||||
/// never indentation.
|
||||
private func tabIndentationError() -> PureYAML.Parsing.LibYAML.ScannerError {
|
||||
ScannerError(
|
||||
context: "while scanning for the next token",
|
||||
problem: "found a tab character where an indentation space is expected",
|
||||
mark: reader.mark,
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether this flow level's pending simple key is a JSON (quoted) node, which
|
||||
/// lets an adjacent `:` open its value without a separating space.
|
||||
private var flowKeyIsJSON: Bool {
|
||||
@ -283,7 +300,10 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_fetch_block_entry`.
|
||||
private mutating func fetchBlockEntry() {
|
||||
private mutating func fetchBlockEntry() throws {
|
||||
if tabbedSeparation {
|
||||
throw tabIndentationError()
|
||||
}
|
||||
if flowLevel == 0 {
|
||||
indentEngine.rollIndent(column: reader.column, number: -1, type: .blockSequenceStart, mark: reader.mark)
|
||||
}
|
||||
@ -295,7 +315,10 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_fetch_key`.
|
||||
private mutating func fetchKey() {
|
||||
private mutating func fetchKey() throws {
|
||||
if tabbedSeparation {
|
||||
throw tabIndentationError()
|
||||
}
|
||||
if flowLevel == 0 {
|
||||
indentEngine.rollIndent(column: reader.column, number: -1, type: .blockMappingStart, mark: reader.mark)
|
||||
}
|
||||
@ -312,6 +335,9 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
private mutating func fetchValue() throws {
|
||||
let top = simpleKeys.count - 1
|
||||
if simpleKeys[top].possible {
|
||||
if simpleKeys[top].tabbed {
|
||||
throw tabIndentationError()
|
||||
}
|
||||
let key = simpleKeys[top]
|
||||
indentEngine.insert(Token(type: .key, startMark: key.mark, endMark: key.mark), atTokenNumber: key.tokenNumber)
|
||||
indentEngine.rollIndent(column: key.mark.column, number: key.tokenNumber, type: .blockMappingStart, mark: key.mark)
|
||||
|
||||
@ -14,6 +14,10 @@ extension PureYAML.Parsing.LibYAML {
|
||||
var simpleKeys: [SimpleKey] = [.empty]
|
||||
/// `parser->simple_key_allowed`. True at stream start.
|
||||
var simpleKeyAllowed = true
|
||||
/// Whether a tab appeared in the indentation before the pending token. A
|
||||
/// tab is valid in-line separation but never indentation, so a block
|
||||
/// indicator or implicit mapping key reached through one is an error.
|
||||
var tabbedSeparation = false
|
||||
/// `parser->mark`, kept in sync with the reader at each fetch step.
|
||||
var mark: PureYAML.Parsing.Mark = .start
|
||||
/// `parser->stream_start_produced`.
|
||||
@ -74,6 +78,7 @@ extension PureYAML.Parsing.LibYAML {
|
||||
required: required,
|
||||
tokenNumber: indentEngine.tokensParsed + indentEngine.tokens.count,
|
||||
mark: mark,
|
||||
tabbed: tabbedSeparation,
|
||||
)
|
||||
try removeSimpleKey()
|
||||
simpleKeys[simpleKeys.count - 1] = key
|
||||
|
||||
@ -15,6 +15,10 @@ extension PureYAML.Parsing.LibYAML {
|
||||
/// adjacent to such a key is a value indicator even without a following
|
||||
/// space, per the YAML flow "adjacent value" rule.
|
||||
var isJSON = false
|
||||
/// Whether a tab preceded this key in its indentation. A tab may be
|
||||
/// separation but never indentation, so promoting such a key to a block
|
||||
/// mapping key (a `:` follows) is an error.
|
||||
var tabbed = false
|
||||
|
||||
/// The empty entry pushed for a new flow level (`{ 0, 0, 0, {0,0,0} }`).
|
||||
static var empty: Self {
|
||||
|
||||
@ -12,7 +12,7 @@ struct CanonicalEventConformanceTests {
|
||||
/// Valid suite cases whose event stream PureYAML does NOT yet emit exactly.
|
||||
/// This is the authoritative gap set: a fix removes an id (and the test fails
|
||||
/// until it is removed), a regression adds one (and the test fails). Of these,
|
||||
/// 7 currently throw on valid input and 2 emit a wrong event stream.
|
||||
/// 1 currently throws on valid input and 2 emit a wrong event stream.
|
||||
static let knownEventGaps: Set<String> = [
|
||||
// Gap set for the ported libyaml engine. Verified against the libyaml
|
||||
// reference dumper: on all but four of these the port matches libyaml
|
||||
@ -21,11 +21,7 @@ struct CanonicalEventConformanceTests {
|
||||
// itself handles.
|
||||
//
|
||||
// false-rejects (parseEvents throws on valid input, as libyaml does)
|
||||
"6BCT", "6CA3",
|
||||
"A2M4", "DK95/00",
|
||||
"DK95/04",
|
||||
"Q5MG",
|
||||
"Y79Y/010",
|
||||
// wrong event stream: the port matches libyaml but both deviate from the
|
||||
// canonical events (libyaml quirks to be surpassed).
|
||||
"JEF9/02", "W4TN",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user