Mihaela M
b45b7a4b7d
An alias may stand where a mapping key is expected (`*a : value`). The scanner emits the implicit-key indicator before a scalar key that a `:` follows, but not before an alias, so the parser, which already accepts an alias node in key position, never entered the pair and rejected the alias as an unexpected token. The scanner now emits the same indicator before an alias that a `:` (after optional spaces) follows, in block context. A `:` written directly against the alias stays part of its name, since `:` is a valid anchor-name character, so only a space-separated `:` marks the alias as a key. Fixes yaml-test-suite E76Z and 26DV; raises parse-level conformance to 310/333 (93.1%) and value-level to 203/203 (100.0%), with the should-error rate unchanged.
398 lines
18 KiB
Swift
398 lines
18 KiB
Swift
extension PureYAML.Parsing {
|
|
struct Scanner {
|
|
init() {}
|
|
|
|
func scan(_ yaml: String) throws -> [Token] {
|
|
var state = State(reader: Reader(yaml))
|
|
state.append(.streamStart)
|
|
|
|
while !state.reader.isAtEnd {
|
|
if isLineBreak(state.reader.peek()) {
|
|
state.reader.advance()
|
|
captureBlockScalarBlankLine(&state)
|
|
state.isAtLineStart = true
|
|
state.sawContentOnLine = false
|
|
state.sawMappingValueOnLine = false
|
|
state.sawFlowCloseOnLine = false
|
|
continue
|
|
}
|
|
|
|
if state.isAtLineStart {
|
|
try scanIndentation(&state)
|
|
if state.reader.isAtEnd {
|
|
break
|
|
}
|
|
if isLineBreak(state.reader.peek()) {
|
|
continue
|
|
}
|
|
if try scanDirectiveOrDocumentMarker(&state) {
|
|
continue
|
|
}
|
|
state.isAtLineStart = false
|
|
}
|
|
|
|
try scanToken(&state)
|
|
}
|
|
|
|
closeIndentation(&state)
|
|
try enforceDirectiveHasDocument(state)
|
|
state.append(.streamEnd)
|
|
return state.tokens
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
struct State {
|
|
var reader: PureYAML.Parsing.Reader
|
|
var tokens: [PureYAML.Parsing.Token] = []
|
|
var indentation: [Int] = [0]
|
|
var isAtLineStart = true
|
|
var hasDocumentContent = false
|
|
var hasDocumentStartMarker = false
|
|
var isDocumentClosed = false
|
|
var hasYAMLDirective = false
|
|
// The line of a directive awaiting its document. A directive must be followed by an explicit
|
|
// `---`; this is set when one is scanned, cleared at a document start, and checked at a
|
|
// document end and at stream end so a directive with no document is rejected.
|
|
var pendingDirectiveLine: Int?
|
|
var flowDepth = 0
|
|
var flowContexts: [PureYAML.Parsing.ScannerFlowContext] = []
|
|
var validImplicitIndentation: Set<Int> = []
|
|
var pendingBlockScalarIndentation = false
|
|
var pendingBlockScalarIndentationIndicator: Int?
|
|
/// Whether the pending block scalar is the document's own root node (no enclosing collection),
|
|
/// so its content may begin at column 0 (the document parent's indentation being -1).
|
|
var pendingBlockScalarIsRootNode = false
|
|
/// Whether the pending block scalar is an inline mapping value (`- key: |2`), whose parent
|
|
/// mapping sits at an implicit indentation off the main stack, so an explicit indentation
|
|
/// indicator is measured relative to that level rather than the stack's.
|
|
var pendingBlockScalarInlineMappingValue = false
|
|
/// The leading-whitespace characters of each blank line between a block scalar header and its
|
|
/// first content line. Stored verbatim (not just counted) so once the content indentation is
|
|
/// known, any surplus past it, notably a tab on an otherwise-blank line, survives as content.
|
|
var pendingBlockScalarLeadingBlankWhitespace: [String] = []
|
|
/// The widest indentation seen on a block scalar's leading blank lines (before its first
|
|
/// content line). The spec forbids a leading empty line from being more indented than the
|
|
/// content, so this is compared to the content indentation once that is known.
|
|
var pendingBlockScalarMaxBlankIndent = 0
|
|
/// The count of leading space characters (before any tab) from the most recent indentation
|
|
/// scan. Indentation is measured in spaces, so this, not the tab-expanded width, is what the
|
|
/// leading-blank comparison uses: a tab is not indentation.
|
|
var lastLeadingSpaceCount = 0
|
|
// The literal leading-whitespace characters from the most recent indentation scan. Block
|
|
// scalar content keeps tabs beyond its (space-only) indentation verbatim, so the surplus is
|
|
// reconstructed from these characters, not as spaces from the tab-expanded width.
|
|
var lastLeadingWhitespace = ""
|
|
var blockScalarIndentation: Int?
|
|
var blockScalarContentPrefix = ""
|
|
/// Whether a real token has been scanned on the current physical line. Reset when the scan
|
|
/// loop consumes a line break and set as each token is scanned, so the scanner can tell a
|
|
/// comment that opens a line (valid) from one glued to earlier content on it (invalid). A
|
|
/// simple flag, not a token-line comparison, because a blank line inside a block scalar
|
|
/// emits an empty scalar attributed to the next line, which would read as phantom content.
|
|
var sawContentOnLine = false
|
|
// Whether a block-level mapping-value indicator (`:`) has been emitted on the current line.
|
|
// Once one has, a later scalar followed by `:` is not a new implicit key: block style allows
|
|
// only one mapping pair per line, so the document would be a forbidden inline nested mapping
|
|
// (`a: b: c`). Reset at each line break. Flow `:` do not set it, since flow allows several
|
|
// pairs on a line (`{a: b, c: d}`).
|
|
var sawMappingValueOnLine = false
|
|
/// Whether a flow collection has closed on the current line at the top level. A block
|
|
/// sequence entry cannot follow a flow node on the same line (`{a: b}- x`), unlike a nested
|
|
/// block entry (`- -`), which is not preceded by a flow close. Reset at each line break.
|
|
var sawFlowCloseOnLine = false
|
|
// Whether an explicit `?` mapping key is awaiting its `:` value indicator. A `:` that opens
|
|
// a line is that key's value when one is pending, otherwise it is an implicit null key
|
|
// (`: value`). Only the `?` indicator sets this; a synthesized implicit key does not.
|
|
var pendingExplicitKey = false
|
|
var tagHandles = [
|
|
"!": "!",
|
|
"!!": "tag:yaml.org,2002:",
|
|
]
|
|
|
|
mutating func append(_ kind: PureYAML.Parsing.TokenKind) {
|
|
let mark = reader.mark
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: mark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func append(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
mark: PureYAML.Parsing.Mark,
|
|
endMark: PureYAML.Parsing.Mark,
|
|
) {
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: endMark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func markDocumentContentIfNeeded(_ kind: PureYAML.Parsing.TokenKind) {
|
|
switch kind {
|
|
case .comment, .dedent, .documentEnd, .documentStart, .indent, .streamEnd, .streamStart:
|
|
return
|
|
default:
|
|
hasDocumentContent = true
|
|
}
|
|
}
|
|
|
|
var isExpectingFlowMappingKey: Bool {
|
|
flowContexts.last?.kind == .mapping
|
|
&& flowContexts.last?.isExpectingKey == true
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
func scanToken(_ state: inout State) throws {
|
|
let columnBeforeSeparation = state.reader.mark.column
|
|
skipSeparation(&state)
|
|
let consumedSeparation = state.reader.mark.column > columnBeforeSeparation
|
|
guard let character = state.reader.peek(), !isLineBreak(character) else {
|
|
return
|
|
}
|
|
|
|
if state.blockScalarIndentation != nil {
|
|
scanBlockScalarContent(&state)
|
|
return
|
|
}
|
|
// A `#` begins a comment only at the start of a line's content or after whitespace. Glued to
|
|
// earlier same-line content (e.g. `"x"#c`, `]#c`, `>#c`) it is not a plain-scalar character
|
|
// either, so the document is malformed; a plain scalar's own `#` is consumed before reaching
|
|
// here, so this rejects only the genuinely invalid case.
|
|
if character == "#", !consumedSeparation, state.sawContentOnLine {
|
|
throw PureYAML.Parsing.ParseError.commentNotPrecededByWhitespace(
|
|
line: state.reader.mark.line,
|
|
column: state.reader.mark.column,
|
|
)
|
|
}
|
|
let hadContentOnLine = state.sawContentOnLine
|
|
state.sawContentOnLine = true
|
|
// An implicit-key mapping value (`:` preceded by its key on the same line) means the line
|
|
// now holds a pair, so a later scalar followed by `:` cannot start another: that would be a
|
|
// forbidden inline nested block mapping (`a: b: c`). A `:` that opens the line is an explicit-
|
|
// or null-key value indicator, whose value may legitimately be a compact mapping, so it does
|
|
// not set the flag.
|
|
if character == ":", hadContentOnLine, state.flowDepth == 0, isAtMappingValueIndicator(state: state) {
|
|
state.sawMappingValueOnLine = true
|
|
}
|
|
synthesizeNullMappingKeyIfNeeded(character, hadContentOnLine: hadContentOnLine, state: &state)
|
|
if try scanLineIndicatorToken(character, state: &state) {
|
|
return
|
|
}
|
|
if scanFlowToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanQuotedOrBlockScalarToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanNodePropertyToken(character, state: &state) {
|
|
return
|
|
}
|
|
// In a flow collection, a `-` that begins a plain scalar must be followed by a non-indicator
|
|
// character. Immediately before a flow indicator (`[-]`, `[-, -]`) it is not a valid scalar.
|
|
if state.flowDepth > 0, character == "-", let next = state.reader.peek(offset: 1), isFlowDelimiter(next) {
|
|
throw PureYAML.Parsing.ParseError.unexpectedToken(
|
|
expected: "a flow value",
|
|
actual: "-",
|
|
line: state.reader.mark.line,
|
|
column: state.reader.mark.column,
|
|
)
|
|
}
|
|
scanPlainScalar(&state)
|
|
}
|
|
|
|
/// A `:` that opens a line with no key before it is an implicit null mapping key (`: value`),
|
|
/// unless it is the value indicator of a pending explicit `?` key. Synthesizes the empty key and
|
|
/// its node so the block mapping machinery treats the pair uniformly.
|
|
func synthesizeNullMappingKeyIfNeeded(
|
|
_ character: Character,
|
|
hadContentOnLine: Bool,
|
|
state: inout State,
|
|
) {
|
|
guard character == ":", !hadContentOnLine, state.flowDepth == 0, !state.pendingExplicitKey,
|
|
isAtMappingValueIndicator(state: state)
|
|
else {
|
|
return
|
|
}
|
|
let mark = state.reader.mark
|
|
state.append(.mappingKey, mark: mark, endMark: mark)
|
|
state.append(.scalar(value: "", style: .plain), mark: mark, endMark: mark)
|
|
}
|
|
|
|
func scanLineIndicatorToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "#":
|
|
scanComment(&state)
|
|
return true
|
|
case "-" where isBlockEntry(state.reader):
|
|
// A block sequence cannot be an implicit key's value on the same line as its `:`
|
|
// (`key: - a`); the value must begin on a following line. As with an inline nested
|
|
// mapping, an explicit- or null-key `:` opens its line and does not set the flag, so a
|
|
// compact sequence value after it stays valid.
|
|
if state.sawMappingValueOnLine, state.flowDepth == 0 {
|
|
throw PureYAML.Parsing.ParseError.blockSequenceAsInlineMappingValue(line: state.reader.mark.line)
|
|
}
|
|
// A block sequence entry cannot follow a flow collection that closed earlier on the
|
|
// line (`{a: b}- x`); the entry would begin a sequence inline after a complete node.
|
|
if state.sawFlowCloseOnLine {
|
|
throw PureYAML.Parsing.ParseError.unexpectedToken(
|
|
expected: "a line break or flow indicator",
|
|
actual: "-",
|
|
line: state.reader.mark.line,
|
|
column: state.reader.mark.column,
|
|
)
|
|
}
|
|
scanFixedToken(.blockEntry, state: &state)
|
|
return true
|
|
case "?" where isIndicatorBoundary(state.reader.peek(offset: 1)):
|
|
scanFixedToken(.mappingKey, state: &state)
|
|
return true
|
|
case ":" where isAtMappingValueIndicator(state: state):
|
|
scanFixedToken(.mappingValue, state: &state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanFlowToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) -> Bool {
|
|
switch character {
|
|
case "[":
|
|
appendFlowImplicitKeyMarkerIfNeeded(state: &state)
|
|
state.flowDepth += 1
|
|
state.flowContexts.append(.init(kind: .sequence, isExpectingKey: false))
|
|
scanFixedToken(.flowSequenceStart, state: &state)
|
|
return true
|
|
case "]":
|
|
state.flowDepth = max(0, state.flowDepth - 1)
|
|
scanFixedToken(.flowSequenceEnd, state: &state)
|
|
_ = state.flowContexts.popLast()
|
|
if state.flowDepth == 0 {
|
|
state.sawFlowCloseOnLine = true
|
|
}
|
|
return true
|
|
case "{":
|
|
appendFlowImplicitKeyMarkerIfNeeded(state: &state)
|
|
state.flowDepth += 1
|
|
state.flowContexts.append(.init(kind: .mapping, isExpectingKey: true))
|
|
scanFixedToken(.flowMappingStart, state: &state)
|
|
return true
|
|
case "}":
|
|
state.flowDepth = max(0, state.flowDepth - 1)
|
|
scanFixedToken(.flowMappingEnd, state: &state)
|
|
_ = state.flowContexts.popLast()
|
|
if state.flowDepth == 0 {
|
|
state.sawFlowCloseOnLine = true
|
|
}
|
|
return true
|
|
case ",":
|
|
scanFixedToken(.flowEntry, state: &state)
|
|
if state.flowContexts.last?.kind == .mapping {
|
|
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = true
|
|
}
|
|
if !state.flowContexts.isEmpty {
|
|
state.flowContexts[state.flowContexts.count - 1].hasExplicitKey = false
|
|
}
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanNodePropertyToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "&":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.anchor,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "*":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.alias,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAliasName,
|
|
canBeImplicitKey: true,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "!":
|
|
try scanTag(&state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanQuotedOrBlockScalarToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "\"":
|
|
try scanQuotedScalar(.doubleQuoted, state: &state)
|
|
return true
|
|
case "'":
|
|
try scanQuotedScalar(.singleQuoted, state: &state)
|
|
return true
|
|
case "|":
|
|
scanBlockScalarHeader(.literal, state: &state)
|
|
return true
|
|
case ">":
|
|
scanBlockScalarHeader(.folded, state: &state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func skipSeparation(_ state: inout State) {
|
|
// Mid-line separation may be spaces or tabs: a tab is separation white space in YAML and is
|
|
// forbidden only as indentation, which the line-start indentation scan handles before this.
|
|
while state.reader.peek() == " " || state.reader.peek() == "\t" {
|
|
state.reader.advance()
|
|
}
|
|
}
|
|
|
|
func scanFixedToken(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
state: inout State,
|
|
) {
|
|
let start = state.reader.mark
|
|
state.reader.advance()
|
|
state.append(kind, mark: start, endMark: state.reader.mark)
|
|
if case .blockEntry = kind {
|
|
recordBlockEntryImplicitIndentation(start: start, state: &state)
|
|
}
|
|
if case .mappingValue = kind, state.flowContexts.last?.kind == .mapping {
|
|
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = false
|
|
}
|
|
// Only the explicit `?` indicator emits a mapping key through this path (an implicit key is
|
|
// synthesized directly), so track a block-level one as pending until its `:` value indicator
|
|
// is scanned. Flow `:`/`?` (inside a complex key, say) do not affect a pending block key.
|
|
if state.flowDepth == 0, case .mappingKey = kind {
|
|
state.pendingExplicitKey = true
|
|
}
|
|
if state.flowDepth == 0, case .mappingValue = kind {
|
|
state.pendingExplicitKey = false
|
|
}
|
|
// In flow the same role is tracked on the current flow context, so it is per-level and a
|
|
// nested collection's `:` does not affect an outer explicit key (`{? {a: b} : c}`).
|
|
if state.flowDepth > 0, !state.flowContexts.isEmpty, case .mappingKey = kind {
|
|
state.flowContexts[state.flowContexts.count - 1].hasExplicitKey = true
|
|
}
|
|
if state.flowDepth > 0, !state.flowContexts.isEmpty, case .mappingValue = kind {
|
|
state.flowContexts[state.flowContexts.count - 1].hasExplicitKey = false
|
|
}
|
|
}
|
|
}
|