Mihaela M
8d245ae335
A block key that is only a node property (`&a : v`, `!!str : v`) parsed to an `expectedNode` error. The scanner synthesized an empty mapping key for a bare `: v` line but not when the line's sole content before the `:` was an anchor or tag, so no key marker was emitted and the parser rejected the `:`. Track non-property content per line (a leading `&`/`!` does not count, and a block entry opens a fresh node position) and synthesize the empty key whenever only properties precede the `:`. The property tokens already bind to the synthesized empty scalar through the existing mapping machinery, so no parser change is needed. Relocate the synthesis helper to Scanner.Helpers to keep Scanner under the file-length limit. yaml-test-suite FH7J now passes (parse-level 313/333 to 314/333); PW8X still needs the empty-anchored explicit-value form. should-error rejections and value-level matches are unchanged, and a value-level regression test pins the produced tree for each property-only-key shape.
396 lines
18 KiB
Swift
396 lines
18 KiB
Swift
extension PureYAML.Parsing {
|
|
struct Scanner {
|
|
init() {}
|
|
|
|
func scan(_ yaml: String) throws -> [Token] {
|
|
var state = State(reader: Reader(yaml))
|
|
state.append(.streamStart)
|
|
|
|
while !state.reader.isAtEnd {
|
|
if isLineBreak(state.reader.peek()) {
|
|
state.reader.advance()
|
|
captureBlockScalarBlankLine(&state)
|
|
state.isAtLineStart = true
|
|
state.sawContentOnLine = false
|
|
state.sawNonPropertyContentOnLine = false
|
|
state.sawMappingValueOnLine = false
|
|
state.sawFlowCloseOnLine = false
|
|
continue
|
|
}
|
|
|
|
if state.isAtLineStart {
|
|
try scanIndentation(&state)
|
|
if state.reader.isAtEnd {
|
|
break
|
|
}
|
|
if isLineBreak(state.reader.peek()) {
|
|
continue
|
|
}
|
|
if try scanDirectiveOrDocumentMarker(&state) {
|
|
continue
|
|
}
|
|
state.isAtLineStart = false
|
|
}
|
|
|
|
try scanToken(&state)
|
|
}
|
|
|
|
closeIndentation(&state)
|
|
try enforceDirectiveHasDocument(state)
|
|
state.append(.streamEnd)
|
|
return state.tokens
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
struct State {
|
|
var reader: PureYAML.Parsing.Reader
|
|
var tokens: [PureYAML.Parsing.Token] = []
|
|
var indentation: [Int] = [0]
|
|
var isAtLineStart = true
|
|
var hasDocumentContent = false
|
|
var hasDocumentStartMarker = false
|
|
var isDocumentClosed = false
|
|
var hasYAMLDirective = false
|
|
// The line of a directive awaiting its document. A directive must be followed by an explicit
|
|
// `---`; this is set when one is scanned, cleared at a document start, and checked at a
|
|
// document end and at stream end so a directive with no document is rejected.
|
|
var pendingDirectiveLine: Int?
|
|
var flowDepth = 0
|
|
var flowContexts: [PureYAML.Parsing.ScannerFlowContext] = []
|
|
var validImplicitIndentation: Set<Int> = []
|
|
var pendingBlockScalarIndentation = false
|
|
var pendingBlockScalarIndentationIndicator: Int?
|
|
/// Whether the pending block scalar is the document's own root node (no enclosing collection),
|
|
/// so its content may begin at column 0 (the document parent's indentation being -1).
|
|
var pendingBlockScalarIsRootNode = false
|
|
/// Whether the pending block scalar is an inline mapping value (`- key: |2`), whose parent
|
|
/// mapping sits at an implicit indentation off the main stack, so an explicit indentation
|
|
/// indicator is measured relative to that level rather than the stack's.
|
|
var pendingBlockScalarInlineMappingValue = false
|
|
/// The leading-whitespace characters of each blank line between a block scalar header and its
|
|
/// first content line. Stored verbatim (not just counted) so once the content indentation is
|
|
/// known, any surplus past it, notably a tab on an otherwise-blank line, survives as content.
|
|
var pendingBlockScalarLeadingBlankWhitespace: [String] = []
|
|
/// The widest indentation seen on a block scalar's leading blank lines (before its first
|
|
/// content line). The spec forbids a leading empty line from being more indented than the
|
|
/// content, so this is compared to the content indentation once that is known.
|
|
var pendingBlockScalarMaxBlankIndent = 0
|
|
/// The count of leading space characters (before any tab) from the most recent indentation
|
|
/// scan. Indentation is measured in spaces, so this, not the tab-expanded width, is what the
|
|
/// leading-blank comparison uses: a tab is not indentation.
|
|
var lastLeadingSpaceCount = 0
|
|
// The literal leading-whitespace characters from the most recent indentation scan. Block
|
|
// scalar content keeps tabs beyond its (space-only) indentation verbatim, so the surplus is
|
|
// reconstructed from these characters, not as spaces from the tab-expanded width.
|
|
var lastLeadingWhitespace = ""
|
|
var blockScalarIndentation: Int?
|
|
var blockScalarContentPrefix = ""
|
|
/// Whether a real token has been scanned on the current physical line. Reset when the scan
|
|
/// loop consumes a line break and set as each token is scanned, so the scanner can tell a
|
|
/// comment that opens a line (valid) from one glued to earlier content on it (invalid). A
|
|
/// simple flag, not a token-line comparison, because a blank line inside a block scalar
|
|
/// emits an empty scalar attributed to the next line, which would read as phantom content.
|
|
var sawContentOnLine = false
|
|
/// Like ``sawContentOnLine`` but not set by a node property (an anchor or tag). A `:` whose
|
|
/// only preceding same-line content is a property (`&a :`, `!!str :`) is an empty implicit
|
|
/// key carrying that property, so a null mapping key is synthesized for it exactly as for a
|
|
/// bare `: value`. Reset at each line break.
|
|
var sawNonPropertyContentOnLine = false
|
|
// Whether a block-level mapping-value indicator (`:`) has been emitted on the current line.
|
|
// Once one has, a later scalar followed by `:` is not a new implicit key: block style allows
|
|
// only one mapping pair per line, so the document would be a forbidden inline nested mapping
|
|
// (`a: b: c`). Reset at each line break. Flow `:` do not set it, since flow allows several
|
|
// pairs on a line (`{a: b, c: d}`).
|
|
var sawMappingValueOnLine = false
|
|
/// Whether a flow collection has closed on the current line at the top level. A block
|
|
/// sequence entry cannot follow a flow node on the same line (`{a: b}- x`), unlike a nested
|
|
/// block entry (`- -`), which is not preceded by a flow close. Reset at each line break.
|
|
var sawFlowCloseOnLine = false
|
|
// Whether an explicit `?` mapping key is awaiting its `:` value indicator. A `:` that opens
|
|
// a line is that key's value when one is pending, otherwise it is an implicit null key
|
|
// (`: value`). Only the `?` indicator sets this; a synthesized implicit key does not.
|
|
var pendingExplicitKey = false
|
|
var tagHandles = [
|
|
"!": "!",
|
|
"!!": "tag:yaml.org,2002:",
|
|
]
|
|
|
|
mutating func append(_ kind: PureYAML.Parsing.TokenKind) {
|
|
let mark = reader.mark
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: mark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func append(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
mark: PureYAML.Parsing.Mark,
|
|
endMark: PureYAML.Parsing.Mark,
|
|
) {
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: endMark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func markDocumentContentIfNeeded(_ kind: PureYAML.Parsing.TokenKind) {
|
|
switch kind {
|
|
case .comment, .dedent, .documentEnd, .documentStart, .indent, .streamEnd, .streamStart:
|
|
return
|
|
default:
|
|
hasDocumentContent = true
|
|
}
|
|
}
|
|
|
|
var isExpectingFlowMappingKey: Bool {
|
|
flowContexts.last?.kind == .mapping
|
|
&& flowContexts.last?.isExpectingKey == true
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
func scanToken(_ state: inout State) throws {
|
|
let columnBeforeSeparation = state.reader.mark.column
|
|
skipSeparation(&state)
|
|
let consumedSeparation = state.reader.mark.column > columnBeforeSeparation
|
|
guard let character = state.reader.peek(), !isLineBreak(character) else {
|
|
return
|
|
}
|
|
|
|
if state.blockScalarIndentation != nil {
|
|
scanBlockScalarContent(&state)
|
|
return
|
|
}
|
|
// A `#` begins a comment only at the start of a line's content or after whitespace. Glued to
|
|
// earlier same-line content (e.g. `"x"#c`, `]#c`, `>#c`) it is not a plain-scalar character
|
|
// either, so the document is malformed; a plain scalar's own `#` is consumed before reaching
|
|
// here, so this rejects only the genuinely invalid case.
|
|
if character == "#", !consumedSeparation, state.sawContentOnLine {
|
|
throw PureYAML.Parsing.ParseError.commentNotPrecededByWhitespace(
|
|
line: state.reader.mark.line,
|
|
column: state.reader.mark.column,
|
|
)
|
|
}
|
|
let hadContentOnLine = state.sawContentOnLine
|
|
let hadNonPropertyContentOnLine = state.sawNonPropertyContentOnLine
|
|
state.sawContentOnLine = true
|
|
// A leading `&` or `!` begins a node property (anchor or tag), which is not itself a key or
|
|
// value; every other token character is real content. A property-only line followed by `:`
|
|
// is thus an empty implicit key (`&a :`), synthesized below like a bare `: value`.
|
|
if character != "&", character != "!" {
|
|
state.sawNonPropertyContentOnLine = true
|
|
}
|
|
// An implicit-key mapping value (`:` preceded by its key on the same line) means the line
|
|
// now holds a pair, so a later scalar followed by `:` cannot start another: that would be a
|
|
// forbidden inline nested block mapping (`a: b: c`). A `:` that opens the line is an explicit-
|
|
// or null-key value indicator, whose value may legitimately be a compact mapping, so it does
|
|
// not set the flag.
|
|
if character == ":", hadContentOnLine, state.flowDepth == 0, isAtMappingValueIndicator(state: state) {
|
|
state.sawMappingValueOnLine = true
|
|
}
|
|
synthesizeNullMappingKeyIfNeeded(character, hadNonPropertyContentOnLine: hadNonPropertyContentOnLine, state: &state)
|
|
if try scanLineIndicatorToken(character, state: &state) {
|
|
return
|
|
}
|
|
if scanFlowToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanQuotedOrBlockScalarToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanNodePropertyToken(character, state: &state) {
|
|
return
|
|
}
|
|
// In a flow collection, a `-` that begins a plain scalar must be followed by a non-indicator
|
|
// character. Immediately before a flow indicator (`[-]`, `[-, -]`) it is not a valid scalar.
|
|
if state.flowDepth > 0, character == "-", let next = state.reader.peek(offset: 1), isFlowDelimiter(next) {
|
|
throw PureYAML.Parsing.ParseError.unexpectedToken(
|
|
expected: "a flow value",
|
|
actual: "-",
|
|
line: state.reader.mark.line,
|
|
column: state.reader.mark.column,
|
|
)
|
|
}
|
|
scanPlainScalar(&state)
|
|
}
|
|
|
|
func scanLineIndicatorToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "#":
|
|
scanComment(&state)
|
|
return true
|
|
case "-" where isBlockEntry(state.reader):
|
|
// A block sequence cannot be an implicit key's value on the same line as its `:`
|
|
// (`key: - a`); the value must begin on a following line. As with an inline nested
|
|
// mapping, an explicit- or null-key `:` opens its line and does not set the flag, so a
|
|
// compact sequence value after it stays valid.
|
|
if state.sawMappingValueOnLine, state.flowDepth == 0 {
|
|
throw PureYAML.Parsing.ParseError.blockSequenceAsInlineMappingValue(line: state.reader.mark.line)
|
|
}
|
|
// A block sequence entry cannot follow a flow collection that closed earlier on the
|
|
// line (`{a: b}- x`); the entry would begin a sequence inline after a complete node.
|
|
if state.sawFlowCloseOnLine {
|
|
throw PureYAML.Parsing.ParseError.unexpectedToken(
|
|
expected: "a line break or flow indicator",
|
|
actual: "-",
|
|
line: state.reader.mark.line,
|
|
column: state.reader.mark.column,
|
|
)
|
|
}
|
|
scanFixedToken(.blockEntry, state: &state)
|
|
return true
|
|
case "?" where isIndicatorBoundary(state.reader.peek(offset: 1)):
|
|
scanFixedToken(.mappingKey, state: &state)
|
|
return true
|
|
case ":" where isAtMappingValueIndicator(state: state):
|
|
scanFixedToken(.mappingValue, state: &state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanFlowToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) -> Bool {
|
|
switch character {
|
|
case "[":
|
|
appendFlowImplicitKeyMarkerIfNeeded(state: &state)
|
|
state.flowDepth += 1
|
|
state.flowContexts.append(.init(kind: .sequence, isExpectingKey: false))
|
|
scanFixedToken(.flowSequenceStart, state: &state)
|
|
return true
|
|
case "]":
|
|
state.flowDepth = max(0, state.flowDepth - 1)
|
|
scanFixedToken(.flowSequenceEnd, state: &state)
|
|
_ = state.flowContexts.popLast()
|
|
if state.flowDepth == 0 {
|
|
state.sawFlowCloseOnLine = true
|
|
}
|
|
return true
|
|
case "{":
|
|
appendFlowImplicitKeyMarkerIfNeeded(state: &state)
|
|
state.flowDepth += 1
|
|
state.flowContexts.append(.init(kind: .mapping, isExpectingKey: true))
|
|
scanFixedToken(.flowMappingStart, state: &state)
|
|
return true
|
|
case "}":
|
|
state.flowDepth = max(0, state.flowDepth - 1)
|
|
scanFixedToken(.flowMappingEnd, state: &state)
|
|
_ = state.flowContexts.popLast()
|
|
if state.flowDepth == 0 {
|
|
state.sawFlowCloseOnLine = true
|
|
}
|
|
return true
|
|
case ",":
|
|
scanFixedToken(.flowEntry, state: &state)
|
|
if state.flowContexts.last?.kind == .mapping {
|
|
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = true
|
|
}
|
|
if !state.flowContexts.isEmpty {
|
|
state.flowContexts[state.flowContexts.count - 1].hasExplicitKey = false
|
|
}
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanNodePropertyToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "&":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.anchor,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "*":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.alias,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAliasName,
|
|
canBeImplicitKey: true,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "!":
|
|
try scanTag(&state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanQuotedOrBlockScalarToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "\"":
|
|
try scanQuotedScalar(.doubleQuoted, state: &state)
|
|
return true
|
|
case "'":
|
|
try scanQuotedScalar(.singleQuoted, state: &state)
|
|
return true
|
|
case "|":
|
|
scanBlockScalarHeader(.literal, state: &state)
|
|
return true
|
|
case ">":
|
|
scanBlockScalarHeader(.folded, state: &state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func skipSeparation(_ state: inout State) {
|
|
// Mid-line separation may be spaces or tabs: a tab is separation white space in YAML and is
|
|
// forbidden only as indentation, which the line-start indentation scan handles before this.
|
|
while state.reader.peek() == " " || state.reader.peek() == "\t" {
|
|
state.reader.advance()
|
|
}
|
|
}
|
|
|
|
func scanFixedToken(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
state: inout State,
|
|
) {
|
|
let start = state.reader.mark
|
|
state.reader.advance()
|
|
state.append(kind, mark: start, endMark: state.reader.mark)
|
|
if case .blockEntry = kind {
|
|
recordBlockEntryImplicitIndentation(start: start, state: &state)
|
|
// A block entry opens a fresh node position, so a property-only key after it on the same
|
|
// line (`- !!str : v`) is still an empty implicit key: the `-` is not content of the key.
|
|
state.sawNonPropertyContentOnLine = false
|
|
}
|
|
if case .mappingValue = kind, state.flowContexts.last?.kind == .mapping {
|
|
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = false
|
|
}
|
|
// Only the explicit `?` indicator emits a mapping key through this path (an implicit key is
|
|
// synthesized directly), so track a block-level one as pending until its `:` value indicator
|
|
// is scanned. Flow `:`/`?` (inside a complex key, say) do not affect a pending block key.
|
|
if state.flowDepth == 0, case .mappingKey = kind {
|
|
state.pendingExplicitKey = true
|
|
}
|
|
if state.flowDepth == 0, case .mappingValue = kind {
|
|
state.pendingExplicitKey = false
|
|
}
|
|
// In flow the same role is tracked on the current flow context, so it is per-level and a
|
|
// nested collection's `:` does not affect an outer explicit key (`{? {a: b} : c}`).
|
|
if state.flowDepth > 0, !state.flowContexts.isEmpty, case .mappingKey = kind {
|
|
state.flowContexts[state.flowContexts.count - 1].hasExplicitKey = true
|
|
}
|
|
if state.flowDepth > 0, !state.flowContexts.isEmpty, case .mappingValue = kind {
|
|
state.flowContexts[state.flowContexts.count - 1].hasExplicitKey = false
|
|
}
|
|
}
|
|
}
|