Mihaela M
d79df8c4bc
A block sequence entry cannot follow a flow collection that closed earlier
on the same line ('- { y: z }- invalid', '- [a] - b'): the entry would
begin a sequence inline after a complete node. The scanner accepted it,
producing extra sequence items.
The scanner now records when a flow collection closes at the top level on
the current line and rejects a following block entry. The flag resets at
each line break, so a flow collection on its own line is unaffected, and a
nested block entry ('- -') has no preceding flow close, so it stays valid.
Measured against the official YAML test-suite, parse-level conformance
rises from 276/333 (82.9%) to 277/333 (83.2%) with no regression.
336 lines
14 KiB
Swift
336 lines
14 KiB
Swift
extension PureYAML.Parsing {
|
|
struct Scanner {
|
|
init() {}
|
|
|
|
func scan(_ yaml: String) throws -> [Token] {
|
|
var state = State(reader: Reader(yaml))
|
|
state.append(.streamStart)
|
|
|
|
while !state.reader.isAtEnd {
|
|
if isLineBreak(state.reader.peek()) {
|
|
state.reader.advance()
|
|
captureBlockScalarBlankLine(&state)
|
|
state.isAtLineStart = true
|
|
state.sawContentOnLine = false
|
|
state.sawMappingValueOnLine = false
|
|
state.sawFlowCloseOnLine = false
|
|
continue
|
|
}
|
|
|
|
if state.isAtLineStart {
|
|
try scanIndentation(&state)
|
|
if state.reader.isAtEnd {
|
|
break
|
|
}
|
|
if isLineBreak(state.reader.peek()) {
|
|
continue
|
|
}
|
|
if try scanDirectiveOrDocumentMarker(&state) {
|
|
continue
|
|
}
|
|
state.isAtLineStart = false
|
|
}
|
|
|
|
try scanToken(&state)
|
|
}
|
|
|
|
closeIndentation(&state)
|
|
try enforceDirectiveHasDocument(state)
|
|
state.append(.streamEnd)
|
|
return state.tokens
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
struct State {
|
|
var reader: PureYAML.Parsing.Reader
|
|
var tokens: [PureYAML.Parsing.Token] = []
|
|
var indentation: [Int] = [0]
|
|
var isAtLineStart = true
|
|
var hasDocumentContent = false
|
|
var hasDocumentStartMarker = false
|
|
var isDocumentClosed = false
|
|
var hasYAMLDirective = false
|
|
// The line of a directive awaiting its document. A directive must be followed by an explicit
|
|
// `---`; this is set when one is scanned, cleared at a document start, and checked at a
|
|
// document end and at stream end so a directive with no document is rejected.
|
|
var pendingDirectiveLine: Int?
|
|
var flowDepth = 0
|
|
var flowContexts: [PureYAML.Parsing.ScannerFlowContext] = []
|
|
var validImplicitIndentation: Set<Int> = []
|
|
var pendingBlockScalarIndentation = false
|
|
var pendingBlockScalarIndentationIndicator: Int?
|
|
var pendingBlockScalarLeadingBlankLines = 0
|
|
/// The widest indentation seen on a block scalar's leading blank lines (before its first
|
|
/// content line). The spec forbids a leading empty line from being more indented than the
|
|
/// content, so this is compared to the content indentation once that is known.
|
|
var pendingBlockScalarMaxBlankIndent = 0
|
|
// The count of leading space characters (before any tab) from the most recent indentation
|
|
// scan. Indentation is measured in spaces, so this, not the tab-expanded width, is what the
|
|
// leading-blank comparison uses: a tab is not indentation.
|
|
var lastLeadingSpaceCount = 0
|
|
var blockScalarIndentation: Int?
|
|
var blockScalarContentPrefix = ""
|
|
/// Whether a real token has been scanned on the current physical line. Reset when the scan
|
|
/// loop consumes a line break and set as each token is scanned, so the scanner can tell a
|
|
/// comment that opens a line (valid) from one glued to earlier content on it (invalid). A
|
|
/// simple flag, not a token-line comparison, because a blank line inside a block scalar
|
|
/// emits an empty scalar attributed to the next line, which would read as phantom content.
|
|
var sawContentOnLine = false
|
|
// Whether a block-level mapping-value indicator (`:`) has been emitted on the current line.
|
|
// Once one has, a later scalar followed by `:` is not a new implicit key: block style allows
|
|
// only one mapping pair per line, so the document would be a forbidden inline nested mapping
|
|
// (`a: b: c`). Reset at each line break. Flow `:` do not set it, since flow allows several
|
|
// pairs on a line (`{a: b, c: d}`).
|
|
var sawMappingValueOnLine = false
|
|
// Whether a flow collection has closed on the current line at the top level. A block
|
|
// sequence entry cannot follow a flow node on the same line (`{a: b}- x`), unlike a nested
|
|
// block entry (`- -`), which is not preceded by a flow close. Reset at each line break.
|
|
var sawFlowCloseOnLine = false
|
|
var tagHandles = [
|
|
"!": "!",
|
|
"!!": "tag:yaml.org,2002:",
|
|
]
|
|
|
|
mutating func append(_ kind: PureYAML.Parsing.TokenKind) {
|
|
let mark = reader.mark
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: mark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func append(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
mark: PureYAML.Parsing.Mark,
|
|
endMark: PureYAML.Parsing.Mark,
|
|
) {
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: endMark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func markDocumentContentIfNeeded(_ kind: PureYAML.Parsing.TokenKind) {
|
|
switch kind {
|
|
case .comment, .dedent, .documentEnd, .documentStart, .indent, .streamEnd, .streamStart:
|
|
return
|
|
default:
|
|
hasDocumentContent = true
|
|
}
|
|
}
|
|
|
|
var isExpectingFlowMappingKey: Bool {
|
|
flowContexts.last?.kind == .mapping
|
|
&& flowContexts.last?.isExpectingKey == true
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
func scanToken(_ state: inout State) throws {
|
|
let columnBeforeSeparation = state.reader.mark.column
|
|
skipSeparation(&state)
|
|
let consumedSeparation = state.reader.mark.column > columnBeforeSeparation
|
|
guard let character = state.reader.peek(), !isLineBreak(character) else {
|
|
return
|
|
}
|
|
|
|
if state.blockScalarIndentation != nil {
|
|
scanBlockScalarContent(&state)
|
|
return
|
|
}
|
|
// A `#` begins a comment only at the start of a line's content or after whitespace. Glued to
|
|
// earlier same-line content (e.g. `"x"#c`, `]#c`, `>#c`) it is not a plain-scalar character
|
|
// either, so the document is malformed; a plain scalar's own `#` is consumed before reaching
|
|
// here, so this rejects only the genuinely invalid case.
|
|
if character == "#", !consumedSeparation, state.sawContentOnLine {
|
|
throw PureYAML.Parsing.ParseError.commentNotPrecededByWhitespace(
|
|
line: state.reader.mark.line,
|
|
column: state.reader.mark.column,
|
|
)
|
|
}
|
|
let hadContentOnLine = state.sawContentOnLine
|
|
state.sawContentOnLine = true
|
|
// An implicit-key mapping value (`:` preceded by its key on the same line) means the line
|
|
// now holds a pair, so a later scalar followed by `:` cannot start another: that would be a
|
|
// forbidden inline nested block mapping (`a: b: c`). A `:` that opens the line is an explicit-
|
|
// or null-key value indicator, whose value may legitimately be a compact mapping, so it does
|
|
// not set the flag.
|
|
if character == ":", hadContentOnLine, state.flowDepth == 0, isAtMappingValueIndicator(state: state) {
|
|
state.sawMappingValueOnLine = true
|
|
}
|
|
if try scanLineIndicatorToken(character, state: &state) {
|
|
return
|
|
}
|
|
if scanFlowToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanQuotedOrBlockScalarToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanNodePropertyToken(character, state: &state) {
|
|
return
|
|
}
|
|
// In a flow collection, a `-` that begins a plain scalar must be followed by a non-indicator
|
|
// character. Immediately before a flow indicator (`[-]`, `[-, -]`) it is not a valid scalar.
|
|
if state.flowDepth > 0, character == "-", let next = state.reader.peek(offset: 1), isFlowDelimiter(next) {
|
|
throw PureYAML.Parsing.ParseError.unexpectedToken(
|
|
expected: "a flow value",
|
|
actual: "-",
|
|
line: state.reader.mark.line,
|
|
column: state.reader.mark.column,
|
|
)
|
|
}
|
|
scanPlainScalar(&state)
|
|
}
|
|
|
|
func scanLineIndicatorToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "#":
|
|
scanComment(&state)
|
|
return true
|
|
case "-" where isBlockEntry(state.reader):
|
|
// A block sequence cannot be an implicit key's value on the same line as its `:`
|
|
// (`key: - a`); the value must begin on a following line. As with an inline nested
|
|
// mapping, an explicit- or null-key `:` opens its line and does not set the flag, so a
|
|
// compact sequence value after it stays valid.
|
|
if state.sawMappingValueOnLine, state.flowDepth == 0 {
|
|
throw PureYAML.Parsing.ParseError.blockSequenceAsInlineMappingValue(line: state.reader.mark.line)
|
|
}
|
|
// A block sequence entry cannot follow a flow collection that closed earlier on the
|
|
// line (`{a: b}- x`); the entry would begin a sequence inline after a complete node.
|
|
if state.sawFlowCloseOnLine {
|
|
throw PureYAML.Parsing.ParseError.unexpectedToken(
|
|
expected: "a line break or flow indicator",
|
|
actual: "-",
|
|
line: state.reader.mark.line,
|
|
column: state.reader.mark.column,
|
|
)
|
|
}
|
|
scanFixedToken(.blockEntry, state: &state)
|
|
return true
|
|
case "?" where isIndicatorBoundary(state.reader.peek(offset: 1)):
|
|
scanFixedToken(.mappingKey, state: &state)
|
|
return true
|
|
case ":" where isAtMappingValueIndicator(state: state):
|
|
scanFixedToken(.mappingValue, state: &state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanFlowToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) -> Bool {
|
|
switch character {
|
|
case "[":
|
|
state.flowDepth += 1
|
|
state.flowContexts.append(.init(kind: .sequence, isExpectingKey: false))
|
|
scanFixedToken(.flowSequenceStart, state: &state)
|
|
return true
|
|
case "]":
|
|
state.flowDepth = max(0, state.flowDepth - 1)
|
|
scanFixedToken(.flowSequenceEnd, state: &state)
|
|
_ = state.flowContexts.popLast()
|
|
if state.flowDepth == 0 {
|
|
state.sawFlowCloseOnLine = true
|
|
}
|
|
return true
|
|
case "{":
|
|
state.flowDepth += 1
|
|
state.flowContexts.append(.init(kind: .mapping, isExpectingKey: true))
|
|
scanFixedToken(.flowMappingStart, state: &state)
|
|
return true
|
|
case "}":
|
|
state.flowDepth = max(0, state.flowDepth - 1)
|
|
scanFixedToken(.flowMappingEnd, state: &state)
|
|
_ = state.flowContexts.popLast()
|
|
if state.flowDepth == 0 {
|
|
state.sawFlowCloseOnLine = true
|
|
}
|
|
return true
|
|
case ",":
|
|
scanFixedToken(.flowEntry, state: &state)
|
|
if state.flowContexts.last?.kind == .mapping {
|
|
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = true
|
|
}
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanNodePropertyToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "&":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.anchor,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "*":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.alias,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAliasName,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "!":
|
|
try scanTag(&state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanQuotedOrBlockScalarToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "\"":
|
|
try scanQuotedScalar(.doubleQuoted, state: &state)
|
|
return true
|
|
case "'":
|
|
try scanQuotedScalar(.singleQuoted, state: &state)
|
|
return true
|
|
case "|":
|
|
scanBlockScalarHeader(.literal, state: &state)
|
|
return true
|
|
case ">":
|
|
scanBlockScalarHeader(.folded, state: &state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func skipSeparation(_ state: inout State) {
|
|
while state.reader.peek() == " " {
|
|
state.reader.advance()
|
|
}
|
|
}
|
|
|
|
func scanFixedToken(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
state: inout State,
|
|
) {
|
|
let start = state.reader.mark
|
|
state.reader.advance()
|
|
state.append(kind, mark: start, endMark: state.reader.mark)
|
|
if case .blockEntry = kind, hasNodeOnSameLine(afterCurrentPositionIn: state.reader) {
|
|
state.validImplicitIndentation.insert(start.column + 1)
|
|
}
|
|
if case .mappingValue = kind, state.flowContexts.last?.kind == .mapping {
|
|
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = false
|
|
}
|
|
}
|
|
}
|