A block scalar's leading empty lines must not be more indented than its
first non-empty line, so a blank line wider than the content (a 5-space
blank above 2-space content, or 2,3-space blanks above 1-space content) is
malformed. The scanner buffered the leading blanks without comparing their
indentation to the content, folding such input into the value.
The scanner now records the widest leading blank and rejects the scalar
when it exceeds the content indentation, established for both auto-detected
and explicit indicators. Indentation is counted in leading spaces, not
tab-expanded width, so a tab on a blank line ('␣\t') is one space rather
than a jump to the next tab stop, keeping such valid block scalars intact.
Measured against the official YAML test-suite, parse-level conformance
rises from 270/333 (81.1%) to 273/333 (82.0%) with no regression.
305 lines
12 KiB
Swift
305 lines
12 KiB
Swift
extension PureYAML.Parsing {
|
|
struct Scanner {
|
|
init() {}
|
|
|
|
func scan(_ yaml: String) throws -> [Token] {
|
|
var state = State(reader: Reader(yaml))
|
|
state.append(.streamStart)
|
|
|
|
while !state.reader.isAtEnd {
|
|
if isLineBreak(state.reader.peek()) {
|
|
state.reader.advance()
|
|
captureBlockScalarBlankLine(&state)
|
|
state.isAtLineStart = true
|
|
state.sawContentOnLine = false
|
|
state.sawMappingValueOnLine = false
|
|
continue
|
|
}
|
|
|
|
if state.isAtLineStart {
|
|
try scanIndentation(&state)
|
|
if state.reader.isAtEnd {
|
|
break
|
|
}
|
|
if isLineBreak(state.reader.peek()) {
|
|
continue
|
|
}
|
|
if try scanDirectiveOrDocumentMarker(&state) {
|
|
continue
|
|
}
|
|
state.isAtLineStart = false
|
|
}
|
|
|
|
try scanToken(&state)
|
|
}
|
|
|
|
closeIndentation(&state)
|
|
try enforceDirectiveHasDocument(state)
|
|
state.append(.streamEnd)
|
|
return state.tokens
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
struct State {
|
|
var reader: PureYAML.Parsing.Reader
|
|
var tokens: [PureYAML.Parsing.Token] = []
|
|
var indentation: [Int] = [0]
|
|
var isAtLineStart = true
|
|
var hasDocumentContent = false
|
|
var hasDocumentStartMarker = false
|
|
var isDocumentClosed = false
|
|
var hasYAMLDirective = false
|
|
// The line of a directive awaiting its document. A directive must be followed by an explicit
|
|
// `---`; this is set when one is scanned, cleared at a document start, and checked at a
|
|
// document end and at stream end so a directive with no document is rejected.
|
|
var pendingDirectiveLine: Int?
|
|
var flowDepth = 0
|
|
var flowContexts: [PureYAML.Parsing.ScannerFlowContext] = []
|
|
var validImplicitIndentation: Set<Int> = []
|
|
var pendingBlockScalarIndentation = false
|
|
var pendingBlockScalarIndentationIndicator: Int?
|
|
var pendingBlockScalarLeadingBlankLines = 0
|
|
/// The widest indentation seen on a block scalar's leading blank lines (before its first
|
|
/// content line). The spec forbids a leading empty line from being more indented than the
|
|
/// content, so this is compared to the content indentation once that is known.
|
|
var pendingBlockScalarMaxBlankIndent = 0
|
|
// The count of leading space characters (before any tab) from the most recent indentation
|
|
// scan. Indentation is measured in spaces, so this, not the tab-expanded width, is what the
|
|
// leading-blank comparison uses: a tab is not indentation.
|
|
var lastLeadingSpaceCount = 0
|
|
var blockScalarIndentation: Int?
|
|
var blockScalarContentPrefix = ""
|
|
/// Whether a real token has been scanned on the current physical line. Reset when the scan
|
|
/// loop consumes a line break and set as each token is scanned, so the scanner can tell a
|
|
/// comment that opens a line (valid) from one glued to earlier content on it (invalid). A
|
|
/// simple flag, not a token-line comparison, because a blank line inside a block scalar
|
|
/// emits an empty scalar attributed to the next line, which would read as phantom content.
|
|
var sawContentOnLine = false
|
|
// Whether a block-level mapping-value indicator (`:`) has been emitted on the current line.
|
|
// Once one has, a later scalar followed by `:` is not a new implicit key: block style allows
|
|
// only one mapping pair per line, so the document would be a forbidden inline nested mapping
|
|
// (`a: b: c`). Reset at each line break. Flow `:` do not set it, since flow allows several
|
|
// pairs on a line (`{a: b, c: d}`).
|
|
var sawMappingValueOnLine = false
|
|
var tagHandles = [
|
|
"!": "!",
|
|
"!!": "tag:yaml.org,2002:",
|
|
]
|
|
|
|
mutating func append(_ kind: PureYAML.Parsing.TokenKind) {
|
|
let mark = reader.mark
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: mark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func append(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
mark: PureYAML.Parsing.Mark,
|
|
endMark: PureYAML.Parsing.Mark,
|
|
) {
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: endMark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func markDocumentContentIfNeeded(_ kind: PureYAML.Parsing.TokenKind) {
|
|
switch kind {
|
|
case .comment, .dedent, .documentEnd, .documentStart, .indent, .streamEnd, .streamStart:
|
|
return
|
|
default:
|
|
hasDocumentContent = true
|
|
}
|
|
}
|
|
|
|
var isExpectingFlowMappingKey: Bool {
|
|
flowContexts.last?.kind == .mapping
|
|
&& flowContexts.last?.isExpectingKey == true
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
func scanToken(_ state: inout State) throws {
|
|
let columnBeforeSeparation = state.reader.mark.column
|
|
skipSeparation(&state)
|
|
let consumedSeparation = state.reader.mark.column > columnBeforeSeparation
|
|
guard let character = state.reader.peek(), !isLineBreak(character) else {
|
|
return
|
|
}
|
|
|
|
if state.blockScalarIndentation != nil {
|
|
scanBlockScalarContent(&state)
|
|
return
|
|
}
|
|
// A `#` begins a comment only at the start of a line's content or after whitespace. Glued to
|
|
// earlier same-line content (e.g. `"x"#c`, `]#c`, `>#c`) it is not a plain-scalar character
|
|
// either, so the document is malformed; a plain scalar's own `#` is consumed before reaching
|
|
// here, so this rejects only the genuinely invalid case.
|
|
if character == "#", !consumedSeparation, state.sawContentOnLine {
|
|
throw PureYAML.Parsing.ParseError.commentNotPrecededByWhitespace(
|
|
line: state.reader.mark.line,
|
|
column: state.reader.mark.column,
|
|
)
|
|
}
|
|
let hadContentOnLine = state.sawContentOnLine
|
|
state.sawContentOnLine = true
|
|
// An implicit-key mapping value (`:` preceded by its key on the same line) means the line
|
|
// now holds a pair, so a later scalar followed by `:` cannot start another: that would be a
|
|
// forbidden inline nested block mapping (`a: b: c`). A `:` that opens the line is an explicit-
|
|
// or null-key value indicator, whose value may legitimately be a compact mapping, so it does
|
|
// not set the flag.
|
|
if character == ":", hadContentOnLine, state.flowDepth == 0, isAtMappingValueIndicator(state: state) {
|
|
state.sawMappingValueOnLine = true
|
|
}
|
|
if try scanLineIndicatorToken(character, state: &state) {
|
|
return
|
|
}
|
|
if scanFlowToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanQuotedOrBlockScalarToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanNodePropertyToken(character, state: &state) {
|
|
return
|
|
}
|
|
scanPlainScalar(&state)
|
|
}
|
|
|
|
func scanLineIndicatorToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "#":
|
|
scanComment(&state)
|
|
return true
|
|
case "-" where isBlockEntry(state.reader):
|
|
// A block sequence cannot be an implicit key's value on the same line as its `:`
|
|
// (`key: - a`); the value must begin on a following line. As with an inline nested
|
|
// mapping, an explicit- or null-key `:` opens its line and does not set the flag, so a
|
|
// compact sequence value after it stays valid.
|
|
if state.sawMappingValueOnLine, state.flowDepth == 0 {
|
|
throw PureYAML.Parsing.ParseError.blockSequenceAsInlineMappingValue(line: state.reader.mark.line)
|
|
}
|
|
scanFixedToken(.blockEntry, state: &state)
|
|
return true
|
|
case "?" where isIndicatorBoundary(state.reader.peek(offset: 1)):
|
|
scanFixedToken(.mappingKey, state: &state)
|
|
return true
|
|
case ":" where isAtMappingValueIndicator(state: state):
|
|
scanFixedToken(.mappingValue, state: &state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanFlowToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) -> Bool {
|
|
switch character {
|
|
case "[":
|
|
state.flowDepth += 1
|
|
state.flowContexts.append(.init(kind: .sequence, isExpectingKey: false))
|
|
scanFixedToken(.flowSequenceStart, state: &state)
|
|
return true
|
|
case "]":
|
|
state.flowDepth = max(0, state.flowDepth - 1)
|
|
scanFixedToken(.flowSequenceEnd, state: &state)
|
|
_ = state.flowContexts.popLast()
|
|
return true
|
|
case "{":
|
|
state.flowDepth += 1
|
|
state.flowContexts.append(.init(kind: .mapping, isExpectingKey: true))
|
|
scanFixedToken(.flowMappingStart, state: &state)
|
|
return true
|
|
case "}":
|
|
state.flowDepth = max(0, state.flowDepth - 1)
|
|
scanFixedToken(.flowMappingEnd, state: &state)
|
|
_ = state.flowContexts.popLast()
|
|
return true
|
|
case ",":
|
|
scanFixedToken(.flowEntry, state: &state)
|
|
if state.flowContexts.last?.kind == .mapping {
|
|
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = true
|
|
}
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanNodePropertyToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "&":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.anchor,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "*":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.alias,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAliasName,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "!":
|
|
try scanTag(&state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanQuotedOrBlockScalarToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "\"":
|
|
try scanQuotedScalar(.doubleQuoted, state: &state)
|
|
return true
|
|
case "'":
|
|
try scanQuotedScalar(.singleQuoted, state: &state)
|
|
return true
|
|
case "|":
|
|
scanBlockScalarHeader(.literal, state: &state)
|
|
return true
|
|
case ">":
|
|
scanBlockScalarHeader(.folded, state: &state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func skipSeparation(_ state: inout State) {
|
|
while state.reader.peek() == " " {
|
|
state.reader.advance()
|
|
}
|
|
}
|
|
|
|
func scanFixedToken(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
state: inout State,
|
|
) {
|
|
let start = state.reader.mark
|
|
state.reader.advance()
|
|
state.append(kind, mark: start, endMark: state.reader.mark)
|
|
if case .blockEntry = kind, hasNodeOnSameLine(afterCurrentPositionIn: state.reader) {
|
|
state.validImplicitIndentation.insert(start.column + 1)
|
|
}
|
|
if case .mappingValue = kind, state.flowContexts.last?.kind == .mapping {
|
|
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = false
|
|
}
|
|
}
|
|
}
|