Mihaela Mihaljevic
16da7248c7
Implement libyaml's simple-key mechanism for flow context: the scanner records a potential key position before a plain or quoted scalar, an anchor/tag, or a [/{ opener, and a later ':' splices a mappingKey token retroactively in front of that node. This lets a quoted scalar ("k":v), a nested collection ({k}:v), or a property-carrying node (&c c: d) become a flow mapping key. In flow context ':' is always a value indicator and '?' always an explicit-key indicator regardless of the next character (the block-only following-blank lookahead is disabled), so adjacent forms and trailing '?' parse, and multi-line plain keys fold before the ':' is read. The flow parser is rebuilt around the flow_sequence_entry / flow_mapping_key / flow_mapping_value productions, composing empty scalars for absent keys and values. The streaming token cursor now buffers an open flow collection's tokens until it closes so the retroactive key splice stays valid.
Flips 10 official YAML test suite cases (348/402): 4FJ6, 8KB6, 9BXH, 9MMW, CN3R, DFF7, FRK4, NJ66, UT92, CT4Q. Flow collections used as block mapping keys (LX3P, 6BFJ) need the splice in block context and stay tracked. Updates manifest, fixtures, counts, and roadmap.
365 lines
14 KiB
Swift
365 lines
14 KiB
Swift
extension PureYAML.Parsing {
|
|
struct Scanner {
|
|
init() {}
|
|
|
|
func scan(_ yaml: String) throws -> [Token] {
|
|
var state = State(reader: Reader(yaml))
|
|
state.append(.streamStart)
|
|
|
|
while !state.reader.isAtEnd {
|
|
if isLineBreak(state.reader.peek()) {
|
|
state.reader.advance()
|
|
state.isAtLineStart = true
|
|
if state.flowDepth == 0 {
|
|
state.simpleKeyAllowed = true
|
|
}
|
|
continue
|
|
}
|
|
|
|
if state.isAtLineStart {
|
|
try scanIndentation(&state)
|
|
if state.reader.isAtEnd {
|
|
break
|
|
}
|
|
if isLineBreak(state.reader.peek()) {
|
|
continue
|
|
}
|
|
if try scanDirectiveOrDocumentMarker(&state) {
|
|
continue
|
|
}
|
|
state.isAtLineStart = false
|
|
}
|
|
|
|
try scanToken(&state)
|
|
}
|
|
|
|
if state.hasPendingDirective {
|
|
throw PureYAML.Parsing.ParseError.directiveWithoutDocument(line: state.reader.mark.line)
|
|
}
|
|
closeIndentation(&state)
|
|
state.append(.streamEnd)
|
|
return state.tokens
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
struct State {
|
|
var reader: PureYAML.Parsing.Reader
|
|
var tokens: [PureYAML.Parsing.Token] = []
|
|
var indentation: [Int] = [0]
|
|
var isAtLineStart = true
|
|
var hasDocumentContent = false
|
|
var hasDocumentStartMarker = false
|
|
var isDocumentClosed = false
|
|
/// True once a `%YAML` directive has been scanned for the document that
|
|
/// the next `---` will start. A second `%YAML` before that `---` is a
|
|
/// duplicate and is rejected; the flag resets at every document boundary.
|
|
var sawYAMLDirective = false
|
|
/// True once any directive has been scanned in the current directive
|
|
/// section. A directive section must be terminated by a `---` document
|
|
/// start; reaching a `...` end marker or the end of the stream with a
|
|
/// directive still pending is an error.
|
|
var hasPendingDirective = false
|
|
var flowDepth = 0
|
|
/// Mirrors libyaml's `parser->indent`: the 0-based column of the innermost
|
|
/// open block collection (sequence or mapping), or -1 at the bare document
|
|
/// root. Plain-scalar continuation folding compares a continuation line's
|
|
/// indentation against this value, so a root scalar (block indent -1) folds
|
|
/// column-0 continuations while a scalar inside a collection only folds lines
|
|
/// indented past the collection. Tracked separately from ``indentation`` --
|
|
/// which records every line-indentation level -- because libyaml rolls this
|
|
/// only on block-collection indicators, not on every indentation increase.
|
|
var blockIndent = -1
|
|
/// Saved ``blockIndent`` values, mirroring libyaml's `parser->indents` stack,
|
|
/// so a dedent restores the enclosing collection's indent.
|
|
var blockIndents: [Int] = []
|
|
/// Mirrors libyaml's `simple_key_allowed`. In block context a tab may only
|
|
/// be consumed as whitespace when a simple key is NOT allowed (i.e. mid-line,
|
|
/// after a node has been emitted). It is true at the start of every block
|
|
/// line and immediately after a `-`, `?`, or `:` indicator, so a tab in those
|
|
/// positions provides block indentation and is rejected. See scanner.c:1948.
|
|
var simpleKeyAllowed = true
|
|
var flowContexts: [PureYAML.Parsing.ScannerFlowContext] = []
|
|
/// A stack of potential simple keys, one entry per open flow collection,
|
|
/// mirroring libyaml's `simple_keys`. The slot for the innermost flow level
|
|
/// records the most recent position a `:` could promote to a mapping key.
|
|
/// Empty outside flow context; block keys do not use this stack.
|
|
var flowSimpleKeys: [PureYAML.Parsing.ScannerFlowSimpleKey?] = []
|
|
var validImplicitIndentation: Set<Int> = []
|
|
var tagHandles = [
|
|
"!": "!",
|
|
"!!": "tag:yaml.org,2002:",
|
|
]
|
|
|
|
mutating func append(_ kind: PureYAML.Parsing.TokenKind) {
|
|
let mark = reader.mark
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: mark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func append(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
mark: PureYAML.Parsing.Mark,
|
|
endMark: PureYAML.Parsing.Mark,
|
|
) {
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: endMark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func markDocumentContentIfNeeded(_ kind: PureYAML.Parsing.TokenKind) {
|
|
switch kind {
|
|
case .alias, .anchor, .blockScalarHeader, .scalar, .tag:
|
|
// A node (or one of its properties) has appeared on this line, so a
|
|
// simple key is no longer allowed and a following tab is ordinary
|
|
// content rather than indentation. Mirrors libyaml setting
|
|
// simple_key_allowed = 0 after scanning these tokens.
|
|
hasDocumentContent = true
|
|
simpleKeyAllowed = false
|
|
case .comment, .dedent, .documentEnd, .documentStart, .indent, .streamEnd, .streamStart:
|
|
return
|
|
default:
|
|
hasDocumentContent = true
|
|
}
|
|
}
|
|
|
|
var isExpectingFlowMappingKey: Bool {
|
|
flowContexts.last?.kind == .mapping
|
|
&& flowContexts.last?.isExpectingKey == true
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
func scanToken(_ state: inout State) throws {
|
|
skipSeparation(&state)
|
|
try rejectTabSeparation(&state)
|
|
guard let character = state.reader.peek(), !isLineBreak(character) else {
|
|
return
|
|
}
|
|
|
|
if try scanLineIndicatorToken(character, state: &state) {
|
|
return
|
|
}
|
|
if scanFlowToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanQuotedOrBlockScalarToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanNodePropertyToken(character, state: &state) {
|
|
return
|
|
}
|
|
scanPlainScalar(&state)
|
|
}
|
|
|
|
func scanLineIndicatorToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "#":
|
|
scanComment(&state)
|
|
return true
|
|
case "-" where isBlockEntry(state.reader):
|
|
scanFixedToken(.blockEntry, state: &state)
|
|
if state.flowDepth == 0 {
|
|
state.simpleKeyAllowed = true
|
|
}
|
|
return true
|
|
case "?" where state.flowDepth > 0 || isIndicatorBoundary(state.reader.peek(offset: 1)):
|
|
// In flow context a `?` is always an explicit-key indicator (libyaml's
|
|
// fetch_more_tokens checks `flow_level || IS_BLANKZ_AT`); a pending
|
|
// simple key is abandoned and no implicit key may follow on this entry.
|
|
if state.flowDepth > 0 {
|
|
removeFlowSimpleKey(&state)
|
|
}
|
|
scanFixedToken(.mappingKey, state: &state)
|
|
if state.flowDepth == 0 {
|
|
state.simpleKeyAllowed = true
|
|
} else {
|
|
state.simpleKeyAllowed = false
|
|
}
|
|
return true
|
|
case ":" where state.flowDepth > 0 || isAtMappingValueIndicator(state: state):
|
|
// In flow context a `:` is always a value indicator regardless of what
|
|
// follows it (libyaml), so `"k":v` and `{k}:v` are key/value pairs. A
|
|
// pending simple key is spliced into a `mappingKey` before its node.
|
|
if state.flowDepth > 0 {
|
|
promoteFlowSimpleKey(&state)
|
|
}
|
|
scanFixedToken(.mappingValue, state: &state)
|
|
if state.flowDepth == 0 {
|
|
state.simpleKeyAllowed = true
|
|
} else {
|
|
state.simpleKeyAllowed = false
|
|
}
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanNodePropertyToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "&":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.anchor,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "*":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.alias,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAliasName,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "!":
|
|
try scanTag(&state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanQuotedOrBlockScalarToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "\"":
|
|
try scanQuotedScalar(.doubleQuoted, state: &state)
|
|
return true
|
|
case "'":
|
|
try scanQuotedScalar(.singleQuoted, state: &state)
|
|
return true
|
|
case "|":
|
|
try scanBlockScalarHeader(.literal, state: &state)
|
|
return true
|
|
case ">":
|
|
try scanBlockScalarHeader(.folded, state: &state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func skipSeparation(_ state: inout State) {
|
|
while state.reader.peek() == " " {
|
|
state.reader.advance()
|
|
}
|
|
}
|
|
|
|
func scanFixedToken(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
state: inout State,
|
|
) {
|
|
let start = state.reader.mark
|
|
// Mirror libyaml rolling the block indent when a block sequence entry (`-`)
|
|
// or an explicit `?` mapping key opens a collection at this column. The
|
|
// implicit-key case rolls in `appendMappingKeyIfNeeded` at the key column;
|
|
// the `:` value indicator does not roll again, because for a simple key the
|
|
// mapping indent was already set at the key's (shallower) column and rolling
|
|
// to the deeper `:` column would wrongly raise the continuation threshold.
|
|
switch kind {
|
|
case .blockEntry, .mappingKey:
|
|
rollBlockIndent(toColumn: start.column - 1, state: &state)
|
|
default:
|
|
break
|
|
}
|
|
state.reader.advance()
|
|
state.append(kind, mark: start, endMark: state.reader.mark)
|
|
if case .blockEntry = kind, hasNodeOnSameLine(afterCurrentPositionIn: state.reader) {
|
|
state.validImplicitIndentation.insert(start.column + 1)
|
|
}
|
|
if case .mappingValue = kind, state.flowContexts.last?.kind == .mapping {
|
|
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = false
|
|
}
|
|
}
|
|
|
|
func scanComment(_ state: inout State) {
|
|
let start = state.reader.mark
|
|
state.reader.advance()
|
|
if state.reader.peek() == " " {
|
|
state.reader.advance()
|
|
}
|
|
let value = state.reader.consume { character in
|
|
!isLineBreak(character)
|
|
}
|
|
state.append(.comment(value), mark: start, endMark: state.reader.mark)
|
|
}
|
|
|
|
func scanQuotedScalar(
|
|
_ style: PureYAML.Parsing.ScalarStyle,
|
|
state: inout State,
|
|
) throws {
|
|
let start = state.reader.mark
|
|
// A quoted scalar may be a flow simple key: record its position before the
|
|
// content is consumed so a following `:` can splice a key in front of it.
|
|
saveFlowSimpleKey(&state)
|
|
let quote: Character = style == .doubleQuoted ? "\"" : "'"
|
|
state.reader.advance()
|
|
var value = ""
|
|
var previousWasEscape = false
|
|
while let character = state.reader.peek() {
|
|
// A `---` or `...` document marker at column 0 cannot appear inside an
|
|
// open quoted scalar: it means the scalar was never closed before a
|
|
// document boundary (scanner.c:3058-3070). Reject it as unterminated.
|
|
if state.reader.mark.column == 1, isDocumentMarkerHere(state) {
|
|
throw PureYAML.Parsing.ParseError.unterminatedQuotedString(line: start.line)
|
|
}
|
|
if style == .singleQuoted, character == quote, state.reader.peek(offset: 1) == quote {
|
|
value.append(character)
|
|
state.reader.advance()
|
|
value.append(character)
|
|
state.reader.advance()
|
|
continue
|
|
}
|
|
if character == quote, !previousWasEscape {
|
|
break
|
|
}
|
|
value.append(character)
|
|
previousWasEscape = character == "\\" && style == .doubleQuoted && !previousWasEscape
|
|
state.reader.advance()
|
|
if character != "\\" {
|
|
previousWasEscape = false
|
|
}
|
|
}
|
|
guard state.reader.peek() == quote else {
|
|
throw PureYAML.Parsing.ParseError.unterminatedQuotedString(line: start.line)
|
|
}
|
|
state.reader.advance()
|
|
appendMappingKeyIfNeeded(start: start, state: &state)
|
|
state.append(.scalar(value: value, style: style), mark: start, endMark: state.reader.mark)
|
|
}
|
|
|
|
func scanPlainScalar(_ state: inout State) {
|
|
let start = state.reader.mark
|
|
// A plain scalar may be a flow simple key: record its position before the
|
|
// content is consumed so a following `:` can splice a key in front of it.
|
|
saveFlowSimpleKey(&state)
|
|
var value = ""
|
|
while let character = state.reader.peek() {
|
|
if shouldEndPlainScalar(character, state: state, accumulated: value) {
|
|
break
|
|
}
|
|
value.append(character)
|
|
state.reader.advance()
|
|
}
|
|
var trimmed = trimTrailingSpaces(value)
|
|
let isMappingKey = appendMappingKeyIfNeeded(start: start, state: &state)
|
|
if !isMappingKey {
|
|
trimmed = scanPlainScalarContinuations(
|
|
after: trimmed,
|
|
state: &state,
|
|
)
|
|
}
|
|
state.append(.scalar(value: trimmed, style: .plain), mark: start, endMark: state.reader.mark)
|
|
}
|
|
}
|