Mihaela Mihaljevic
1de1fcf2a8
Tighten error detection per the YAML 1.2 spec across four rules: (1) a tab may never supply block-structure indentation (line-start indentation, block-scalar indentation, and separation before a nested block node are rejected; a tab separating an indicator from a scalar value stays valid); (2) a ---/... document marker inside an open quoted scalar or flow collection is rejected instead of silently ending the document; (3) a %YAML/%TAG line that opens a second explicit document with no intervening ... is rejected; (4) an anchor cannot be attached to an alias node. This repairs 4 of the 5 error-detection regressions introduced by buckets 3-5 (MUS6/01, N782, SU74, Y79Y/009); 9MQT/01 is deferred (its invalidity is in same-line content after the --- marker, which needs document-content scanning that risks the baseline). Rejects 13 accepted-invalid suite cases and fixes 2 more by value (6BCT, Y79Y/010); 305/402 passing. BEHAVIOR CHANGE: tab-indented block structure (previously parsed permissively) is now rejected per spec; no real-world corpus seed relied on this. Updates the manifest, fixtures, counts, two affected tests (with spec justification), and the roadmap (bucket 6 marked PARTIAL).
398 lines
14 KiB
Swift
398 lines
14 KiB
Swift
extension PureYAML.Parsing {
|
|
struct Scanner {
|
|
init() {}
|
|
|
|
func scan(_ yaml: String) throws -> [Token] {
|
|
var state = State(reader: Reader(yaml))
|
|
state.append(.streamStart)
|
|
|
|
while !state.reader.isAtEnd {
|
|
if isLineBreak(state.reader.peek()) {
|
|
state.reader.advance()
|
|
state.isAtLineStart = true
|
|
if state.flowDepth == 0 {
|
|
state.simpleKeyAllowed = true
|
|
}
|
|
continue
|
|
}
|
|
|
|
if state.isAtLineStart {
|
|
try scanIndentation(&state)
|
|
if state.reader.isAtEnd {
|
|
break
|
|
}
|
|
if isLineBreak(state.reader.peek()) {
|
|
continue
|
|
}
|
|
if try scanDirectiveOrDocumentMarker(&state) {
|
|
continue
|
|
}
|
|
state.isAtLineStart = false
|
|
}
|
|
|
|
try scanToken(&state)
|
|
}
|
|
|
|
if state.hasPendingDirective {
|
|
throw PureYAML.Parsing.ParseError.directiveWithoutDocument(line: state.reader.mark.line)
|
|
}
|
|
closeIndentation(&state)
|
|
state.append(.streamEnd)
|
|
return state.tokens
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
struct State {
|
|
var reader: PureYAML.Parsing.Reader
|
|
var tokens: [PureYAML.Parsing.Token] = []
|
|
var indentation: [Int] = [0]
|
|
var isAtLineStart = true
|
|
var hasDocumentContent = false
|
|
var hasDocumentStartMarker = false
|
|
var isDocumentClosed = false
|
|
/// True once a `%YAML` directive has been scanned for the document that
|
|
/// the next `---` will start. A second `%YAML` before that `---` is a
|
|
/// duplicate and is rejected; the flag resets at every document boundary.
|
|
var sawYAMLDirective = false
|
|
/// True once any directive has been scanned in the current directive
|
|
/// section. A directive section must be terminated by a `---` document
|
|
/// start; reaching a `...` end marker or the end of the stream with a
|
|
/// directive still pending is an error.
|
|
var hasPendingDirective = false
|
|
var flowDepth = 0
|
|
/// Mirrors libyaml's `simple_key_allowed`. In block context a tab may only
|
|
/// be consumed as whitespace when a simple key is NOT allowed (i.e. mid-line,
|
|
/// after a node has been emitted). It is true at the start of every block
|
|
/// line and immediately after a `-`, `?`, or `:` indicator, so a tab in those
|
|
/// positions provides block indentation and is rejected. See scanner.c:1948.
|
|
var simpleKeyAllowed = true
|
|
var flowContexts: [PureYAML.Parsing.ScannerFlowContext] = []
|
|
var validImplicitIndentation: Set<Int> = []
|
|
var tagHandles = [
|
|
"!": "!",
|
|
"!!": "tag:yaml.org,2002:",
|
|
]
|
|
|
|
mutating func append(_ kind: PureYAML.Parsing.TokenKind) {
|
|
let mark = reader.mark
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: mark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func append(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
mark: PureYAML.Parsing.Mark,
|
|
endMark: PureYAML.Parsing.Mark,
|
|
) {
|
|
tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: endMark))
|
|
markDocumentContentIfNeeded(kind)
|
|
}
|
|
|
|
mutating func markDocumentContentIfNeeded(_ kind: PureYAML.Parsing.TokenKind) {
|
|
switch kind {
|
|
case .alias, .anchor, .blockScalarHeader, .scalar, .tag:
|
|
// A node (or one of its properties) has appeared on this line, so a
|
|
// simple key is no longer allowed and a following tab is ordinary
|
|
// content rather than indentation. Mirrors libyaml setting
|
|
// simple_key_allowed = 0 after scanning these tokens.
|
|
hasDocumentContent = true
|
|
simpleKeyAllowed = false
|
|
case .comment, .dedent, .documentEnd, .documentStart, .indent, .streamEnd, .streamStart:
|
|
return
|
|
default:
|
|
hasDocumentContent = true
|
|
}
|
|
}
|
|
|
|
var isExpectingFlowMappingKey: Bool {
|
|
flowContexts.last?.kind == .mapping
|
|
&& flowContexts.last?.isExpectingKey == true
|
|
}
|
|
}
|
|
}
|
|
|
|
extension PureYAML.Parsing.Scanner {
|
|
func scanToken(_ state: inout State) throws {
|
|
skipSeparation(&state)
|
|
try rejectTabSeparation(&state)
|
|
guard let character = state.reader.peek(), !isLineBreak(character) else {
|
|
return
|
|
}
|
|
|
|
if try scanLineIndicatorToken(character, state: &state) {
|
|
return
|
|
}
|
|
if scanFlowToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanQuotedOrBlockScalarToken(character, state: &state) {
|
|
return
|
|
}
|
|
if try scanNodePropertyToken(character, state: &state) {
|
|
return
|
|
}
|
|
scanPlainScalar(&state)
|
|
}
|
|
|
|
func scanLineIndicatorToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "#":
|
|
scanComment(&state)
|
|
return true
|
|
case "-" where isBlockEntry(state.reader):
|
|
scanFixedToken(.blockEntry, state: &state)
|
|
if state.flowDepth == 0 {
|
|
state.simpleKeyAllowed = true
|
|
}
|
|
return true
|
|
case "?" where isIndicatorBoundary(state.reader.peek(offset: 1)):
|
|
scanFixedToken(.mappingKey, state: &state)
|
|
if state.flowDepth == 0 {
|
|
state.simpleKeyAllowed = true
|
|
}
|
|
return true
|
|
case ":" where isAtMappingValueIndicator(state: state):
|
|
scanFixedToken(.mappingValue, state: &state)
|
|
if state.flowDepth == 0 {
|
|
state.simpleKeyAllowed = true
|
|
}
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanFlowToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) -> Bool {
|
|
switch character {
|
|
case "[":
|
|
state.flowDepth += 1
|
|
state.flowContexts.append(.init(kind: .sequence, isExpectingKey: false))
|
|
scanFixedToken(.flowSequenceStart, state: &state)
|
|
return true
|
|
case "]":
|
|
state.flowDepth = max(0, state.flowDepth - 1)
|
|
scanFixedToken(.flowSequenceEnd, state: &state)
|
|
_ = state.flowContexts.popLast()
|
|
return true
|
|
case "{":
|
|
state.flowDepth += 1
|
|
state.flowContexts.append(.init(kind: .mapping, isExpectingKey: true))
|
|
scanFixedToken(.flowMappingStart, state: &state)
|
|
return true
|
|
case "}":
|
|
state.flowDepth = max(0, state.flowDepth - 1)
|
|
scanFixedToken(.flowMappingEnd, state: &state)
|
|
_ = state.flowContexts.popLast()
|
|
return true
|
|
case ",":
|
|
scanFixedToken(.flowEntry, state: &state)
|
|
if state.flowContexts.last?.kind == .mapping {
|
|
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = true
|
|
}
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanNodePropertyToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "&":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.anchor,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "*":
|
|
try scanNamedToken(
|
|
kind: PureYAML.Parsing.TokenKind.alias,
|
|
missingNameError: PureYAML.Parsing.ParseError.expectedAliasName,
|
|
state: &state,
|
|
)
|
|
return true
|
|
case "!":
|
|
try scanTag(&state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func scanQuotedOrBlockScalarToken(
|
|
_ character: Character,
|
|
state: inout State,
|
|
) throws -> Bool {
|
|
switch character {
|
|
case "\"":
|
|
try scanQuotedScalar(.doubleQuoted, state: &state)
|
|
return true
|
|
case "'":
|
|
try scanQuotedScalar(.singleQuoted, state: &state)
|
|
return true
|
|
case "|":
|
|
try scanBlockScalarHeader(.literal, state: &state)
|
|
return true
|
|
case ">":
|
|
try scanBlockScalarHeader(.folded, state: &state)
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func skipSeparation(_ state: inout State) {
|
|
while state.reader.peek() == " " {
|
|
state.reader.advance()
|
|
}
|
|
}
|
|
|
|
func scanFixedToken(
|
|
_ kind: PureYAML.Parsing.TokenKind,
|
|
state: inout State,
|
|
) {
|
|
let start = state.reader.mark
|
|
state.reader.advance()
|
|
state.append(kind, mark: start, endMark: state.reader.mark)
|
|
if case .blockEntry = kind, hasNodeOnSameLine(afterCurrentPositionIn: state.reader) {
|
|
state.validImplicitIndentation.insert(start.column + 1)
|
|
}
|
|
if case .mappingValue = kind, state.flowContexts.last?.kind == .mapping {
|
|
state.flowContexts[state.flowContexts.count - 1].isExpectingKey = false
|
|
}
|
|
}
|
|
|
|
func scanComment(_ state: inout State) {
|
|
let start = state.reader.mark
|
|
state.reader.advance()
|
|
if state.reader.peek() == " " {
|
|
state.reader.advance()
|
|
}
|
|
let value = state.reader.consume { character in
|
|
!isLineBreak(character)
|
|
}
|
|
state.append(.comment(value), mark: start, endMark: state.reader.mark)
|
|
}
|
|
|
|
func scanQuotedScalar(
|
|
_ style: PureYAML.Parsing.ScalarStyle,
|
|
state: inout State,
|
|
) throws {
|
|
let start = state.reader.mark
|
|
let quote: Character = style == .doubleQuoted ? "\"" : "'"
|
|
state.reader.advance()
|
|
var value = ""
|
|
var previousWasEscape = false
|
|
while let character = state.reader.peek() {
|
|
// A `---` or `...` document marker at column 0 cannot appear inside an
|
|
// open quoted scalar: it means the scalar was never closed before a
|
|
// document boundary (scanner.c:3058-3070). Reject it as unterminated.
|
|
if state.reader.mark.column == 1, isDocumentMarkerHere(state) {
|
|
throw PureYAML.Parsing.ParseError.unterminatedQuotedString(line: start.line)
|
|
}
|
|
if style == .singleQuoted, character == quote, state.reader.peek(offset: 1) == quote {
|
|
value.append(character)
|
|
state.reader.advance()
|
|
value.append(character)
|
|
state.reader.advance()
|
|
continue
|
|
}
|
|
if character == quote, !previousWasEscape {
|
|
break
|
|
}
|
|
value.append(character)
|
|
previousWasEscape = character == "\\" && style == .doubleQuoted && !previousWasEscape
|
|
state.reader.advance()
|
|
if character != "\\" {
|
|
previousWasEscape = false
|
|
}
|
|
}
|
|
guard state.reader.peek() == quote else {
|
|
throw PureYAML.Parsing.ParseError.unterminatedQuotedString(line: start.line)
|
|
}
|
|
state.reader.advance()
|
|
appendMappingKeyIfNeeded(start: start, state: &state)
|
|
state.append(.scalar(value: value, style: style), mark: start, endMark: state.reader.mark)
|
|
}
|
|
|
|
func scanNamedToken(
|
|
kind: (String) -> PureYAML.Parsing.TokenKind,
|
|
missingNameError: (Int) -> PureYAML.Parsing.ParseError,
|
|
state: inout State,
|
|
) throws {
|
|
let start = state.reader.mark
|
|
state.reader.advance()
|
|
// An anchor or alias name is a run of name characters that ends at the
|
|
// first non-name character, mirroring `yaml_parser_scan_anchor`: the
|
|
// name must stop at whitespace, `:`, `?`, flow indicators, and the other
|
|
// indicator characters so that `*alias:` reads the alias `alias`
|
|
// followed by a `:` mapping indicator rather than swallowing the colon.
|
|
let name = state.reader.consume { character in
|
|
isAnchorNameCharacter(character)
|
|
}
|
|
guard !name.isEmpty else {
|
|
throw missingNameError(start.line)
|
|
}
|
|
state.append(kind(name), mark: start, endMark: state.reader.mark)
|
|
}
|
|
|
|
func scanTag(_ state: inout State) throws {
|
|
let start = state.reader.mark
|
|
state.reader.advance()
|
|
var value = "!"
|
|
if state.reader.peek() == "<" {
|
|
value.append(state.reader.advance() ?? "<")
|
|
var didClose = false
|
|
while let character = state.reader.peek() {
|
|
value.append(character)
|
|
state.reader.advance()
|
|
if character == ">" {
|
|
didClose = true
|
|
break
|
|
}
|
|
}
|
|
guard didClose else {
|
|
throw PureYAML.Parsing.ParseError.unterminatedTag(line: start.line)
|
|
}
|
|
} else {
|
|
value += state.reader.consume { character in
|
|
!isTokenTerminator(character)
|
|
}
|
|
value = expandTag(value, state: state)
|
|
}
|
|
state.append(.tag(value), mark: start, endMark: state.reader.mark)
|
|
}
|
|
|
|
func scanPlainScalar(_ state: inout State) {
|
|
let start = state.reader.mark
|
|
var value = ""
|
|
while let character = state.reader.peek() {
|
|
if shouldEndPlainScalar(character, state: state, accumulated: value) {
|
|
break
|
|
}
|
|
value.append(character)
|
|
state.reader.advance()
|
|
}
|
|
var trimmed = trimTrailingSpaces(value)
|
|
let isMappingKey = appendMappingKeyIfNeeded(start: start, state: &state)
|
|
if !isMappingKey {
|
|
trimmed = scanPlainScalarContinuations(
|
|
after: trimmed,
|
|
state: &state,
|
|
)
|
|
}
|
|
state.append(.scalar(value: trimmed, style: .plain), mark: start, endMark: state.reader.mark)
|
|
}
|
|
}
|