extension PureYAML.Parsing { struct Scanner { init() {} func scan(_ yaml: String) throws -> [Token] { try run(yaml).tokens } /// The tag-handle issues found while scanning, for the diagnostic layer. Issues collected /// before a scan-aborting error are still returned; a clean scan returns them all. func tagHandleIssues(_ yaml: String) -> [PureYAML.Parsing.TagHandleIssue] { (try? run(yaml))?.tagHandleIssues ?? [] } private func run(_ yaml: String) throws -> State { var state = State(reader: Reader(yaml)) state.append(.streamStart) while !state.reader.isAtEnd { if isLineBreak(state.reader.peek()) { state.reader.advance() captureBlockScalarBlankLine(&state) state.isAtLineStart = true state.sawContentOnLine = false state.sawNonPropertyContentOnLine = false state.sawMappingValueOnLine = false state.sawFlowCloseOnLine = false continue } if state.isAtLineStart { try scanIndentation(&state) if state.reader.isAtEnd { break } if isLineBreak(state.reader.peek()) { continue } if try scanDirectiveOrDocumentMarker(&state) { continue } state.isAtLineStart = false } try scanToken(&state) } closeIndentation(&state) try enforceDirectiveHasDocument(state) state.append(.streamEnd) return state } } } extension PureYAML.Parsing.Scanner { func scanToken(_ state: inout State) throws { let columnBeforeSeparation = state.reader.mark.column skipSeparation(&state) let consumedSeparation = state.reader.mark.column > columnBeforeSeparation guard let character = state.reader.peek(), !isLineBreak(character) else { return } if state.blockScalarIndentation != nil { scanBlockScalarContent(&state) return } // A `#` begins a comment only at the start of a line's content or after whitespace. Glued to // earlier same-line content (e.g. `"x"#c`, `]#c`, `>#c`) it is not a plain-scalar character // either, so the document is malformed; a plain scalar's own `#` is consumed before reaching // here, so this rejects only the genuinely invalid case. if character == "#", !consumedSeparation, state.sawContentOnLine { throw PureYAML.Parsing.ParseError.commentNotPrecededByWhitespace( line: state.reader.mark.line, column: state.reader.mark.column, ) } let hadContentOnLine = state.sawContentOnLine let hadNonPropertyContentOnLine = state.sawNonPropertyContentOnLine state.sawContentOnLine = true // A leading `&` or `!` begins a node property (anchor or tag), which is not itself a key or // value; every other token character is real content. A property-only line followed by `:` // is thus an empty implicit key (`&a :`), synthesized below like a bare `: value`. if character != "&", character != "!" { state.sawNonPropertyContentOnLine = true } // An implicit-key mapping value (`:` preceded by its key on the same line) means the line // now holds a pair, so a later scalar followed by `:` cannot start another: that would be a // forbidden inline nested block mapping (`a: b: c`). A `:` that opens the line is an explicit- // or null-key value indicator, whose value may legitimately be a compact mapping, so it does // not set the flag. if character == ":", hadContentOnLine, state.flowDepth == 0, isAtMappingValueIndicator(state: state) { state.sawMappingValueOnLine = true } synthesizeNullMappingKeyIfNeeded(character, hadNonPropertyContentOnLine: hadNonPropertyContentOnLine, state: &state) if try scanLineIndicatorToken(character, state: &state) { return } if scanFlowToken(character, state: &state) { return } if try scanQuotedOrBlockScalarToken(character, state: &state) { return } if try scanNodePropertyToken(character, state: &state) { return } // In a flow collection, a `-` that begins a plain scalar must be followed by a non-indicator // character. Immediately before a flow indicator (`[-]`, `[-, -]`) it is not a valid scalar. if state.flowDepth > 0, character == "-", let next = state.reader.peek(offset: 1), isFlowDelimiter(next) { throw PureYAML.Parsing.ParseError.unexpectedToken( expected: "a flow value", actual: "-", line: state.reader.mark.line, column: state.reader.mark.column, ) } scanPlainScalar(&state) } func scanLineIndicatorToken( _ character: Character, state: inout State, ) throws -> Bool { switch character { case "#": scanComment(&state) return true case "-" where isBlockEntry(state.reader): // A block sequence cannot be an implicit key's value on the same line as its `:` // (`key: - a`); the value must begin on a following line. As with an inline nested // mapping, an explicit- or null-key `:` opens its line and does not set the flag, so a // compact sequence value after it stays valid. if state.sawMappingValueOnLine, state.flowDepth == 0 { throw PureYAML.Parsing.ParseError.blockSequenceAsInlineMappingValue(line: state.reader.mark.line) } // A block sequence entry cannot follow a flow collection that closed earlier on the // line (`{a: b}- x`); the entry would begin a sequence inline after a complete node. if state.sawFlowCloseOnLine { throw PureYAML.Parsing.ParseError.unexpectedToken( expected: "a line break or flow indicator", actual: "-", line: state.reader.mark.line, column: state.reader.mark.column, ) } scanFixedToken(.blockEntry, state: &state) return true case "?" where isIndicatorBoundary(state.reader.peek(offset: 1)): scanFixedToken(.mappingKey, state: &state) return true case ":" where isAtMappingValueIndicator(state: state): scanFixedToken(.mappingValue, state: &state) return true default: return false } } func scanFlowToken( _ character: Character, state: inout State, ) -> Bool { switch character { case "[": appendFlowImplicitKeyMarkerIfNeeded(state: &state) state.flowDepth += 1 state.flowContexts.append(.init(kind: .sequence, isExpectingKey: false)) scanFixedToken(.flowSequenceStart, state: &state) return true case "]": state.flowDepth = max(0, state.flowDepth - 1) scanFixedToken(.flowSequenceEnd, state: &state) _ = state.flowContexts.popLast() if state.flowDepth == 0 { state.sawFlowCloseOnLine = true } return true case "{": appendFlowImplicitKeyMarkerIfNeeded(state: &state) state.flowDepth += 1 state.flowContexts.append(.init(kind: .mapping, isExpectingKey: true)) scanFixedToken(.flowMappingStart, state: &state) return true case "}": state.flowDepth = max(0, state.flowDepth - 1) scanFixedToken(.flowMappingEnd, state: &state) _ = state.flowContexts.popLast() if state.flowDepth == 0 { state.sawFlowCloseOnLine = true } return true case ",": scanFixedToken(.flowEntry, state: &state) if state.flowContexts.last?.kind == .mapping { state.flowContexts[state.flowContexts.count - 1].isExpectingKey = true } if !state.flowContexts.isEmpty { state.flowContexts[state.flowContexts.count - 1].hasExplicitKey = false } return true default: return false } } func scanNodePropertyToken( _ character: Character, state: inout State, ) throws -> Bool { switch character { case "&": try scanNamedToken( kind: PureYAML.Parsing.TokenKind.anchor, missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName, state: &state, ) return true case "*": try scanNamedToken( kind: PureYAML.Parsing.TokenKind.alias, missingNameError: PureYAML.Parsing.ParseError.expectedAliasName, canBeImplicitKey: true, state: &state, ) return true case "!": try scanTag(&state) return true default: return false } } func scanQuotedOrBlockScalarToken( _ character: Character, state: inout State, ) throws -> Bool { switch character { case "\"": try scanQuotedScalar(.doubleQuoted, state: &state) return true case "'": try scanQuotedScalar(.singleQuoted, state: &state) return true case "|": scanBlockScalarHeader(.literal, state: &state) return true case ">": scanBlockScalarHeader(.folded, state: &state) return true default: return false } } func skipSeparation(_ state: inout State) { // Mid-line separation may be spaces or tabs: a tab is separation white space in YAML and is // forbidden only as indentation, which the line-start indentation scan handles before this. while state.reader.peek() == " " || state.reader.peek() == "\t" { state.reader.advance() } } func scanFixedToken( _ kind: PureYAML.Parsing.TokenKind, state: inout State, ) { let start = state.reader.mark state.reader.advance() state.append(kind, mark: start, endMark: state.reader.mark) if case .blockEntry = kind { recordBlockEntryImplicitIndentation(start: start, state: &state) // A block entry opens a fresh node position, so a property-only key after it on the same // line (`- !!str : v`) is still an empty implicit key: the `-` is not content of the key. state.sawNonPropertyContentOnLine = false } if case .mappingValue = kind, state.flowContexts.last?.kind == .mapping { state.flowContexts[state.flowContexts.count - 1].isExpectingKey = false } // Only the explicit `?` indicator emits a mapping key through this path (an implicit key is // synthesized directly), so track a block-level one as pending until its `:` value indicator // is scanned. Flow `:`/`?` (inside a complex key, say) do not affect a pending block key. if state.flowDepth == 0, case .mappingKey = kind { state.pendingExplicitKey = true } if state.flowDepth == 0, case .mappingValue = kind { state.pendingExplicitKey = false } // In flow the same role is tracked on the current flow context, so it is per-level and a // nested collection's `:` does not affect an outer explicit key (`{? {a: b} : c}`). if state.flowDepth > 0, !state.flowContexts.isEmpty, case .mappingKey = kind { state.flowContexts[state.flowContexts.count - 1].hasExplicitKey = true } if state.flowDepth > 0, !state.flowContexts.isEmpty, case .mappingValue = kind { state.flowContexts[state.flowContexts.count - 1].hasExplicitKey = false } } }