PureYAML/Sources/Parsing/Scanner.swift

extension PureYAML.Parsing {
    struct Scanner {
        init() {}

        func scan(_ yaml: String) throws -> [Token] {
            var state = State(reader: Reader(yaml))
            state.append(.streamStart)

            loop: while !state.reader.isAtEnd {
                switch try scanStep(&state) {
                case .endOfInput:
                    break loop
                case .advanced, .scannedMarker, .scannedToken:
                    continue
                }
            }

            if state.hasPendingDirective {
                throw PureYAML.Parsing.ParseError.directiveWithoutDocument(line: state.reader.mark.line)
            }
            closeIndentation(&state)
            state.append(.streamEnd)
            return state.tokens
        }
    }
}

extension PureYAML.Parsing.Scanner {
    struct State {
        var reader: PureYAML.Parsing.Reader
        var tokens: [PureYAML.Parsing.Token] = []
        var indentation: [Int] = [0]
        var isAtLineStart = true
        var hasDocumentContent = false
        var hasDocumentStartMarker = false
        var isDocumentClosed = false
        /// True once a `%YAML` directive has been scanned for the document that
        /// the next `---` will start. A second `%YAML` before that `---` is a
        /// duplicate and is rejected; the flag resets at every document boundary.
        var sawYAMLDirective = false
        /// True once any directive has been scanned in the current directive
        /// section. A directive section must be terminated by a `---` document
        /// start; reaching a `...` end marker or the end of the stream with a
        /// directive still pending is an error.
        var hasPendingDirective = false
        var flowDepth = 0
        /// Mirrors libyaml's `parser->indent`: the 0-based column of the innermost
        /// open block collection (sequence or mapping), or -1 at the bare document
        /// root. Plain-scalar continuation folding compares a continuation line's
        /// indentation against this value, so a root scalar (block indent -1) folds
        /// column-0 continuations while a scalar inside a collection only folds lines
        /// indented past the collection. Tracked separately from ``indentation`` --
        /// which records every line-indentation level -- because libyaml rolls this
        /// only on block-collection indicators, not on every indentation increase.
        var blockIndent = -1
        /// Saved ``blockIndent`` values, mirroring libyaml's `parser->indents` stack,
        /// so a dedent restores the enclosing collection's indent.
        var blockIndents: [Int] = []
        /// Mirrors libyaml's `simple_key_allowed`. In block context a tab may only
        /// be consumed as whitespace when a simple key is NOT allowed (i.e. mid-line,
        /// after a node has been emitted). It is true at the start of every block
        /// line and immediately after a `-`, `?`, or `:` indicator, so a tab in those
        /// positions provides block indentation and is rejected. See scanner.c:1948.
        var simpleKeyAllowed = true
        var flowContexts: [PureYAML.Parsing.ScannerFlowContext] = []
        /// A stack of potential simple keys, one entry per open flow collection,
        /// mirroring libyaml's `simple_keys`. The slot for the innermost flow level
        /// records the most recent position a `:` could promote to a mapping key.
        /// Empty outside flow context; block keys do not use this stack.
        var flowSimpleKeys: [PureYAML.Parsing.ScannerFlowSimpleKey?] = []
        var validImplicitIndentation: Set<Int> = []
        var tagHandles = [
            "!": "!",
            "!!": "tag:yaml.org,2002:",
        ]

        mutating func append(_ kind: PureYAML.Parsing.TokenKind) {
            let mark = reader.mark
            tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: mark))
            markDocumentContentIfNeeded(kind)
        }

        mutating func append(
            _ kind: PureYAML.Parsing.TokenKind,
            mark: PureYAML.Parsing.Mark,
            endMark: PureYAML.Parsing.Mark,
        ) {
            tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: endMark))
            markDocumentContentIfNeeded(kind)
        }

        mutating func markDocumentContentIfNeeded(_ kind: PureYAML.Parsing.TokenKind) {
            switch kind {
            case .alias, .anchor, .blockScalarHeader, .scalar, .tag:
                // A node (or one of its properties) has appeared on this line, so a
                // simple key is no longer allowed and a following tab is ordinary
                // content rather than indentation. Mirrors libyaml setting
                // simple_key_allowed = 0 after scanning these tokens.
                hasDocumentContent = true
                simpleKeyAllowed = false
            case .comment, .dedent, .documentEnd, .documentStart, .indent, .streamEnd, .streamStart:
                return
            default:
                hasDocumentContent = true
            }
        }

        var isExpectingFlowMappingKey: Bool {
            flowContexts.last?.kind == .mapping
                && flowContexts.last?.isExpectingKey == true
        }
    }
}

extension PureYAML.Parsing.Scanner {
    func scanToken(_ state: inout State) throws {
        skipSpaceSeparation(&state)
        try rejectTabSeparation(&state)
        skipSeparation(&state)
        guard let character = state.reader.peek(), !isLineBreak(character) else {
            return
        }

        if try scanLineIndicatorToken(character, state: &state) {
            return
        }
        if scanFlowToken(character, state: &state) {
            return
        }
        if try scanQuotedOrBlockScalarToken(character, state: &state) {
            return
        }
        if try scanNodePropertyToken(character, state: &state) {
            return
        }
        scanPlainScalar(&state)
    }

    func scanLineIndicatorToken(
        _ character: Character,
        state: inout State,
    ) throws -> Bool {
        switch character {
        case "#":
            scanComment(&state)
            return true
        case "-" where isBlockEntry(state.reader):
            scanFixedToken(.blockEntry, state: &state)
            if state.flowDepth == 0 {
                state.simpleKeyAllowed = true
            }
            return true
        case "?" where state.flowDepth > 0 || isIndicatorBoundary(state.reader.peek(offset: 1)):
            // In flow context a `?` is always an explicit-key indicator (libyaml's
            // fetch_more_tokens checks `flow_level || IS_BLANKZ_AT`); a pending
            // simple key is abandoned and no implicit key may follow on this entry.
            if state.flowDepth > 0 {
                removeFlowSimpleKey(&state)
            }
            scanFixedToken(.mappingKey, state: &state)
            if state.flowDepth == 0 {
                state.simpleKeyAllowed = true
            } else {
                state.simpleKeyAllowed = false
            }
            return true
        case ":" where state.flowDepth > 0 || isAtMappingValueIndicator(state: state):
            // In flow context a `:` is always a value indicator regardless of what
            // follows it (libyaml), so `"k":v` and `{k}:v` are key/value pairs. A
            // pending simple key is spliced into a `mappingKey` before its node.
            if state.flowDepth > 0 {
                promoteFlowSimpleKey(&state)
            }
            scanFixedToken(.mappingValue, state: &state)
            if state.flowDepth == 0 {
                state.simpleKeyAllowed = true
            } else {
                state.simpleKeyAllowed = false
            }
            return true
        default:
            return false
        }
    }

    func scanNodePropertyToken(
        _ character: Character,
        state: inout State,
    ) throws -> Bool {
        switch character {
        case "&":
            try scanNamedToken(
                kind: PureYAML.Parsing.TokenKind.anchor,
                missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName,
                state: &state,
            )
            return true
        case "*":
            try scanNamedToken(
                kind: PureYAML.Parsing.TokenKind.alias,
                missingNameError: PureYAML.Parsing.ParseError.expectedAliasName,
                state: &state,
            )
            return true
        case "!":
            try scanTag(&state)
            return true
        default:
            return false
        }
    }

    func scanQuotedOrBlockScalarToken(
        _ character: Character,
        state: inout State,
    ) throws -> Bool {
        switch character {
        case "\"":
            try scanQuotedScalar(.doubleQuoted, state: &state)
            return true
        case "'":
            try scanQuotedScalar(.singleQuoted, state: &state)
            return true
        case "|":
            try scanBlockScalarHeader(.literal, state: &state)
            return true
        case ">":
            try scanBlockScalarHeader(.folded, state: &state)
            return true
        default:
            return false
        }
    }

    func skipSpaceSeparation(_ state: inout State) {
        while state.reader.peek() == " " {
            state.reader.advance()
        }
    }

    func skipSeparation(_ state: inout State) {
        // In-line separation (s-white) is spaces and tabs. `rejectTabSeparation` has
        // already rejected any tab that would supply the indentation of a nested block
        // node, so a tab still under the reader here is ordinary separation before a
        // scalar value or flow node and is consumed like a space (scanner.c IS_BLANK).
        while let character = state.reader.peek(), character == " " || character == "\t" {
            state.reader.advance()
        }
    }

    func scanFixedToken(
        _ kind: PureYAML.Parsing.TokenKind,
        state: inout State,
    ) {
        let start = state.reader.mark
        // Mirror libyaml rolling the block indent when a block sequence entry (`-`)
        // or an explicit `?` mapping key opens a collection at this column. The
        // implicit-key case rolls in `appendMappingKeyIfNeeded` at the key column;
        // the `:` value indicator does not roll again, because for a simple key the
        // mapping indent was already set at the key's (shallower) column and rolling
        // to the deeper `:` column would wrongly raise the continuation threshold.
        switch kind {
        case .blockEntry, .mappingKey:
            rollBlockIndent(toColumn: start.column - 1, state: &state)
        default:
            break
        }
        state.reader.advance()
        state.append(kind, mark: start, endMark: state.reader.mark)
        if case .blockEntry = kind, hasNodeOnSameLine(afterCurrentPositionIn: state.reader) {
            state.validImplicitIndentation.insert(start.column + 1)
        }
        if case .mappingValue = kind, state.flowContexts.last?.kind == .mapping {
            state.flowContexts[state.flowContexts.count - 1].isExpectingKey = false
        }
    }

    func scanComment(_ state: inout State) {
        let start = state.reader.mark
        state.reader.advance()
        if state.reader.peek() == " " {
            state.reader.advance()
        }
        let value = state.reader.consume { character in
            !isLineBreak(character)
        }
        state.append(.comment(value), mark: start, endMark: state.reader.mark)
    }

    func scanQuotedScalar(
        _ style: PureYAML.Parsing.ScalarStyle,
        state: inout State,
    ) throws {
        let start = state.reader.mark
        // A quoted scalar may be a flow simple key: record its position before the
        // content is consumed so a following `:` can splice a key in front of it.
        saveFlowSimpleKey(&state)
        let quote: Character = style == .doubleQuoted ? "\"" : "'"
        state.reader.advance()
        var value = ""
        var previousWasEscape = false
        while let character = state.reader.peek() {
            // A `---` or `...` document marker at column 0 cannot appear inside an
            // open quoted scalar: it means the scalar was never closed before a
            // document boundary (scanner.c:3058-3070). Reject it as unterminated.
            if state.reader.mark.column == 1, isDocumentMarkerHere(state) {
                throw PureYAML.Parsing.ParseError.unterminatedQuotedString(line: start.line)
            }
            if style == .singleQuoted, character == quote, state.reader.peek(offset: 1) == quote {
                value.append(character)
                state.reader.advance()
                value.append(character)
                state.reader.advance()
                continue
            }
            if character == quote, !previousWasEscape {
                break
            }
            value.append(character)
            previousWasEscape = character == "\\" && style == .doubleQuoted && !previousWasEscape
            state.reader.advance()
            if character != "\\" {
                previousWasEscape = false
            }
        }
        guard state.reader.peek() == quote else {
            throw PureYAML.Parsing.ParseError.unterminatedQuotedString(line: start.line)
        }
        state.reader.advance()
        appendMappingKeyIfNeeded(start: start, state: &state)
        state.append(.scalar(value: value, style: style), mark: start, endMark: state.reader.mark)
    }

    func scanPlainScalar(_ state: inout State) {
        let start = state.reader.mark
        // A plain scalar may be a flow simple key: record its position before the
        // content is consumed so a following `:` can splice a key in front of it.
        saveFlowSimpleKey(&state)
        var value = ""
        while let character = state.reader.peek() {
            if shouldEndPlainScalar(character, state: state, accumulated: value) {
                break
            }
            value.append(character)
            state.reader.advance()
        }
        var trimmed = trimTrailingSpaces(value)
        let isMappingKey = appendMappingKeyIfNeeded(start: start, state: &state)
        if !isMappingKey {
            trimmed = scanPlainScalarContinuations(
                after: trimmed,
                state: &state,
            )
        }
        state.append(.scalar(value: trimmed, style: .plain), mark: start, endMark: state.reader.mark)
    }
}