PureYAML/Sources/Parsing/LibYAML.Scanner.Fetch.swift

extension PureYAML.Parsing.LibYAML.Scanner {
    private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
    private typealias Token = PureYAML.Parsing.LibYAML.Token
    private typealias ScannerError = PureYAML.Parsing.LibYAML.ScannerError

    /// Produce every token for the input, mirroring repeated calls to libyaml's
    /// `yaml_parser_fetch_next_token` until STREAM-END. Tokens are never dequeued,
    /// so `tokens_parsed` stays 0 and `QUEUE_INSERT` positions are absolute, which
    /// matches the simple-key splice.
    mutating func scanAll() throws -> [PureYAML.Parsing.LibYAML.Token] {
        while !streamEndProduced {
            try fetchNextToken()
        }
        return indentEngine.tokens
    }

    /// Port of `yaml_parser_fetch_next_token`: emit STREAM-START first, otherwise
    /// skip to the next token, expire stale simple keys, unroll the indentation to
    /// the current column, and dispatch on the leading character. Constructs not
    /// yet ported throw an explicit error rather than mis-scan.
    mutating func fetchNextToken() throws {
        if !streamStartProduced {
            fetchStreamStart()
            return
        }
        // Gated on `reader.index >= badIndex`, not just "is one set", so a real
        // syntax error earlier in the document (which normal scanning would reach
        // and throw first) is not hidden behind a control character that only
        // occurs later: this only fires once the reader has genuinely scanned up
        // to the offending position, matching the order libyaml's own byte-by-byte
        // reader would hit them in. `resumePlainScalar`'s own suspend/resume path
        // (`scanStep()`, the streaming API) does not route through
        // `fetchNextToken()` and so is not covered by this check; a token
        // containing a control character can be returned to a streaming caller one
        // token before this fires. `scanAll()`/`PureYAML.parse()`, the primary
        // target, always calls `fetchNextToken()` again before returning, so this
        // still surfaces before a batch parse completes.
        if let badIndex = reader.invalidControlCharacterIndex, reader.index >= badIndex {
            throw ScannerError(
                context: "while scanning for the next token",
                problem: "control characters are not allowed",
                mark: reader.mark(atIndex: badIndex),
            )
        }

        try scanToNextToken()
        try checkFlowContinuationIndent()
        mark = reader.mark
        try staleSimpleKeys()
        indentEngine.unrollIndent(column: reader.column, mark: reader.mark)
        mark = reader.mark
        try dispatchToken()
    }

    /// Dispatch on the leading character: stream end, directives and document
    /// markers at column 0, then the indicators.
    private mutating func dispatchToken() throws {
        let character = reader.peek()
        if Classify.isZ(character) {
            fetchStreamEnd()
            return
        }
        if reader.column == 0, character == "%" {
            try fetchDirective()
            return
        }
        if reader.column == 0, isDocumentMarker("-") {
            try fetchDocumentIndicator(.documentStart)
            return
        }
        if reader.column == 0, isDocumentMarker(".") {
            try fetchDocumentIndicator(.documentEnd)
            return
        }
        guard let character else {
            return
        }
        try dispatchIndicator(character)
    }

    /// Dispatch the block-structure indicators and plain scalars; unported
    /// constructs throw an explicit error.
    private mutating func dispatchIndicator(_ character: Unicode.Scalar) throws {
        if try fetchFlowToken(character) {
            return
        }
        if try fetchNodeToken(character) {
            return
        }
        switch character {
        case "-" where Classify.isBlankZ(reader.peek(1)):
            try fetchBlockEntry()
        case "-" where flowLevel > 0 && Classify.isFlowIndicator(reader.peek(1)):
            // Per `ns-plain-first` a `-` begins a plain scalar in flow only when a
            // flow-safe character follows; before a flow indicator (`[-]`, `[-,`)
            // it cannot start a node. libyaml reads it as the scalar `-`.
            throw ScannerError(
                context: "while scanning for the next token",
                problem: "found unexpected '-' indicator",
                mark: reader.mark,
            )
        case "?" where Classify.isBlankZ(reader.peek(1)) || (flowLevel > 0 && Classify.isFlowIndicator(reader.peek(1))):
            // In flow context libyaml treats every `?` as a key indicator, but per
            // `ns-plain-first` a `?` followed by a flow-safe character (anything but
            // whitespace, end, or a flow indicator) begins a plain scalar (`?foo`).
            try fetchKey()
        case ":" where Classify.isBlankZ(reader.peek(1))
            || (flowLevel > 0 && (Classify.isFlowIndicator(reader.peek(1)) || flowKeyIsJSON)):
            // In flow context libyaml makes every `:` a value indicator. Per the
            // spec a `:` only ends a node when followed by whitespace, end of
            // input, or a flow indicator, except an adjacent `:` after a JSON
            // (quoted) key. Otherwise the `:` is plain-scalar content (`:x`).
            try fetchValue()
        default:
            // A plain scalar may begin with any non-blank character; a blank (a
            // tab here, since spaces and breaks were already consumed) cannot
            // start a token, matching libyaml's final guard in fetch_next_token.
            guard !Classify.isBlankZ(character) else {
                throw ScannerError(
                    context: "while scanning for the next token",
                    problem: "found character that cannot start any token",
                    mark: reader.mark,
                )
            }
            try fetchPlainScalar()
        }
    }

    /// Fetch a flow-collection indicator (`[` `]` `{` `}` `,`); returns false
    /// otherwise.
    private mutating func fetchFlowToken(_ character: Unicode.Scalar) throws -> Bool {
        switch character {
        case "[": fetchFlowCollectionStart(.flowSequenceStart)
        case "{": fetchFlowCollectionStart(.flowMappingStart)
        case "]": try fetchFlowCollectionEnd(.flowSequenceEnd)
        case "}": try fetchFlowCollectionEnd(.flowMappingEnd)
        case ",": try fetchFlowEntry()
        default: return false
        }
        return true
    }

    /// Fetch a quoted scalar, an anchor/alias, or a block scalar; returns false
    /// otherwise.
    private mutating func fetchNodeToken(_ character: Unicode.Scalar) throws -> Bool {
        switch character {
        case "'": try fetchFlowScalar(single: true)
        case "\"": try fetchFlowScalar(single: false)
        case "&": try fetchAnchor(isAnchor: true)
        case "*": try fetchAnchor(isAnchor: false)
        case "!": try fetchTag()
        case "|" where flowLevel == 0: try fetchBlockScalar(literal: true)
        case ">" where flowLevel == 0: try fetchBlockScalar(literal: false)
        default: return false
        }
        return true
    }

    /// Port of `yaml_parser_fetch_tag`.
    private mutating func fetchTag() throws {
        mark = reader.mark
        try saveSimpleKey()
        simpleKeyAllowed = false
        let token = try scanTag()
        indentEngine.enqueue(token)
    }

    /// Port of `yaml_parser_fetch_block_scalar`.
    private mutating func fetchBlockScalar(literal: Bool) throws {
        try removeSimpleKey()
        simpleKeyAllowed = true
        let token = try scanBlockScalar(literal: literal)
        indentEngine.enqueue(token)
    }

    /// Port of `yaml_parser_fetch_flow_scalar`.
    private mutating func fetchFlowScalar(single: Bool) throws {
        mark = reader.mark
        let saved = simpleKeyAllowed
        try saveSimpleKey()
        // A quoted scalar that became this level's simple key is a JSON node, so a
        // later adjacent `:` (no space) still opens its value.
        if saved {
            simpleKeys[simpleKeys.count - 1].isJSON = true
        }
        simpleKeyAllowed = false
        let token = try scanFlowScalar(single: single)
        indentEngine.enqueue(token)
    }

    /// Port of `yaml_parser_fetch_anchor`.
    private mutating func fetchAnchor(isAnchor: Bool) throws {
        mark = reader.mark
        try saveSimpleKey()
        simpleKeyAllowed = false
        let token = try scanAnchor(isAnchor: isAnchor)
        indentEngine.enqueue(token)
    }

    /// Port of `yaml_parser_fetch_flow_collection_start`.
    private mutating func fetchFlowCollectionStart(_ type: PureYAML.Parsing.LibYAML.TokenType) {
        // saveSimpleKey cannot throw in flow: a required key needs block context.
        let saved = simpleKeyAllowed
        try? saveSimpleKey()
        // A flow collection is a JSON node, so when it is this level's simple key
        // an adjacent `:` after its closing `]`/`}` still opens its value.
        if saved {
            simpleKeys[simpleKeys.count - 1].isJSON = true
        }
        increaseFlowLevel(isSequence: type == .flowSequenceStart)
        simpleKeyAllowed = true
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: type, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_flow_collection_end`.
    private mutating func fetchFlowCollectionEnd(_ type: PureYAML.Parsing.LibYAML.TokenType) throws {
        try removeSimpleKey()
        decreaseFlowLevel()
        simpleKeyAllowed = false
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: type, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_flow_entry`.
    private mutating func fetchFlowEntry() throws {
        try removeSimpleKey()
        simpleKeyAllowed = true
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: .flowEntry, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_stream_start`.
    private mutating func fetchStreamStart() {
        // indent is already -1 and simpleKeys already holds the block-context slot.
        simpleKeyAllowed = true
        streamStartProduced = true
        indentEngine.enqueue(Token(type: .streamStart, startMark: reader.mark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_stream_end`.
    private mutating func fetchStreamEnd() {
        if reader.column != 0 {
            // Force a new line in the mark, as libyaml does.
            mark = PureYAML.Parsing.Mark(line: reader.line + 1, column: 0, index: reader.index)
        }
        indentEngine.unrollIndent(column: -1, mark: mark)
        // remove_simple_key cannot throw here: a required key would already have
        // been resolved before stream end in the block + plain core.
        simpleKeys[simpleKeys.count - 1].possible = false
        simpleKeyAllowed = false
        streamEndProduced = true
        indentEngine.enqueue(Token(type: .streamEnd, startMark: mark, endMark: mark))
    }

    /// Port of `yaml_parser_fetch_document_indicator`.
    private mutating func fetchDocumentIndicator(_ type: PureYAML.Parsing.LibYAML.TokenType) throws {
        indentEngine.unrollIndent(column: -1, mark: reader.mark)
        simpleKeys[simpleKeys.count - 1].possible = false
        simpleKeyAllowed = false
        let startMark = reader.mark
        reader.skip()
        reader.skip()
        reader.skip()
        indentEngine.enqueue(Token(type: type, startMark: startMark, endMark: reader.mark))
        if case .documentEnd = type {
            // Only whitespace and a comment may follow `...` on its line; a `--- x`
            // document start can carry inline content, but `... x` cannot.
            while reader.peek() == " " || Classify.isTab(reader.peek()) {
                reader.skip()
            }
            if let next = reader.peek(), next != "#", !Classify.isBreak(next) {
                throw ScannerError(
                    context: "while scanning a document end marker",
                    problem: "found content after the document end marker",
                    mark: reader.mark,
                )
            }
        }
    }

    /// Port of `yaml_parser_fetch_block_entry`.
    private mutating func fetchBlockEntry() throws {
        if tabbedSeparation {
            throw tabIndentationError()
        }
        // A block sequence must begin its own line. A `-` mid-line after a mapping
        // value (`key: - a`) or a node property (`&a - x`) is invalid, but a `-`
        // after another `-` is a valid compact sequence (`- - a`). libyaml accepts
        // all three.
        if flowLevel == 0, !crossedLineBreak, lastTokenForbidsInlineSequence() {
            throw ScannerError(
                context: "while scanning a node",
                problem: "found a block sequence on the same line as an earlier node",
                mark: reader.mark,
            )
        }
        if flowLevel == 0 {
            indentEngine.rollIndent(column: reader.column, number: -1, type: .blockSequenceStart, mark: reader.mark)
        }
        simpleKeys[simpleKeys.count - 1].possible = false
        simpleKeyAllowed = true
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: .blockEntry, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_key`.
    private mutating func fetchKey() throws {
        if tabbedSeparation {
            throw tabIndentationError()
        }
        if flowLevel == 0 {
            indentEngine.rollIndent(column: reader.column, number: -1, type: .blockMappingStart, mark: reader.mark)
        }
        simpleKeys[simpleKeys.count - 1].possible = false
        simpleKeyAllowed = flowLevel == 0
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: .key, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_value`, including the simple-key splice: a
    /// possible simple key inserts its `KEY` token at the recorded position and
    /// rolls the block mapping indent to the key's column.
    private mutating func fetchValue() throws {
        let top = simpleKeys.count - 1
        // A pending simple key means this `:` closes an inline implicit key (key
        // and `:` on one line). An explicit `?` key's `:` has no pending simple key
        // (it went stale across the line break), so its value may open a sequence.
        lastValueWasInline = simpleKeys[top].possible
        // A flow-sequence single-pair implicit key must be single-line, so its `:`
        // must sit on the key's line (`[ "k"\n :v ]` and `[ k\n : v ]` are both
        // invalid); a flow-mapping adjacent value may span lines (`{ "k"\n :v }`).
        if simpleKeys[top].possible, flowIsSequence.last == true, simpleKeys[top].mark.line != mark.line {
            throw ScannerError(
                context: "while scanning a simple key",
                problem: "could not find expected ':'",
                mark: simpleKeys[top].mark,
            )
        }
        if simpleKeys[top].possible {
            if simpleKeys[top].tabbed {
                throw tabIndentationError()
            }
            let key = simpleKeys[top]
            indentEngine.insert(Token(type: .key, startMark: key.mark, endMark: key.mark), atTokenNumber: key.tokenNumber)
            indentEngine.rollIndent(column: key.mark.column, number: key.tokenNumber, type: .blockMappingStart, mark: key.mark)
            simpleKeys[top].possible = false
            simpleKeyAllowed = false
        } else {
            if flowLevel == 0 {
                indentEngine.rollIndent(column: reader.column, number: -1, type: .blockMappingStart, mark: reader.mark)
            }
            simpleKeyAllowed = flowLevel == 0
        }
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: .value, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_plain_scalar`.
    private mutating func fetchPlainScalar() throws {
        mark = reader.mark
        try saveSimpleKey()
        simpleKeyAllowed = false
        let token = try scanPlainScalar()
        indentEngine.enqueue(token)
    }

    private mutating func isDocumentMarker(_ character: Unicode.Scalar) -> Bool {
        reader.peek(0) == character
            && reader.peek(1) == character
            && reader.peek(2) == character
            && Classify.isBlankZ(reader.peek(3))
    }
}