PureYAML/Sources/Parsing/LibYAML.Scanner.Fetch.swift

extension PureYAML.Parsing.LibYAML.Scanner {
    private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
    private typealias Token = PureYAML.Parsing.LibYAML.Token
    private typealias ScannerError = PureYAML.Parsing.LibYAML.ScannerError

    /// Produce every token for the input, mirroring repeated calls to libyaml's
    /// `yaml_parser_fetch_next_token` until STREAM-END. Tokens are never dequeued,
    /// so `tokens_parsed` stays 0 and `QUEUE_INSERT` positions are absolute, which
    /// matches the simple-key splice.
    mutating func scanAll() throws -> [PureYAML.Parsing.LibYAML.Token] {
        while !streamEndProduced {
            try fetchNextToken()
        }
        return indentEngine.tokens
    }

    /// Port of `yaml_parser_scan_to_next_token`: skip a leading BOM, spaces and
    /// tabs, a comment, and a single line break, which re-allows a simple key in
    /// the block context. A tab is valid in-line separation, so it is skipped
    /// here; `tabbedSeparation` records when one preceded the token so a block
    /// indicator or implicit key (where the tab would be indentation) can reject.
    private mutating func scanToNextToken() {
        while true {
            tabbedSeparation = false
            if reader.column == 0, reader.peek() == "\u{FEFF}" {
                reader.skip()
            }
            while reader.peek() == " " || Classify.isTab(reader.peek()) {
                if Classify.isTab(reader.peek()), flowLevel == 0, simpleKeyAllowed {
                    tabbedSeparation = true
                }
                reader.skip()
            }
            // YAML 1.2: a line containing only whitespace is blank even when that
            // whitespace includes tabs. libyaml leaves a leading tab in place and
            // later rejects it; instead consume the trailing blanks so the line
            // break below is reached and the blank line is skipped.
            if blankLineAhead() {
                while reader.peek() == " " || Classify.isTab(reader.peek()) {
                    reader.skip()
                }
            }
            if reader.peek() == "#" {
                while !Classify.isBreakZ(reader.peek()) {
                    reader.skip()
                }
            }
            if Classify.isBreak(reader.peek()) {
                reader.skipLine()
                if flowLevel == 0 {
                    simpleKeyAllowed = true
                }
            } else {
                break
            }
        }
    }

    /// Port of `yaml_parser_fetch_next_token`: emit STREAM-START first, otherwise
    /// skip to the next token, expire stale simple keys, unroll the indentation to
    /// the current column, and dispatch on the leading character. Constructs not
    /// yet ported throw an explicit error rather than mis-scan.
    mutating func fetchNextToken() throws {
        if !streamStartProduced {
            fetchStreamStart()
            return
        }

        scanToNextToken()
        mark = reader.mark
        try staleSimpleKeys()
        indentEngine.unrollIndent(column: reader.column, mark: reader.mark)
        mark = reader.mark
        try dispatchToken()
    }

    /// Dispatch on the leading character: stream end, directives and document
    /// markers at column 0, then the indicators.
    private mutating func dispatchToken() throws {
        let character = reader.peek()
        if Classify.isZ(character) {
            fetchStreamEnd()
            return
        }
        if reader.column == 0, character == "%" {
            try fetchDirective()
            return
        }
        if reader.column == 0, isDocumentMarker("-") {
            fetchDocumentIndicator(.documentStart)
            return
        }
        if reader.column == 0, isDocumentMarker(".") {
            fetchDocumentIndicator(.documentEnd)
            return
        }
        guard let character else {
            return
        }
        try dispatchIndicator(character)
    }

    /// Dispatch the block-structure indicators and plain scalars; unported
    /// constructs throw an explicit error.
    private mutating func dispatchIndicator(_ character: Unicode.Scalar) throws {
        if try fetchFlowToken(character) {
            return
        }
        if try fetchNodeToken(character) {
            return
        }
        switch character {
        case "-" where Classify.isBlankZ(reader.peek(1)):
            try fetchBlockEntry()
        case "?" where Classify.isBlankZ(reader.peek(1)) || (flowLevel > 0 && Classify.isFlowIndicator(reader.peek(1))):
            // In flow context libyaml treats every `?` as a key indicator, but per
            // `ns-plain-first` a `?` followed by a flow-safe character (anything but
            // whitespace, end, or a flow indicator) begins a plain scalar (`?foo`).
            try fetchKey()
        case ":" where Classify.isBlankZ(reader.peek(1))
            || (flowLevel > 0 && (Classify.isFlowIndicator(reader.peek(1)) || flowKeyIsJSON)):
            // In flow context libyaml makes every `:` a value indicator. Per the
            // spec a `:` only ends a node when followed by whitespace, end of
            // input, or a flow indicator, except an adjacent `:` after a JSON
            // (quoted) key. Otherwise the `:` is plain-scalar content (`:x`).
            try fetchValue()
        default:
            // A plain scalar may begin with any non-blank character; a blank (a
            // tab here, since spaces and breaks were already consumed) cannot
            // start a token, matching libyaml's final guard in fetch_next_token.
            guard !Classify.isBlankZ(character) else {
                throw ScannerError(
                    context: "while scanning for the next token",
                    problem: "found character that cannot start any token",
                    mark: reader.mark,
                )
            }
            try fetchPlainScalar()
        }
    }

    /// The error raised when a tab is used where indentation is required (before a
    /// block indicator or an implicit mapping key). A tab is valid separation but
    /// never indentation.
    private func tabIndentationError() -> PureYAML.Parsing.LibYAML.ScannerError {
        ScannerError(
            context: "while scanning for the next token",
            problem: "found a tab character where an indentation space is expected",
            mark: reader.mark,
        )
    }

    /// Whether this flow level's pending simple key is a JSON (quoted) node, which
    /// lets an adjacent `:` open its value without a separating space.
    private var flowKeyIsJSON: Bool {
        let key = simpleKeys[simpleKeys.count - 1]
        return key.possible && key.isJSON
    }

    /// Fetch a flow-collection indicator (`[` `]` `{` `}` `,`); returns false
    /// otherwise.
    private mutating func fetchFlowToken(_ character: Unicode.Scalar) throws -> Bool {
        switch character {
        case "[": fetchFlowCollectionStart(.flowSequenceStart)
        case "{": fetchFlowCollectionStart(.flowMappingStart)
        case "]": try fetchFlowCollectionEnd(.flowSequenceEnd)
        case "}": try fetchFlowCollectionEnd(.flowMappingEnd)
        case ",": try fetchFlowEntry()
        default: return false
        }
        return true
    }

    /// Fetch a quoted scalar, an anchor/alias, or a block scalar; returns false
    /// otherwise.
    private mutating func fetchNodeToken(_ character: Unicode.Scalar) throws -> Bool {
        switch character {
        case "'": try fetchFlowScalar(single: true)
        case "\"": try fetchFlowScalar(single: false)
        case "&": try fetchAnchor(isAnchor: true)
        case "*": try fetchAnchor(isAnchor: false)
        case "!": try fetchTag()
        case "|" where flowLevel == 0: try fetchBlockScalar(literal: true)
        case ">" where flowLevel == 0: try fetchBlockScalar(literal: false)
        default: return false
        }
        return true
    }

    /// Port of `yaml_parser_fetch_tag`.
    private mutating func fetchTag() throws {
        mark = reader.mark
        try saveSimpleKey()
        simpleKeyAllowed = false
        let token = try scanTag()
        indentEngine.enqueue(token)
    }

    /// Port of `yaml_parser_fetch_block_scalar`.
    private mutating func fetchBlockScalar(literal: Bool) throws {
        try removeSimpleKey()
        simpleKeyAllowed = true
        let token = try scanBlockScalar(literal: literal)
        indentEngine.enqueue(token)
    }

    /// Port of `yaml_parser_fetch_flow_scalar`.
    private mutating func fetchFlowScalar(single: Bool) throws {
        mark = reader.mark
        let saved = simpleKeyAllowed
        try saveSimpleKey()
        // A quoted scalar that became this level's simple key is a JSON node, so a
        // later adjacent `:` (no space) still opens its value.
        if saved {
            simpleKeys[simpleKeys.count - 1].isJSON = true
        }
        simpleKeyAllowed = false
        let token = try scanFlowScalar(single: single)
        indentEngine.enqueue(token)
    }

    /// Port of `yaml_parser_fetch_anchor`.
    private mutating func fetchAnchor(isAnchor: Bool) throws {
        mark = reader.mark
        try saveSimpleKey()
        simpleKeyAllowed = false
        let token = try scanAnchor(isAnchor: isAnchor)
        indentEngine.enqueue(token)
    }

    /// Port of `yaml_parser_fetch_flow_collection_start`.
    private mutating func fetchFlowCollectionStart(_ type: PureYAML.Parsing.LibYAML.TokenType) {
        // saveSimpleKey cannot throw in flow: a required key needs block context.
        let saved = simpleKeyAllowed
        try? saveSimpleKey()
        // A flow collection is a JSON node, so when it is this level's simple key
        // an adjacent `:` after its closing `]`/`}` still opens its value.
        if saved {
            simpleKeys[simpleKeys.count - 1].isJSON = true
        }
        increaseFlowLevel()
        simpleKeyAllowed = true
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: type, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_flow_collection_end`.
    private mutating func fetchFlowCollectionEnd(_ type: PureYAML.Parsing.LibYAML.TokenType) throws {
        try removeSimpleKey()
        decreaseFlowLevel()
        simpleKeyAllowed = false
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: type, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_flow_entry`.
    private mutating func fetchFlowEntry() throws {
        try removeSimpleKey()
        simpleKeyAllowed = true
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: .flowEntry, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_stream_start`.
    private mutating func fetchStreamStart() {
        // indent is already -1 and simpleKeys already holds the block-context slot.
        simpleKeyAllowed = true
        streamStartProduced = true
        indentEngine.enqueue(Token(type: .streamStart, startMark: reader.mark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_stream_end`.
    private mutating func fetchStreamEnd() {
        if reader.column != 0 {
            // Force a new line in the mark, as libyaml does.
            mark = PureYAML.Parsing.Mark(line: reader.line + 1, column: 0, index: reader.index)
        }
        indentEngine.unrollIndent(column: -1, mark: mark)
        // remove_simple_key cannot throw here: a required key would already have
        // been resolved before stream end in the block + plain core.
        simpleKeys[simpleKeys.count - 1].possible = false
        simpleKeyAllowed = false
        streamEndProduced = true
        indentEngine.enqueue(Token(type: .streamEnd, startMark: mark, endMark: mark))
    }

    /// Port of `yaml_parser_fetch_document_indicator`.
    private mutating func fetchDocumentIndicator(_ type: PureYAML.Parsing.LibYAML.TokenType) {
        indentEngine.unrollIndent(column: -1, mark: reader.mark)
        simpleKeys[simpleKeys.count - 1].possible = false
        simpleKeyAllowed = false
        let startMark = reader.mark
        reader.skip()
        reader.skip()
        reader.skip()
        indentEngine.enqueue(Token(type: type, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_block_entry`.
    private mutating func fetchBlockEntry() throws {
        if tabbedSeparation {
            throw tabIndentationError()
        }
        if flowLevel == 0 {
            indentEngine.rollIndent(column: reader.column, number: -1, type: .blockSequenceStart, mark: reader.mark)
        }
        simpleKeys[simpleKeys.count - 1].possible = false
        simpleKeyAllowed = true
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: .blockEntry, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_key`.
    private mutating func fetchKey() throws {
        if tabbedSeparation {
            throw tabIndentationError()
        }
        if flowLevel == 0 {
            indentEngine.rollIndent(column: reader.column, number: -1, type: .blockMappingStart, mark: reader.mark)
        }
        simpleKeys[simpleKeys.count - 1].possible = false
        simpleKeyAllowed = flowLevel == 0
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: .key, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_value`, including the simple-key splice: a
    /// possible simple key inserts its `KEY` token at the recorded position and
    /// rolls the block mapping indent to the key's column.
    private mutating func fetchValue() throws {
        let top = simpleKeys.count - 1
        if simpleKeys[top].possible {
            if simpleKeys[top].tabbed {
                throw tabIndentationError()
            }
            let key = simpleKeys[top]
            indentEngine.insert(Token(type: .key, startMark: key.mark, endMark: key.mark), atTokenNumber: key.tokenNumber)
            indentEngine.rollIndent(column: key.mark.column, number: key.tokenNumber, type: .blockMappingStart, mark: key.mark)
            simpleKeys[top].possible = false
            simpleKeyAllowed = false
        } else {
            if flowLevel == 0 {
                indentEngine.rollIndent(column: reader.column, number: -1, type: .blockMappingStart, mark: reader.mark)
            }
            simpleKeyAllowed = flowLevel == 0
        }
        let startMark = reader.mark
        reader.skip()
        indentEngine.enqueue(Token(type: .value, startMark: startMark, endMark: reader.mark))
    }

    /// Port of `yaml_parser_fetch_plain_scalar`.
    private mutating func fetchPlainScalar() throws {
        mark = reader.mark
        try saveSimpleKey()
        simpleKeyAllowed = false
        let token = try scanPlainScalar()
        indentEngine.enqueue(token)
    }

    /// Whether the rest of the current line, from the current position, is only
    /// spaces and tabs up to a line break or end of stream (a blank line).
    private func blankLineAhead() -> Bool {
        var offset = 0
        while let character = reader.peek(offset), character == " " || Classify.isTab(character) {
            offset += 1
        }
        return Classify.isBreakZ(reader.peek(offset))
    }

    private func isDocumentMarker(_ character: Unicode.Scalar) -> Bool {
        reader.peek(0) == character
            && reader.peek(1) == character
            && reader.peek(2) == character
            && Classify.isBlankZ(reader.peek(3))
    }
}