PureYAML/Sources/Parsing/Scanner.swift

extension PureYAML.Parsing {
    struct Scanner {
        init() {}

        func scan(_ yaml: String) throws -> [Token] {
            var state = State(reader: Reader(yaml))
            state.append(.streamStart)

            while !state.reader.isAtEnd {
                if isLineBreak(state.reader.peek()) {
                    state.reader.advance()
                    captureBlockScalarBlankLine(&state)
                    state.isAtLineStart = true
                    state.sawContentOnLine = false
                    continue
                }

                if state.isAtLineStart {
                    try scanIndentation(&state)
                    if state.reader.isAtEnd {
                        break
                    }
                    if isLineBreak(state.reader.peek()) {
                        continue
                    }
                    if try scanDirectiveOrDocumentMarker(&state) {
                        continue
                    }
                    state.isAtLineStart = false
                }

                try scanToken(&state)
            }

            closeIndentation(&state)
            try enforceDirectiveHasDocument(state)
            state.append(.streamEnd)
            return state.tokens
        }
    }
}

extension PureYAML.Parsing.Scanner {
    struct State {
        var reader: PureYAML.Parsing.Reader
        var tokens: [PureYAML.Parsing.Token] = []
        var indentation: [Int] = [0]
        var isAtLineStart = true
        var hasDocumentContent = false
        var hasDocumentStartMarker = false
        var isDocumentClosed = false
        var hasYAMLDirective = false
        // The line of a directive awaiting its document. A directive must be followed by an explicit
        // `---`; this is set when one is scanned, cleared at a document start, and checked at a
        // document end and at stream end so a directive with no document is rejected.
        var pendingDirectiveLine: Int?
        var flowDepth = 0
        var flowContexts: [PureYAML.Parsing.ScannerFlowContext] = []
        var validImplicitIndentation: Set<Int> = []
        var pendingBlockScalarIndentation = false
        var pendingBlockScalarIndentationIndicator: Int?
        var pendingBlockScalarLeadingBlankLines = 0
        var blockScalarIndentation: Int?
        var blockScalarContentPrefix = ""
        // Whether a real token has been scanned on the current physical line. Reset when the scan
        // loop consumes a line break and set as each token is scanned, so the scanner can tell a
        // comment that opens a line (valid) from one glued to earlier content on it (invalid). A
        // simple flag, not a token-line comparison, because a blank line inside a block scalar
        // emits an empty scalar attributed to the next line, which would read as phantom content.
        var sawContentOnLine = false
        var tagHandles = [
            "!": "!",
            "!!": "tag:yaml.org,2002:",
        ]

        mutating func append(_ kind: PureYAML.Parsing.TokenKind) {
            let mark = reader.mark
            tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: mark))
            markDocumentContentIfNeeded(kind)
        }

        mutating func append(
            _ kind: PureYAML.Parsing.TokenKind,
            mark: PureYAML.Parsing.Mark,
            endMark: PureYAML.Parsing.Mark,
        ) {
            tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: endMark))
            markDocumentContentIfNeeded(kind)
        }

        mutating func markDocumentContentIfNeeded(_ kind: PureYAML.Parsing.TokenKind) {
            switch kind {
            case .comment, .dedent, .documentEnd, .documentStart, .indent, .streamEnd, .streamStart:
                return
            default:
                hasDocumentContent = true
            }
        }

        var isExpectingFlowMappingKey: Bool {
            flowContexts.last?.kind == .mapping
                && flowContexts.last?.isExpectingKey == true
        }
    }
}

extension PureYAML.Parsing.Scanner {
    func scanToken(_ state: inout State) throws {
        let columnBeforeSeparation = state.reader.mark.column
        skipSeparation(&state)
        let consumedSeparation = state.reader.mark.column > columnBeforeSeparation
        guard let character = state.reader.peek(), !isLineBreak(character) else {
            return
        }

        if state.blockScalarIndentation != nil {
            scanBlockScalarContent(&state)
            return
        }
        // A `#` begins a comment only at the start of a line's content or after whitespace. Glued to
        // earlier same-line content (e.g. `"x"#c`, `]#c`, `>#c`) it is not a plain-scalar character
        // either, so the document is malformed; a plain scalar's own `#` is consumed before reaching
        // here, so this rejects only the genuinely invalid case.
        if character == "#", !consumedSeparation, state.sawContentOnLine {
            throw PureYAML.Parsing.ParseError.commentNotPrecededByWhitespace(
                line: state.reader.mark.line,
                column: state.reader.mark.column,
            )
        }
        state.sawContentOnLine = true
        if try scanLineIndicatorToken(character, state: &state) {
            return
        }
        if scanFlowToken(character, state: &state) {
            return
        }
        if try scanQuotedOrBlockScalarToken(character, state: &state) {
            return
        }
        if try scanNodePropertyToken(character, state: &state) {
            return
        }
        scanPlainScalar(&state)
    }

    func scanLineIndicatorToken(
        _ character: Character,
        state: inout State,
    ) throws -> Bool {
        switch character {
        case "#":
            scanComment(&state)
            return true
        case "-" where isBlockEntry(state.reader):
            scanFixedToken(.blockEntry, state: &state)
            return true
        case "?" where isIndicatorBoundary(state.reader.peek(offset: 1)):
            scanFixedToken(.mappingKey, state: &state)
            return true
        case ":" where isAtMappingValueIndicator(state: state):
            scanFixedToken(.mappingValue, state: &state)
            return true
        default:
            return false
        }
    }

    func scanFlowToken(
        _ character: Character,
        state: inout State,
    ) -> Bool {
        switch character {
        case "[":
            state.flowDepth += 1
            state.flowContexts.append(.init(kind: .sequence, isExpectingKey: false))
            scanFixedToken(.flowSequenceStart, state: &state)
            return true
        case "]":
            state.flowDepth = max(0, state.flowDepth - 1)
            scanFixedToken(.flowSequenceEnd, state: &state)
            _ = state.flowContexts.popLast()
            return true
        case "{":
            state.flowDepth += 1
            state.flowContexts.append(.init(kind: .mapping, isExpectingKey: true))
            scanFixedToken(.flowMappingStart, state: &state)
            return true
        case "}":
            state.flowDepth = max(0, state.flowDepth - 1)
            scanFixedToken(.flowMappingEnd, state: &state)
            _ = state.flowContexts.popLast()
            return true
        case ",":
            scanFixedToken(.flowEntry, state: &state)
            if state.flowContexts.last?.kind == .mapping {
                state.flowContexts[state.flowContexts.count - 1].isExpectingKey = true
            }
            return true
        default:
            return false
        }
    }

    func scanNodePropertyToken(
        _ character: Character,
        state: inout State,
    ) throws -> Bool {
        switch character {
        case "&":
            try scanNamedToken(
                kind: PureYAML.Parsing.TokenKind.anchor,
                missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName,
                state: &state,
            )
            return true
        case "*":
            try scanNamedToken(
                kind: PureYAML.Parsing.TokenKind.alias,
                missingNameError: PureYAML.Parsing.ParseError.expectedAliasName,
                state: &state,
            )
            return true
        case "!":
            try scanTag(&state)
            return true
        default:
            return false
        }
    }

    func scanQuotedOrBlockScalarToken(
        _ character: Character,
        state: inout State,
    ) throws -> Bool {
        switch character {
        case "\"":
            try scanQuotedScalar(.doubleQuoted, state: &state)
            return true
        case "'":
            try scanQuotedScalar(.singleQuoted, state: &state)
            return true
        case "|":
            scanBlockScalarHeader(.literal, state: &state)
            return true
        case ">":
            scanBlockScalarHeader(.folded, state: &state)
            return true
        default:
            return false
        }
    }

    func skipSeparation(_ state: inout State) {
        while state.reader.peek() == " " {
            state.reader.advance()
        }
    }

    func scanFixedToken(
        _ kind: PureYAML.Parsing.TokenKind,
        state: inout State,
    ) {
        let start = state.reader.mark
        state.reader.advance()
        state.append(kind, mark: start, endMark: state.reader.mark)
        if case .blockEntry = kind, hasNodeOnSameLine(afterCurrentPositionIn: state.reader) {
            state.validImplicitIndentation.insert(start.column + 1)
        }
        if case .mappingValue = kind, state.flowContexts.last?.kind == .mapping {
            state.flowContexts[state.flowContexts.count - 1].isExpectingKey = false
        }
    }
}