diff --git a/Sources/Parsing/LibYAML.IndentEngine.swift b/Sources/Parsing/LibYAML.IndentEngine.swift index cb574ee..1ef725f 100644 --- a/Sources/Parsing/LibYAML.IndentEngine.swift +++ b/Sources/Parsing/LibYAML.IndentEngine.swift @@ -61,6 +61,17 @@ extension PureYAML.Parsing.LibYAML { } } + /// `ENQUEUE`: append a token to the queue. + mutating func enqueue(_ token: Token) { + tokens.append(token) + } + + /// `QUEUE_INSERT`: insert a token at an absolute token number, used to + /// splice a `KEY` token in front of a node already queued. + mutating func insert(_ token: Token, atTokenNumber number: Int) { + tokens.insert(token, at: number - tokensParsed) + } + /// Dequeue the front token, advancing `tokensParsed` like libyaml's /// `yaml_parser_scan` does when it hands a token to the parser. mutating func dequeue() -> Token? { diff --git a/Sources/Parsing/LibYAML.Scanner.Fetch.swift b/Sources/Parsing/LibYAML.Scanner.Fetch.swift new file mode 100644 index 0000000..0ef9ca4 --- /dev/null +++ b/Sources/Parsing/LibYAML.Scanner.Fetch.swift @@ -0,0 +1,221 @@ +extension PureYAML.Parsing.LibYAML.Scanner { + private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass + private typealias Token = PureYAML.Parsing.LibYAML.Token + private typealias ScannerError = PureYAML.Parsing.LibYAML.ScannerError + + /// Produce every token for the input, mirroring repeated calls to libyaml's + /// `yaml_parser_fetch_next_token` until STREAM-END. Tokens are never dequeued, + /// so `tokens_parsed` stays 0 and `QUEUE_INSERT` positions are absolute, which + /// matches the simple-key splice. + mutating func scanAll() throws -> [PureYAML.Parsing.LibYAML.Token] { + while !streamEndProduced { + try fetchNextToken() + } + return indentEngine.tokens + } + + /// Port of `yaml_parser_scan_to_next_token`: skip a leading BOM, spaces (and + /// tabs only in flow context or where a simple key is not allowed), a comment, + /// and a single line break, which re-allows a simple key in the block context. + private mutating func scanToNextToken() { + while true { + if reader.column == 0, reader.peek() == "\u{FEFF}" { + reader.skip() + } + while reader.peek() == " " || ((flowLevel > 0 || !simpleKeyAllowed) && reader.peek() == "\t") { + reader.skip() + } + if reader.peek() == "#" { + while !Classify.isBreakZ(reader.peek()) { + reader.skip() + } + } + if Classify.isBreak(reader.peek()) { + reader.skipLine() + if flowLevel == 0 { + simpleKeyAllowed = true + } + } else { + break + } + } + } + + /// Port of `yaml_parser_fetch_next_token`: emit STREAM-START first, otherwise + /// skip to the next token, expire stale simple keys, unroll the indentation to + /// the current column, and dispatch on the leading character. Constructs not + /// yet ported throw an explicit error rather than mis-scan. + mutating func fetchNextToken() throws { + if !streamStartProduced { + fetchStreamStart() + return + } + + scanToNextToken() + mark = reader.mark + try staleSimpleKeys() + indentEngine.unrollIndent(column: reader.column, mark: reader.mark) + mark = reader.mark + try dispatchToken() + } + + /// Dispatch on the leading character: stream end, directives and document + /// markers at column 0, then the indicators. + private mutating func dispatchToken() throws { + let character = reader.peek() + if Classify.isZ(character) { + fetchStreamEnd() + return + } + if reader.column == 0, character == "%" { + throw notPorted("directive") + } + if reader.column == 0, isDocumentMarker("-") { + fetchDocumentIndicator(.documentStart) + return + } + if reader.column == 0, isDocumentMarker(".") { + fetchDocumentIndicator(.documentEnd) + return + } + guard let character else { + return + } + try dispatchIndicator(character) + } + + /// Dispatch the block-structure indicators and plain scalars; unported + /// constructs throw an explicit error. + private mutating func dispatchIndicator(_ character: Unicode.Scalar) throws { + if let unported = unportedConstruct(character) { + throw notPorted(unported) + } + switch character { + case "-" where Classify.isBlankZ(reader.peek(1)): + fetchBlockEntry() + case "?" where flowLevel > 0 || Classify.isBlankZ(reader.peek(1)): + fetchKey() + case ":" where flowLevel > 0 || Classify.isBlankZ(reader.peek(1)): + try fetchValue() + default: + try fetchPlainScalar() + } + } + + /// The constructs whose fetchers are not yet ported, named for the error. + private func unportedConstruct(_ character: Unicode.Scalar) -> String? { + switch character { + case "[", "]", "{", "}", ",": "flow collection" + case "*", "&": "anchor or alias" + case "!": "tag" + case "|" where flowLevel == 0, ">" where flowLevel == 0: "block scalar" + case "'", "\"": "flow scalar" + default: nil + } + } + + /// Port of `yaml_parser_fetch_stream_start`. + private mutating func fetchStreamStart() { + // indent is already -1 and simpleKeys already holds the block-context slot. + simpleKeyAllowed = true + streamStartProduced = true + indentEngine.enqueue(Token(type: .streamStart, startMark: reader.mark, endMark: reader.mark)) + } + + /// Port of `yaml_parser_fetch_stream_end`. + private mutating func fetchStreamEnd() { + if reader.column != 0 { + // Force a new line in the mark, as libyaml does. + mark = PureYAML.Parsing.Mark(line: reader.line + 1, column: 0, index: reader.index) + } + indentEngine.unrollIndent(column: -1, mark: mark) + // remove_simple_key cannot throw here: a required key would already have + // been resolved before stream end in the block + plain core. + simpleKeys[simpleKeys.count - 1].possible = false + simpleKeyAllowed = false + streamEndProduced = true + indentEngine.enqueue(Token(type: .streamEnd, startMark: mark, endMark: mark)) + } + + /// Port of `yaml_parser_fetch_document_indicator`. + private mutating func fetchDocumentIndicator(_ type: PureYAML.Parsing.LibYAML.TokenType) { + indentEngine.unrollIndent(column: -1, mark: reader.mark) + simpleKeys[simpleKeys.count - 1].possible = false + simpleKeyAllowed = false + let startMark = reader.mark + reader.skip() + reader.skip() + reader.skip() + indentEngine.enqueue(Token(type: type, startMark: startMark, endMark: reader.mark)) + } + + /// Port of `yaml_parser_fetch_block_entry`. + private mutating func fetchBlockEntry() { + if flowLevel == 0 { + indentEngine.rollIndent(column: reader.column, number: -1, type: .blockSequenceStart, mark: reader.mark) + } + simpleKeys[simpleKeys.count - 1].possible = false + simpleKeyAllowed = true + let startMark = reader.mark + reader.skip() + indentEngine.enqueue(Token(type: .blockEntry, startMark: startMark, endMark: reader.mark)) + } + + /// Port of `yaml_parser_fetch_key`. + private mutating func fetchKey() { + if flowLevel == 0 { + indentEngine.rollIndent(column: reader.column, number: -1, type: .blockMappingStart, mark: reader.mark) + } + simpleKeys[simpleKeys.count - 1].possible = false + simpleKeyAllowed = flowLevel == 0 + let startMark = reader.mark + reader.skip() + indentEngine.enqueue(Token(type: .key, startMark: startMark, endMark: reader.mark)) + } + + /// Port of `yaml_parser_fetch_value`, including the simple-key splice: a + /// possible simple key inserts its `KEY` token at the recorded position and + /// rolls the block mapping indent to the key's column. + private mutating func fetchValue() throws { + let top = simpleKeys.count - 1 + if simpleKeys[top].possible { + let key = simpleKeys[top] + indentEngine.insert(Token(type: .key, startMark: key.mark, endMark: key.mark), atTokenNumber: key.tokenNumber) + indentEngine.rollIndent(column: key.mark.column, number: key.tokenNumber, type: .blockMappingStart, mark: key.mark) + simpleKeys[top].possible = false + simpleKeyAllowed = false + } else { + if flowLevel == 0 { + indentEngine.rollIndent(column: reader.column, number: -1, type: .blockMappingStart, mark: reader.mark) + } + simpleKeyAllowed = flowLevel == 0 + } + let startMark = reader.mark + reader.skip() + indentEngine.enqueue(Token(type: .value, startMark: startMark, endMark: reader.mark)) + } + + /// Port of `yaml_parser_fetch_plain_scalar`. + private mutating func fetchPlainScalar() throws { + mark = reader.mark + try saveSimpleKey() + simpleKeyAllowed = false + let token = try scanPlainScalar() + indentEngine.enqueue(token) + } + + private func isDocumentMarker(_ character: Unicode.Scalar) -> Bool { + reader.peek(0) == character + && reader.peek(1) == character + && reader.peek(2) == character + && Classify.isBlankZ(reader.peek(3)) + } + + private func notPorted(_ what: String) -> ScannerError { + ScannerError( + context: "while scanning a token", + problem: "construct not yet ported: \(what)", + mark: reader.mark, + ) + } +} diff --git a/Sources/Parsing/LibYAML.Scanner.PlainScalar.swift b/Sources/Parsing/LibYAML.Scanner.PlainScalar.swift new file mode 100644 index 0000000..0da6474 --- /dev/null +++ b/Sources/Parsing/LibYAML.Scanner.PlainScalar.swift @@ -0,0 +1,143 @@ +extension PureYAML.Parsing.LibYAML.Scanner { + private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass + + /// The folding state carried through `scan_plain_scalar`: the accumulated + /// value plus the pending leading break, trailing breaks, and whitespace that + /// are folded in before the next content character. + private struct PlainState { + var string = "" + var leadingBreak = "" + var trailingBreaks = "" + var whitespaces = "" + var leadingBlanks = false + var endMark: PureYAML.Parsing.Mark + } + + /// Port of `yaml_parser_scan_plain_scalar`. Consumes a plain scalar, folding + /// line breaks: a single break between non-empty lines becomes a space, blank + /// lines become literal newlines, and leading/trailing whitespace on folded + /// lines is dropped. Ends at a `: ` value indicator, a comment, a less-indented + /// line (block context), a flow indicator (flow context), or a document marker. + mutating func scanPlainScalar() throws -> PureYAML.Parsing.LibYAML.Token { + let indent = indentEngine.indent + 1 + let startMark = reader.mark + var state = PlainState(endMark: reader.mark) + + while true { + if reader.column == 0, documentMarkerAhead() { + break + } + if reader.peek() == "#" { + break + } + try consumePlainContent(&state, startMark: startMark) + guard Classify.isBlank(reader.peek()) || Classify.isBreak(reader.peek()) else { + break + } + try consumePlainBreaks(&state, indent: indent, startMark: startMark) + if flowLevel == 0, reader.column < indent { + break + } + } + + if state.leadingBlanks { + simpleKeyAllowed = true + } + return PureYAML.Parsing.LibYAML.Token( + type: .scalar(value: state.string, style: .plain), + startMark: startMark, + endMark: state.endMark, + ) + } + + /// Consume the run of non-blank characters, folding any pending breaks or + /// whitespace ahead of each one. + private mutating func consumePlainContent(_ state: inout PlainState, startMark: PureYAML.Parsing.Mark) throws { + while let character = reader.peek(), !Classify.isBlankZ(character) { + if character == ":", Classify.isBlankZ(reader.peek(1)) { + break + } + if flowLevel > 0, character == ":", isFlowIndicator(reader.peek(1)) { + throw scannerError("found unexpected ':'", at: startMark) + } + if flowLevel > 0, isFlowIndicator(character) { + break + } + foldPending(&state) + state.string.unicodeScalars.append(character) + reader.skip() + state.endMark = reader.mark + } + } + + /// Fold the pending leading break, trailing breaks, and whitespace into the + /// accumulated value (the join logic from libyaml's inner loop). + private func foldPending(_ state: inout PlainState) { + guard state.leadingBlanks || !state.whitespaces.isEmpty else { + return + } + guard state.leadingBlanks else { + state.string += state.whitespaces + state.whitespaces = "" + return + } + if state.leadingBreak.first == "\n" { + if state.trailingBreaks.isEmpty { + state.string += " " + } else { + state.string += state.trailingBreaks + state.trailingBreaks = "" + } + state.leadingBreak = "" + } else { + state.string += state.leadingBreak + state.string += state.trailingBreaks + state.leadingBreak = "" + state.trailingBreaks = "" + } + state.leadingBlanks = false + } + + /// Consume the run of blanks and line breaks between content, recording them as + /// pending whitespace or folded breaks. + private mutating func consumePlainBreaks(_ state: inout PlainState, indent: Int, startMark: PureYAML.Parsing.Mark) throws { + while Classify.isBlank(reader.peek()) || Classify.isBreak(reader.peek()) { + if Classify.isBlank(reader.peek()) { + if state.leadingBlanks, reader.column < indent, Classify.isTab(reader.peek()) { + throw scannerError("found a tab character that violates indentation", at: startMark) + } + if !state.leadingBlanks, let blank = reader.peek() { + state.whitespaces.unicodeScalars.append(blank) + } + reader.skip() + } else if !state.leadingBlanks { + state.whitespaces = "" + reader.skipLine() + state.leadingBreak = "\n" + state.leadingBlanks = true + } else { + reader.skipLine() + state.trailingBreaks += "\n" + } + } + } + + private func scannerError(_ problem: String, at mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError { + PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a plain scalar", problem: problem, mark: mark) + } + + private func documentMarkerAhead() -> Bool { + let first = reader.peek() + guard first == "-" || first == "." else { + return false + } + return reader.peek(1) == first && reader.peek(2) == first && Classify.isBlankZ(reader.peek(3)) + } + + private func isFlowIndicator(_ scalar: Unicode.Scalar?) -> Bool { + switch scalar { + case ",", "[", "]", "{", "}": true + default: false + } + } +} diff --git a/Sources/Parsing/LibYAML.Scanner.swift b/Sources/Parsing/LibYAML.Scanner.swift index 9a60f82..fc75c99 100644 --- a/Sources/Parsing/LibYAML.Scanner.swift +++ b/Sources/Parsing/LibYAML.Scanner.swift @@ -5,6 +5,8 @@ extension PureYAML.Parsing.LibYAML { /// Column values follow libyaml's 0-based convention internally; conversion to /// the package's 1-based `Mark` happens at token emission during integration. struct Scanner { + /// The input buffer and position (`parser->buffer` / `parser->mark`). + var reader: Reader /// The indentation stack and `roll_indent` / `unroll_indent` (also the /// owner of `flow_level` and the token queue). var indentEngine = IndentEngine() @@ -12,8 +14,16 @@ extension PureYAML.Parsing.LibYAML { var simpleKeys: [SimpleKey] = [.empty] /// `parser->simple_key_allowed`. True at stream start. var simpleKeyAllowed = true - /// `parser->mark`, the current position (column is 0-based here). + /// `parser->mark`, kept in sync with the reader at each fetch step. var mark: PureYAML.Parsing.Mark = .start + /// `parser->stream_start_produced`. + var streamStartProduced = false + /// `parser->stream_end_produced`. + var streamEndProduced = false + + init(_ input: String = "") { + reader = Reader(input) + } /// `parser->flow_level`, kept on the indent engine. var flowLevel: Int { diff --git a/Sources/Parsing/LibYAML.TokenType.swift b/Sources/Parsing/LibYAML.TokenType.swift index 2752582..7d04fae 100644 --- a/Sources/Parsing/LibYAML.TokenType.swift +++ b/Sources/Parsing/LibYAML.TokenType.swift @@ -1,10 +1,18 @@ extension PureYAML.Parsing.LibYAML { - /// A scanner token kind, mirroring `yaml_token_type_t`. Only the structural - /// kinds the indentation engine produces are modelled here; scalar, anchor, - /// tag, and document kinds are added as their fetchers are ported. + /// A scanner token kind, mirroring `yaml_token_type_t`. The flow-collection, + /// anchor, tag, alias, and directive kinds are added as their fetchers are + /// ported; the block + plain-scalar core is present. enum TokenType: Equatable { + case streamStart + case streamEnd + case documentStart + case documentEnd case blockSequenceStart case blockMappingStart case blockEnd + case blockEntry + case key + case value + case scalar(value: String, style: PureYAML.Parsing.ScalarStyle) } } diff --git a/Tests/LibYAMLScannerFetchTests.swift b/Tests/LibYAMLScannerFetchTests.swift new file mode 100644 index 0000000..e3a0e90 --- /dev/null +++ b/Tests/LibYAMLScannerFetchTests.swift @@ -0,0 +1,83 @@ +@testable import PureYAML +import Testing + +/// End-to-end tokenization through the ported fetch loop. The headline case is +/// the original block-mapping bug: `- a:` followed by a same-column `b:` must +/// tokenize as siblings (one BLOCK-MAPPING-START), matching libyaml, while a +/// more-indented value nests (two BLOCK-MAPPING-STARTs). +@Suite("LibYAML Scanner Fetch") +struct LibYAMLScannerFetchTests { + private typealias TokenType = PureYAML.Parsing.LibYAML.TokenType + + private func tokenize(_ input: String) throws -> [TokenType] { + var scanner = PureYAML.Parsing.LibYAML.Scanner(input) + return try scanner.scanAll().map(\.type) + } + + @Test("a block mapping inside a sequence entry keeps same-column keys siblings") + func test_sequenceEntryMappingSiblings() throws { + // The original bug: `- a:` / ` b: c` is one mapping {a: null, b: c}. + let tokens = try tokenize("- a:\n b: c\n") + #expect(tokens == [ + .streamStart, + .blockSequenceStart, + .blockEntry, + .blockMappingStart, + .key, + .scalar(value: "a", style: .plain), + .value, + .key, + .scalar(value: "b", style: .plain), + .value, + .scalar(value: "c", style: .plain), + .blockEnd, + .blockEnd, + .streamEnd, + ]) + } + + @Test("a more-indented value nests a second mapping") + func test_moreIndentedValueNests() throws { + let tokens = try tokenize("a:\n b: c\n") + #expect(tokens == [ + .streamStart, + .blockMappingStart, + .key, + .scalar(value: "a", style: .plain), + .value, + .blockMappingStart, + .key, + .scalar(value: "b", style: .plain), + .value, + .scalar(value: "c", style: .plain), + .blockEnd, + .blockEnd, + .streamEnd, + ]) + } + + @Test("a simple block mapping pair") + func test_simplePair() throws { + let tokens = try tokenize("key: value\n") + #expect(tokens == [ + .streamStart, + .blockMappingStart, + .key, + .scalar(value: "key", style: .plain), + .value, + .scalar(value: "value", style: .plain), + .blockEnd, + .streamEnd, + ]) + } + + @Test("a multi-line plain scalar folds its line break to a space") + func test_plainScalarFolding() throws { + let tokens = try tokenize("a\nb\n") + #expect(tokens == [ + .streamStart, + .scalar(value: "a b", style: .plain), + .streamEnd, + ]) + } +}