Mihaela Mihaljevic
43b20ab7bc
Stage 3 of the faithful libyaml scanner port: the token producer. Translates scan_to_next_token, fetch_more_tokens/fetch_next_token dispatch, and the fetchers for stream start/end, document indicators, block entry, key, value (with the simple-key splice and roll-indent), plus scan_plain_scalar with its line-break folding. Reuses the ported indent engine, simple-key mechanism, character classes, and reader. Constructs not yet ported (flow collections, quoted scalars, anchors, tags, directives, block scalars) throw an explicit 'not yet ported' error rather than mis-scan; the port is not yet wired to the public API. Milestone: the ported scanner tokenizes '- a:' followed by a same-column 'b:' as siblings (one BLOCK-MAPPING-START), matching libyaml, fixing the original block-mapping bug by construction; a more-indented value still nests. Unit-tested for the sibling case, the nested case, a simple pair, and multi-line plain-scalar folding. 305 tests, no regression.
84 lines
2.6 KiB
Swift
84 lines
2.6 KiB
Swift
@testable import PureYAML
|
|
import Testing
|
|
|
|
/// End-to-end tokenization through the ported fetch loop. The headline case is
|
|
/// the original block-mapping bug: `- a:` followed by a same-column `b:` must
|
|
/// tokenize as siblings (one BLOCK-MAPPING-START), matching libyaml, while a
|
|
/// more-indented value nests (two BLOCK-MAPPING-STARTs).
|
|
@Suite("LibYAML Scanner Fetch")
|
|
struct LibYAMLScannerFetchTests {
|
|
private typealias TokenType = PureYAML.Parsing.LibYAML.TokenType
|
|
|
|
private func tokenize(_ input: String) throws -> [TokenType] {
|
|
var scanner = PureYAML.Parsing.LibYAML.Scanner(input)
|
|
return try scanner.scanAll().map(\.type)
|
|
}
|
|
|
|
@Test("a block mapping inside a sequence entry keeps same-column keys siblings")
|
|
func test_sequenceEntryMappingSiblings() throws {
|
|
// The original bug: `- a:` / ` b: c` is one mapping {a: null, b: c}.
|
|
let tokens = try tokenize("- a:\n b: c\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.blockSequenceStart,
|
|
.blockEntry,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "a", style: .plain),
|
|
.value,
|
|
.key,
|
|
.scalar(value: "b", style: .plain),
|
|
.value,
|
|
.scalar(value: "c", style: .plain),
|
|
.blockEnd,
|
|
.blockEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a more-indented value nests a second mapping")
|
|
func test_moreIndentedValueNests() throws {
|
|
let tokens = try tokenize("a:\n b: c\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "a", style: .plain),
|
|
.value,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "b", style: .plain),
|
|
.value,
|
|
.scalar(value: "c", style: .plain),
|
|
.blockEnd,
|
|
.blockEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a simple block mapping pair")
|
|
func test_simplePair() throws {
|
|
let tokens = try tokenize("key: value\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "key", style: .plain),
|
|
.value,
|
|
.scalar(value: "value", style: .plain),
|
|
.blockEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a multi-line plain scalar folds its line break to a space")
|
|
func test_plainScalarFolding() throws {
|
|
let tokens = try tokenize("a\nb\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.scalar(value: "a b", style: .plain),
|
|
.streamEnd,
|
|
])
|
|
}
|
|
}
|