Mihaela Mihaljevic
ab05580518
Adds the %YAML and %TAG directive scanner, the final scanner piece. Ports fetch_directive, scan_directive, scan_directive_name, scan_version_directive_value/number, and scan_tag_directive_value, reusing the already-ported tag handle/URI scanners. Adds versionDirective and tagDirective token kinds. The libyaml scanner is now fully ported: stream, document markers, directives, block + flow structure, plain/quoted/block scalars, anchors, aliases, and tags. 319 tests, no regression.
265 lines
8.0 KiB
Swift
265 lines
8.0 KiB
Swift
@testable import PureYAML
|
|
import Testing
|
|
|
|
/// End-to-end tokenization through the ported fetch loop. The headline case is
|
|
/// the original block-mapping bug: `- a:` followed by a same-column `b:` must
|
|
/// tokenize as siblings (one BLOCK-MAPPING-START), matching libyaml, while a
|
|
/// more-indented value nests (two BLOCK-MAPPING-STARTs).
|
|
@Suite("LibYAML Scanner Fetch")
|
|
struct LibYAMLScannerFetchTests {
|
|
private typealias TokenType = PureYAML.Parsing.LibYAML.TokenType
|
|
|
|
private func tokenize(_ input: String) throws -> [TokenType] {
|
|
var scanner = PureYAML.Parsing.LibYAML.Scanner(input)
|
|
return try scanner.scanAll().map(\.type)
|
|
}
|
|
|
|
@Test("a block mapping inside a sequence entry keeps same-column keys siblings")
|
|
func test_sequenceEntryMappingSiblings() throws {
|
|
// The original bug: `- a:` / ` b: c` is one mapping {a: null, b: c}.
|
|
let tokens = try tokenize("- a:\n b: c\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.blockSequenceStart,
|
|
.blockEntry,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "a", style: .plain),
|
|
.value,
|
|
.key,
|
|
.scalar(value: "b", style: .plain),
|
|
.value,
|
|
.scalar(value: "c", style: .plain),
|
|
.blockEnd,
|
|
.blockEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a more-indented value nests a second mapping")
|
|
func test_moreIndentedValueNests() throws {
|
|
let tokens = try tokenize("a:\n b: c\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "a", style: .plain),
|
|
.value,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "b", style: .plain),
|
|
.value,
|
|
.scalar(value: "c", style: .plain),
|
|
.blockEnd,
|
|
.blockEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a simple block mapping pair")
|
|
func test_simplePair() throws {
|
|
let tokens = try tokenize("key: value\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "key", style: .plain),
|
|
.value,
|
|
.scalar(value: "value", style: .plain),
|
|
.blockEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a multi-line plain scalar folds its line break to a space")
|
|
func test_plainScalarFolding() throws {
|
|
let tokens = try tokenize("a\nb\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.scalar(value: "a b", style: .plain),
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a flow sequence of plain scalars")
|
|
func test_flowSequence() throws {
|
|
let tokens = try tokenize("[a, b]\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.flowSequenceStart,
|
|
.scalar(value: "a", style: .plain),
|
|
.flowEntry,
|
|
.scalar(value: "b", style: .plain),
|
|
.flowSequenceEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a flow mapping with a key and value")
|
|
func test_flowMapping() throws {
|
|
let tokens = try tokenize("{k: v}\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.flowMappingStart,
|
|
.key,
|
|
.scalar(value: "k", style: .plain),
|
|
.value,
|
|
.scalar(value: "v", style: .plain),
|
|
.flowMappingEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("single-quoted scalar with an escaped quote")
|
|
func test_singleQuoted() throws {
|
|
let tokens = try tokenize("'it''s here'\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.scalar(value: "it's here", style: .singleQuoted),
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("double-quoted scalar decodes escapes")
|
|
func test_doubleQuotedEscapes() throws {
|
|
let tokens = try tokenize("\"a\\nb\\tc\\u263A\"\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.scalar(value: "a\nb\tc\u{263A}", style: .doubleQuoted),
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a multi-line quoted scalar folds line breaks")
|
|
func test_quotedFolding() throws {
|
|
let tokens = try tokenize("\"a\nb\"\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.scalar(value: "a b", style: .doubleQuoted),
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("an anchored scalar value and an alias")
|
|
func test_anchorAndAlias() throws {
|
|
let tokens = try tokenize("a: &x 1\nb: *x\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "a", style: .plain),
|
|
.value,
|
|
.anchor(name: "x"),
|
|
.scalar(value: "1", style: .plain),
|
|
.key,
|
|
.scalar(value: "b", style: .plain),
|
|
.value,
|
|
.alias(name: "x"),
|
|
.blockEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a literal block scalar keeps line breaks and clips the trailing newline")
|
|
func test_literalBlockScalar() throws {
|
|
let tokens = try tokenize("key: |\n a\n b\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "key", style: .plain),
|
|
.value,
|
|
.scalar(value: "a\nb\n", style: .literal),
|
|
.blockEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a folded block scalar joins lines with a space")
|
|
func test_foldedBlockScalar() throws {
|
|
let tokens = try tokenize("key: >\n a\n b\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "key", style: .plain),
|
|
.value,
|
|
.scalar(value: "a b\n", style: .folded),
|
|
.blockEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a strip chomping indicator drops the trailing newline")
|
|
func test_literalStripChomping() throws {
|
|
let tokens = try tokenize("key: |-\n a\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.blockMappingStart,
|
|
.key,
|
|
.scalar(value: "key", style: .plain),
|
|
.value,
|
|
.scalar(value: "a", style: .literal),
|
|
.blockEnd,
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a secondary-handle shorthand tag")
|
|
func test_shorthandTag() throws {
|
|
let tokens = try tokenize("!!str x\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.tag(handle: "!!", suffix: "str"),
|
|
.scalar(value: "x", style: .plain),
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a local tag uses the primary handle")
|
|
func test_localTag() throws {
|
|
let tokens = try tokenize("!local x\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.tag(handle: "!", suffix: "local"),
|
|
.scalar(value: "x", style: .plain),
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a verbatim tag has an empty handle")
|
|
func test_verbatimTag() throws {
|
|
let tokens = try tokenize("!<tag:yaml.org,2002:str> x\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.tag(handle: "", suffix: "tag:yaml.org,2002:str"),
|
|
.scalar(value: "x", style: .plain),
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a YAML version directive")
|
|
func test_versionDirective() throws {
|
|
let tokens = try tokenize("%YAML 1.2\n---\nx\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.versionDirective(major: 1, minor: 2),
|
|
.documentStart,
|
|
.scalar(value: "x", style: .plain),
|
|
.streamEnd,
|
|
])
|
|
}
|
|
|
|
@Test("a TAG directive maps a handle to a prefix")
|
|
func test_tagDirective() throws {
|
|
let tokens = try tokenize("%TAG !e! tag:example.com,2000:\n---\nx\n")
|
|
#expect(tokens == [
|
|
.streamStart,
|
|
.tagDirective(handle: "!e!", prefix: "tag:example.com,2000:"),
|
|
.documentStart,
|
|
.scalar(value: "x", style: .plain),
|
|
.streamEnd,
|
|
])
|
|
}
|
|
}
|