@testable import PureYAML import Testing /// End-to-end tokenization through the ported fetch loop. The headline case is /// the original block-mapping bug: `- a:` followed by a same-column `b:` must /// tokenize as siblings (one BLOCK-MAPPING-START), matching libyaml, while a /// more-indented value nests (two BLOCK-MAPPING-STARTs). @Suite("LibYAML Scanner Fetch") struct LibYAMLScannerFetchTests { private typealias TokenType = PureYAML.Parsing.LibYAML.TokenType private func tokenize(_ input: String) throws -> [TokenType] { var scanner = PureYAML.Parsing.LibYAML.Scanner(input) return try scanner.scanAll().map(\.type) } @Test("a block mapping inside a sequence entry keeps same-column keys siblings") func test_sequenceEntryMappingSiblings() throws { // The original bug: `- a:` / ` b: c` is one mapping {a: null, b: c}. let tokens = try tokenize("- a:\n b: c\n") #expect(tokens == [ .streamStart, .blockSequenceStart, .blockEntry, .blockMappingStart, .key, .scalar(value: "a", style: .plain), .value, .key, .scalar(value: "b", style: .plain), .value, .scalar(value: "c", style: .plain), .blockEnd, .blockEnd, .streamEnd, ]) } @Test("a more-indented value nests a second mapping") func test_moreIndentedValueNests() throws { let tokens = try tokenize("a:\n b: c\n") #expect(tokens == [ .streamStart, .blockMappingStart, .key, .scalar(value: "a", style: .plain), .value, .blockMappingStart, .key, .scalar(value: "b", style: .plain), .value, .scalar(value: "c", style: .plain), .blockEnd, .blockEnd, .streamEnd, ]) } @Test("a simple block mapping pair") func test_simplePair() throws { let tokens = try tokenize("key: value\n") #expect(tokens == [ .streamStart, .blockMappingStart, .key, .scalar(value: "key", style: .plain), .value, .scalar(value: "value", style: .plain), .blockEnd, .streamEnd, ]) } @Test("a multi-line plain scalar folds its line break to a space") func test_plainScalarFolding() throws { let tokens = try tokenize("a\nb\n") #expect(tokens == [ .streamStart, .scalar(value: "a b", style: .plain), .streamEnd, ]) } @Test("a flow sequence of plain scalars") func test_flowSequence() throws { let tokens = try tokenize("[a, b]\n") #expect(tokens == [ .streamStart, .flowSequenceStart, .scalar(value: "a", style: .plain), .flowEntry, .scalar(value: "b", style: .plain), .flowSequenceEnd, .streamEnd, ]) } @Test("a flow mapping with a key and value") func test_flowMapping() throws { let tokens = try tokenize("{k: v}\n") #expect(tokens == [ .streamStart, .flowMappingStart, .key, .scalar(value: "k", style: .plain), .value, .scalar(value: "v", style: .plain), .flowMappingEnd, .streamEnd, ]) } @Test("single-quoted scalar with an escaped quote") func test_singleQuoted() throws { let tokens = try tokenize("'it''s here'\n") #expect(tokens == [ .streamStart, .scalar(value: "it's here", style: .singleQuoted), .streamEnd, ]) } @Test("double-quoted scalar decodes escapes") func test_doubleQuotedEscapes() throws { let tokens = try tokenize("\"a\\nb\\tc\\u263A\"\n") #expect(tokens == [ .streamStart, .scalar(value: "a\nb\tc\u{263A}", style: .doubleQuoted), .streamEnd, ]) } @Test("a multi-line quoted scalar folds line breaks") func test_quotedFolding() throws { let tokens = try tokenize("\"a\nb\"\n") #expect(tokens == [ .streamStart, .scalar(value: "a b", style: .doubleQuoted), .streamEnd, ]) } @Test("an anchored scalar value and an alias") func test_anchorAndAlias() throws { let tokens = try tokenize("a: &x 1\nb: *x\n") #expect(tokens == [ .streamStart, .blockMappingStart, .key, .scalar(value: "a", style: .plain), .value, .anchor(name: "x"), .scalar(value: "1", style: .plain), .key, .scalar(value: "b", style: .plain), .value, .alias(name: "x"), .blockEnd, .streamEnd, ]) } @Test("a literal block scalar keeps line breaks and clips the trailing newline") func test_literalBlockScalar() throws { let tokens = try tokenize("key: |\n a\n b\n") #expect(tokens == [ .streamStart, .blockMappingStart, .key, .scalar(value: "key", style: .plain), .value, .scalar(value: "a\nb\n", style: .literal), .blockEnd, .streamEnd, ]) } @Test("a folded block scalar joins lines with a space") func test_foldedBlockScalar() throws { let tokens = try tokenize("key: >\n a\n b\n") #expect(tokens == [ .streamStart, .blockMappingStart, .key, .scalar(value: "key", style: .plain), .value, .scalar(value: "a b\n", style: .folded), .blockEnd, .streamEnd, ]) } @Test("a strip chomping indicator drops the trailing newline") func test_literalStripChomping() throws { let tokens = try tokenize("key: |-\n a\n") #expect(tokens == [ .streamStart, .blockMappingStart, .key, .scalar(value: "key", style: .plain), .value, .scalar(value: "a", style: .literal), .blockEnd, .streamEnd, ]) } @Test("a secondary-handle shorthand tag") func test_shorthandTag() throws { let tokens = try tokenize("!!str x\n") #expect(tokens == [ .streamStart, .tag(handle: "!!", suffix: "str"), .scalar(value: "x", style: .plain), .streamEnd, ]) } @Test("a local tag uses the primary handle") func test_localTag() throws { let tokens = try tokenize("!local x\n") #expect(tokens == [ .streamStart, .tag(handle: "!", suffix: "local"), .scalar(value: "x", style: .plain), .streamEnd, ]) } @Test("a verbatim tag has an empty handle") func test_verbatimTag() throws { let tokens = try tokenize("! x\n") #expect(tokens == [ .streamStart, .tag(handle: "", suffix: "tag:yaml.org,2002:str"), .scalar(value: "x", style: .plain), .streamEnd, ]) } @Test("a YAML version directive") func test_versionDirective() throws { let tokens = try tokenize("%YAML 1.2\n---\nx\n") #expect(tokens == [ .streamStart, .versionDirective(major: 1, minor: 2), .documentStart, .scalar(value: "x", style: .plain), .streamEnd, ]) } @Test("a TAG directive maps a handle to a prefix") func test_tagDirective() throws { let tokens = try tokenize("%TAG !e! tag:example.com,2000:\n---\nx\n") #expect(tokens == [ .streamStart, .tagDirective(handle: "!e!", prefix: "tag:example.com,2000:"), .documentStart, .scalar(value: "x", style: .plain), .streamEnd, ]) } }