Mihaela M
7879818ce5
The %YAML version check accepted only 1.1 and 1.2, rejecting anything else, so `%YAML 1.3` errored. The spec calls for processing a document whose minor version is higher than the processor's, with a warning, rather than refusing it: only a different major version is incompatible. Accept any well-formed `1.minor` version and reject only a different major (`2.0`) or a malformed version. Three tests that asserted `%YAML 1.3` was an error are corrected to use `2.0`, the genuinely incompatible case, and a test covers that 1.3 now parses. Raises yaml-test-suite parse-level conformance to 305/333 (91.6%) with the value-level (100%) and should-error rejection rates unchanged. 486 tests pass.
394 lines
12 KiB
Swift
394 lines
12 KiB
Swift
@testable import PureYAML
|
|
import Testing
|
|
|
|
@Suite("Scanner")
|
|
struct ScannerTests {
|
|
@Test("Scans comments indentation block entries and mapping keys")
|
|
func test_commentsIndentationBlockEntriesAndMappingKeys() throws {
|
|
let tokens = try scanKinds(
|
|
"""
|
|
# leading
|
|
root:
|
|
- name: "one" # inline
|
|
enabled: true
|
|
""",
|
|
)
|
|
|
|
expectKinds(tokens, [
|
|
"streamStart",
|
|
"comment value=\"leading\"",
|
|
"mappingKey",
|
|
"scalar value=\"root\" style=plain",
|
|
"mappingValue",
|
|
"indent width=2",
|
|
"blockEntry",
|
|
"mappingKey",
|
|
"scalar value=\"name\" style=plain",
|
|
"mappingValue",
|
|
"scalar value=\"one\" style=doubleQuoted",
|
|
"comment value=\"inline\"",
|
|
"indent width=4",
|
|
"mappingKey",
|
|
"scalar value=\"enabled\" style=plain",
|
|
"mappingValue",
|
|
"scalar value=\"true\" style=plain",
|
|
"dedent width=2",
|
|
"dedent width=0",
|
|
"streamEnd",
|
|
])
|
|
expectDoesNotContain(tokens, [
|
|
"flowSequenceStart",
|
|
"flowMappingStart",
|
|
"tag value=!Thing",
|
|
])
|
|
}
|
|
|
|
@Test("Scans explicit mapping keys and lower-indent dedents")
|
|
func test_explicitMappingKeysAndLowerIndentDedents() throws {
|
|
let tokens = try scanKinds(
|
|
"""
|
|
root:
|
|
nested:
|
|
value: yes
|
|
? explicit
|
|
: answer
|
|
""",
|
|
)
|
|
|
|
expectKinds(tokens, [
|
|
"streamStart",
|
|
"mappingKey",
|
|
"scalar value=\"root\" style=plain",
|
|
"mappingValue",
|
|
"indent width=2",
|
|
"mappingKey",
|
|
"scalar value=\"nested\" style=plain",
|
|
"mappingValue",
|
|
"indent width=4",
|
|
"mappingKey",
|
|
"scalar value=\"value\" style=plain",
|
|
"mappingValue",
|
|
"scalar value=\"yes\" style=plain",
|
|
"dedent width=2",
|
|
"dedent width=0",
|
|
"mappingKey",
|
|
"scalar value=\"explicit\" style=plain",
|
|
"mappingValue",
|
|
"scalar value=\"answer\" style=plain",
|
|
"streamEnd",
|
|
])
|
|
expectContains(tokens, [
|
|
"dedent width=2",
|
|
"dedent width=0",
|
|
"mappingKey",
|
|
"mappingValue",
|
|
])
|
|
}
|
|
|
|
@Test("Scans flow delimiters quoted scalars anchors aliases tags and block scalar headers")
|
|
func test_flowDelimitersQuotedScalarsAnchorsAliasesTagsAndBlockScalarHeaders() throws {
|
|
let tokens = try scanKinds(
|
|
"""
|
|
flow: [!Thing &item "value", *item, {tagged: !<tag:example.com,2026:thing> 'it''s'}]
|
|
text: |+
|
|
line
|
|
folded: >-
|
|
line
|
|
""",
|
|
)
|
|
|
|
expectFlowAndBlockScalarKinds(tokens)
|
|
expectFlowAndBlockScalarRequiredKinds(tokens)
|
|
expectFlowAndBlockScalarForbiddenKinds(tokens)
|
|
}
|
|
|
|
@Test("Plain scalars keep non-indicator hash and colon characters")
|
|
func test_plainScalarsKeepNonIndicatorHashAndColonCharacters() throws {
|
|
let tokens = try scanKinds(
|
|
"""
|
|
url: https://example.com/a:b
|
|
fragment: value#kept # removed
|
|
""",
|
|
)
|
|
|
|
expectKinds(tokens, [
|
|
"streamStart",
|
|
"mappingKey",
|
|
"scalar value=\"url\" style=plain",
|
|
"mappingValue",
|
|
"scalar value=\"https://example.com/a:b\" style=plain",
|
|
"mappingKey",
|
|
"scalar value=\"fragment\" style=plain",
|
|
"mappingValue",
|
|
"scalar value=\"value#kept\" style=plain",
|
|
"comment value=\"removed\"",
|
|
"streamEnd",
|
|
])
|
|
expectContains(tokens, [
|
|
"scalar value=\"https://example.com/a:b\" style=plain",
|
|
"scalar value=\"value#kept\" style=plain",
|
|
"comment value=\"removed\"",
|
|
])
|
|
expectDoesNotContain(tokens, [
|
|
"scalar value=\"https\" style=plain",
|
|
"comment value=\"kept # removed\"",
|
|
])
|
|
}
|
|
|
|
@Test("Scans directives and document markers without value tokens")
|
|
func test_directivesAndDocumentMarkers() throws {
|
|
let tokens = try scanKinds(
|
|
"""
|
|
%YAML 1.2
|
|
%TAG !yaml! tag:yaml.org,2002:
|
|
---
|
|
value: !yaml!str true
|
|
...
|
|
""",
|
|
)
|
|
|
|
expectKinds(tokens, [
|
|
"streamStart",
|
|
"documentStart",
|
|
"mappingKey",
|
|
"scalar value=\"value\" style=plain",
|
|
"mappingValue",
|
|
"tag value=tag:yaml.org,2002:str",
|
|
"scalar value=\"true\" style=plain",
|
|
"documentEnd",
|
|
"streamEnd",
|
|
])
|
|
expectDoesNotContain(tokens, [
|
|
"scalar value=\"%YAML 1.2\" style=plain",
|
|
"scalar value=\"---\" style=plain",
|
|
"scalar value=\"...\" style=plain",
|
|
])
|
|
}
|
|
|
|
@Test("Scans multiple document boundaries as tokens")
|
|
func test_multipleDocumentBoundaries() throws {
|
|
let tokens = try scanKinds(
|
|
"""
|
|
first: one
|
|
---
|
|
- two
|
|
...
|
|
# trailing
|
|
---
|
|
""",
|
|
)
|
|
|
|
expectKinds(tokens, [
|
|
"streamStart",
|
|
"mappingKey",
|
|
"scalar value=\"first\" style=plain",
|
|
"mappingValue",
|
|
"scalar value=\"one\" style=plain",
|
|
"documentStart",
|
|
"blockEntry",
|
|
"scalar value=\"two\" style=plain",
|
|
"documentEnd",
|
|
"comment value=\"trailing\"",
|
|
"documentStart",
|
|
"streamEnd",
|
|
])
|
|
expectContains(tokens, [
|
|
"documentStart",
|
|
"documentEnd",
|
|
"comment value=\"trailing\"",
|
|
])
|
|
expectDoesNotContain(tokens, [
|
|
"scalar value=\"---\" style=plain",
|
|
"scalar value=\"...\" style=plain",
|
|
])
|
|
}
|
|
|
|
@Test("Tracks UTF-8 source marks")
|
|
func test_utf8SourceMarks() throws {
|
|
let tokens = try PureYAML.Parsing.Scanner().scan("caf\u{00E9}: yes")
|
|
guard let key = tokens.first(where: {
|
|
$0.kind == .scalar(value: "caf\u{00E9}", style: .plain)
|
|
}) else {
|
|
recordIssue("expected cafe scalar")
|
|
return
|
|
}
|
|
guard let value = tokens.first(where: {
|
|
$0.kind == .scalar(value: "yes", style: .plain)
|
|
}) else {
|
|
recordIssue("expected yes scalar")
|
|
return
|
|
}
|
|
|
|
expectMarks(key, start: .init(line: 1, column: 1, index: 0), end: .init(line: 1, column: 5, index: 5))
|
|
expectMarks(value, start: .init(line: 1, column: 7, index: 7), end: .init(line: 1, column: 10, index: 10))
|
|
}
|
|
|
|
@Test("Tracks ASCII and extended UTF-8 marks in one scan")
|
|
func test_asciiAndExtendedUTF8Marks() throws {
|
|
let tokens = try PureYAML.Parsing.Scanner().scan("ascii: abc\nemoji: 😀\nnext: ok")
|
|
guard let emoji = tokens.first(where: {
|
|
$0.kind == .scalar(value: "😀", style: .plain)
|
|
}) else {
|
|
recordIssue("expected emoji scalar")
|
|
return
|
|
}
|
|
guard let next = tokens.first(where: {
|
|
$0.kind == .scalar(value: "next", style: .plain)
|
|
}) else {
|
|
recordIssue("expected next scalar")
|
|
return
|
|
}
|
|
|
|
expectMarks(emoji, start: .init(line: 2, column: 8, index: 18), end: .init(line: 2, column: 9, index: 22))
|
|
expectMarks(next, start: .init(line: 3, column: 1, index: 23), end: .init(line: 3, column: 5, index: 27))
|
|
}
|
|
|
|
@Test("Reader advances UTF-8 marks across line breaks")
|
|
func test_readerAdvancesUTF8MarksAcrossLineBreaks() {
|
|
var reader = PureYAML.Parsing.Reader("a\u{00E9}\n\u{03B2}\r\nz")
|
|
|
|
expectReaderMark(&reader, line: 1, column: 1, index: 0)
|
|
expectReaderMark(&reader, line: 1, column: 2, index: 1)
|
|
expectReaderMark(&reader, line: 1, column: 3, index: 3)
|
|
expectReaderMark(&reader, line: 2, column: 1, index: 4)
|
|
expectReaderMark(&reader, line: 2, column: 2, index: 6)
|
|
expectReaderMark(&reader, line: 3, column: 1, index: 8)
|
|
expectReaderMark(&reader, line: 3, column: 2, index: 9, advanceAfterCheck: false)
|
|
}
|
|
|
|
@Test("Reports scanner errors")
|
|
func test_errorReporting() {
|
|
expectScannerError("root:\n child: yes\n wrong: no", .unexpectedIndentation(line: 3))
|
|
expectScannerError("name: \"open", .unterminatedQuotedString(line: 1))
|
|
expectScannerError("name: 'open", .unterminatedQuotedString(line: 1))
|
|
expectScannerError("anchor: &", .expectedAnchorName(line: 1))
|
|
expectScannerError("alias: *", .expectedAliasName(line: 1))
|
|
expectScannerError("tag: !<tag:example.com,2026:thing", .unterminatedTag(line: 1))
|
|
expectScannerError("%YAML 2.0\n---\nvalue: yes", .incompatibleYAMLDirective(line: 1))
|
|
// An unrecognized directive before the document is ignored, but one after content (where a
|
|
// directive cannot appear) is still rejected.
|
|
expectScannerError("value: yes\n%FOO bar", .unsupportedDirective(name: "%FOO", line: 2))
|
|
expectScannerError("---\n%TAG !e! tag:example.com,2026:", .unsupportedDirective(name: "%TAG", line: 2))
|
|
}
|
|
}
|
|
|
|
private func scanKinds(_ yaml: String) throws -> [String] {
|
|
try PureYAML.Parsing.Scanner().scan(yaml).map(\.kind.description)
|
|
}
|
|
|
|
private func expectKinds(
|
|
_ actual: [String],
|
|
_ expected: [String],
|
|
) {
|
|
#expect(actual == expected)
|
|
}
|
|
|
|
private func expectContains(
|
|
_ actual: [String],
|
|
_ required: [String],
|
|
) {
|
|
for item in required {
|
|
#expect(actual.contains(item))
|
|
}
|
|
}
|
|
|
|
private func expectDoesNotContain(
|
|
_ actual: [String],
|
|
_ forbidden: [String],
|
|
) {
|
|
for item in forbidden {
|
|
#expect(!actual.contains(item))
|
|
}
|
|
}
|
|
|
|
private func expectMarks(
|
|
_ token: PureYAML.Parsing.Token,
|
|
start: PureYAML.Parsing.Mark,
|
|
end: PureYAML.Parsing.Mark,
|
|
) {
|
|
#expect(token.mark == start)
|
|
#expect(token.endMark == end)
|
|
}
|
|
|
|
private func expectFlowAndBlockScalarKinds(_ tokens: [String]) {
|
|
expectKinds(tokens, [
|
|
"streamStart",
|
|
"mappingKey",
|
|
"scalar value=\"flow\" style=plain",
|
|
"mappingValue",
|
|
"flowSequenceStart",
|
|
"tag value=!Thing",
|
|
"anchor name=item",
|
|
"scalar value=\"value\" style=doubleQuoted",
|
|
"flowEntry",
|
|
"alias name=item",
|
|
"flowEntry",
|
|
"flowMappingStart",
|
|
"mappingKey",
|
|
"scalar value=\"tagged\" style=plain",
|
|
"mappingValue",
|
|
"tag value=!<tag:example.com,2026:thing>",
|
|
"scalar value=\"it''s\" style=singleQuoted",
|
|
"flowMappingEnd",
|
|
"flowSequenceEnd",
|
|
"mappingKey",
|
|
"scalar value=\"text\" style=plain",
|
|
"mappingValue",
|
|
"blockScalarHeader style=literal chomping=keep",
|
|
"indent width=2",
|
|
"scalar value=\"line\" style=plain",
|
|
"dedent width=0",
|
|
"mappingKey",
|
|
"scalar value=\"folded\" style=plain",
|
|
"mappingValue",
|
|
"blockScalarHeader style=folded chomping=strip",
|
|
"indent width=2",
|
|
"scalar value=\"line\" style=plain",
|
|
"dedent width=0",
|
|
"streamEnd",
|
|
])
|
|
}
|
|
|
|
private func expectFlowAndBlockScalarRequiredKinds(_ tokens: [String]) {
|
|
expectContains(tokens, [
|
|
"flowSequenceStart",
|
|
"flowSequenceEnd",
|
|
"flowMappingStart",
|
|
"flowMappingEnd",
|
|
"tag value=!Thing",
|
|
"anchor name=item",
|
|
"alias name=item",
|
|
"blockScalarHeader style=literal chomping=keep",
|
|
"blockScalarHeader style=folded chomping=strip",
|
|
])
|
|
}
|
|
|
|
private func expectFlowAndBlockScalarForbiddenKinds(_ tokens: [String]) {
|
|
expectDoesNotContain(tokens, [
|
|
"comment value=\"removed\"",
|
|
"scalar value=\"tag:example.com,2026:thing\" style=plain",
|
|
])
|
|
}
|
|
|
|
private func expectReaderMark(
|
|
_ reader: inout PureYAML.Parsing.Reader,
|
|
line: Int,
|
|
column: Int,
|
|
index: Int,
|
|
advanceAfterCheck: Bool = true,
|
|
) {
|
|
#expect(reader.mark == PureYAML.Parsing.Mark(line: line, column: column, index: index))
|
|
if advanceAfterCheck {
|
|
reader.advance()
|
|
}
|
|
}
|
|
|
|
private func expectScannerError(
|
|
_ yaml: String,
|
|
_ expected: PureYAML.Parsing.ParseError,
|
|
) {
|
|
expectError(expected) {
|
|
try PureYAML.Parsing.Scanner().scan(yaml)
|
|
}
|
|
}
|