diff --git a/.swiftformat b/.swiftformat index b27324c..15e5085 100644 --- a/.swiftformat +++ b/.swiftformat @@ -18,3 +18,6 @@ --wrapparameters before-first --wrapcollections before-first --self remove + +# PureYAML uses OpenAPIKit-style `test_...` method names with Swift Testing. +--disable swiftTestingTestCaseNames diff --git a/CHANGELOG.md b/CHANGELOG.md index 1708836..154e3da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add the internal parse event model with marks, scalar styles, collection styles, alias events, anchor/tag metadata, and golden event tests for current parser behavior. +- Add the internal UTF-8 reader and scanner token layer with tests for + comments, indentation, block entries, mapping indicators, flow delimiters, + quoted scalars, block scalar headers, anchors, aliases, tags, and source + marks. - Expand tests into focused parsing, dumping, model, validation, and parse-error suites. - Add macOS, Linux, and WASM verification gates and CI jobs, plus a local Linux diff --git a/README.md b/README.md index 9c410dd..48f18c4 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ flowchart TB classDef epic fill:#f2e5ff,stroke:#af52de,color:#111827 classDef todo fill:#f2f4f7,stroke:#8e8e93,color:#111827 Issue2["#2 Event Model and Golden Tests"]:::done - Issue3["#3 UTF-8 Reader and Scanner"]:::todo + Issue3["#3 UTF-8 Reader and Scanner"]:::done Issue4["#4 Token Stream to Events"]:::todo Issue5["#5 Events to PureYAML Values"]:::todo Issue6["#6 Scalars Tags and Aliases"]:::todo diff --git a/Sources/Parsing/BlockScalarStyle.swift b/Sources/Parsing/BlockScalarStyle.swift new file mode 100644 index 0000000..3addb3d --- /dev/null +++ b/Sources/Parsing/BlockScalarStyle.swift @@ -0,0 +1,10 @@ +extension PureYAML.Parsing { + enum BlockScalarStyle: String, Equatable, CustomStringConvertible { + case literal + case folded + + var description: String { + rawValue + } + } +} diff --git a/Sources/Parsing/ParseError.swift b/Sources/Parsing/ParseError.swift index 88c1d94..c36eb73 100644 --- a/Sources/Parsing/ParseError.swift +++ b/Sources/Parsing/ParseError.swift @@ -6,6 +6,9 @@ public extension PureYAML.Parsing { case unexpectedIndentation(line: Int) case mixedCollectionStyles(line: Int) case expectedMappingKey(line: Int) + case expectedAnchorName(line: Int) + case expectedAliasName(line: Int) + case unterminatedTag(line: Int) case unterminatedQuotedString(line: Int) public var description: String { @@ -20,6 +23,12 @@ public extension PureYAML.Parsing { "mapping and sequence entries are mixed at line \(line)" case let .expectedMappingKey(line): "expected a mapping key at line \(line)" + case let .expectedAnchorName(line): + "expected an anchor name at line \(line)" + case let .expectedAliasName(line): + "expected an alias name at line \(line)" + case let .unterminatedTag(line): + "unterminated tag at line \(line)" case let .unterminatedQuotedString(line): "unterminated quoted string at line \(line)" } diff --git a/Sources/Parsing/Reader.swift b/Sources/Parsing/Reader.swift new file mode 100644 index 0000000..25ad45d --- /dev/null +++ b/Sources/Parsing/Reader.swift @@ -0,0 +1,75 @@ +extension PureYAML.Parsing { + struct Reader { + private let source: String + private var position: String.Index + private var line: Int + private var column: Int + private var utf8Index: Int + + init(_ source: String) { + self.source = source + position = source.startIndex + line = 1 + column = 1 + utf8Index = 0 + } + + var isAtEnd: Bool { + position >= source.endIndex + } + + var mark: Mark { + Mark(line: line, column: column, index: utf8Index) + } + + func peek(offset: Int = 0) -> Character? { + var cursor = position + for _ in 0 ..< offset { + guard cursor < source.endIndex else { + return nil + } + cursor = source.index(after: cursor) + } + guard cursor < source.endIndex else { + return nil + } + return source[cursor] + } + + @discardableResult + mutating func advance() -> Character? { + guard position < source.endIndex else { + return nil + } + let character = source[position] + position = source.index(after: position) + utf8Index += String(character).utf8.count + if character == "\r\n" { + line += 1 + column = 1 + } else if character == "\r" { + if position < source.endIndex, source[position] == "\n" { + position = source.index(after: position) + utf8Index += 1 + } + line += 1 + column = 1 + } else if character == "\n" { + line += 1 + column = 1 + } else { + column += 1 + } + return character + } + + mutating func consume(while predicate: (Character) -> Bool) -> String { + var output = "" + while let character = peek(), predicate(character) { + output.append(character) + advance() + } + return output + } + } +} diff --git a/Sources/Parsing/Scanner.Helpers.swift b/Sources/Parsing/Scanner.Helpers.swift new file mode 100644 index 0000000..0fdd6e1 --- /dev/null +++ b/Sources/Parsing/Scanner.Helpers.swift @@ -0,0 +1,87 @@ +extension PureYAML.Parsing.Scanner { + func isLineBreak(_ character: Character?) -> Bool { + character == "\n" || character == "\r" || character == "\r\n" + } + + func isBlockEntry(_ reader: PureYAML.Parsing.Reader) -> Bool { + reader.peek() == "-" && isIndicatorBoundary(reader.peek(offset: 1)) + } + + func isIndicatorBoundary(_ character: Character?) -> Bool { + guard let character else { + return true + } + return character == " " || isLineBreak(character) + } + + func isMappingValueBoundary(_ character: Character?) -> Bool { + guard let character else { + return true + } + return character == " " + || isLineBreak(character) + || character == "," + || character == "]" + || character == "}" + } + + func isTokenTerminator(_ character: Character) -> Bool { + character == " " + || isLineBreak(character) + || character == "," + || character == "[" + || character == "]" + || character == "{" + || character == "}" + || character == "#" + } + + func isFlowDelimiter(_ character: Character) -> Bool { + character == "," + || character == "[" + || character == "]" + || character == "{" + || character == "}" + } + + func isBlockScalarHeaderCharacter(_ character: Character) -> Bool { + character == "+" + || character == "-" + || character.isNumber + } + + func trimTrailingSpaces(_ value: String) -> String { + var end = value.endIndex + while end > value.startIndex { + let previous = value.index(before: end) + guard value[previous] == " " else { + break + } + end = previous + } + return String(value[.. Bool { + if isLineBreak(character) { + return true + } + if character == " ", state.reader.peek(offset: 1) == "#" { + return true + } + if character == "#", accumulated.isEmpty || accumulated.last == " " { + return true + } + if character == ":", isMappingValueBoundary(state.reader.peek(offset: 1)) { + return true + } + if state.flowDepth > 0, isFlowDelimiter(character) { + return true + } + return false + } +} diff --git a/Sources/Parsing/Scanner.swift b/Sources/Parsing/Scanner.swift new file mode 100644 index 0000000..3b98cab --- /dev/null +++ b/Sources/Parsing/Scanner.swift @@ -0,0 +1,367 @@ +extension PureYAML.Parsing { + struct Scanner { + init() {} + + func scan(_ yaml: String) throws -> [Token] { + var state = State(reader: Reader(yaml)) + state.append(.streamStart) + + while !state.reader.isAtEnd { + if isLineBreak(state.reader.peek()) { + state.reader.advance() + state.isAtLineStart = true + continue + } + + if state.isAtLineStart { + try scanIndentation(&state) + if state.reader.isAtEnd { + break + } + if isLineBreak(state.reader.peek()) { + continue + } + state.isAtLineStart = false + } + + try scanToken(&state) + } + + closeIndentation(&state) + state.append(.streamEnd) + return state.tokens + } + } +} + +extension PureYAML.Parsing.Scanner { + struct State { + var reader: PureYAML.Parsing.Reader + var tokens: [PureYAML.Parsing.Token] = [] + var indentation: [Int] = [0] + var isAtLineStart = true + var flowDepth = 0 + + mutating func append(_ kind: PureYAML.Parsing.TokenKind) { + let mark = reader.mark + tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: mark)) + } + + mutating func append( + _ kind: PureYAML.Parsing.TokenKind, + mark: PureYAML.Parsing.Mark, + endMark: PureYAML.Parsing.Mark, + ) { + tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: endMark)) + } + } +} + +extension PureYAML.Parsing.Scanner { + func scanIndentation(_ state: inout State) throws { + let start = state.reader.mark + var width = 0 + while let character = state.reader.peek() { + if character == " " { + width += 1 + state.reader.advance() + } else if character == "\t" { + throw PureYAML.Parsing.ParseError.tabIndentation(line: state.reader.mark.line) + } else { + break + } + } + + guard !state.reader.isAtEnd, !isLineBreak(state.reader.peek()) else { + return + } + guard state.reader.peek() != "#" else { + return + } + guard state.flowDepth == 0 else { + return + } + + let end = state.reader.mark + let current = state.indentation.last ?? 0 + if width > current { + state.indentation.append(width) + state.append(.indent(width: width), mark: start, endMark: end) + } else if width < current { + while state.indentation.count > 1, width < (state.indentation.last ?? 0) { + state.indentation.removeLast() + state.append(.dedent(width: state.indentation.last ?? 0), mark: end, endMark: end) + } + } + } + + func closeIndentation(_ state: inout State) { + let mark = state.reader.mark + while state.indentation.count > 1 { + state.indentation.removeLast() + state.append(.dedent(width: state.indentation.last ?? 0), mark: mark, endMark: mark) + } + } + + func scanToken(_ state: inout State) throws { + skipSeparation(&state) + guard let character = state.reader.peek(), !isLineBreak(character) else { + return + } + + if try scanLineIndicatorToken(character, state: &state) { + return + } + if scanFlowToken(character, state: &state) { + return + } + if try scanQuotedOrBlockScalarToken(character, state: &state) { + return + } + if try scanNodePropertyToken(character, state: &state) { + return + } + scanPlainScalar(&state) + } + + func scanLineIndicatorToken( + _ character: Character, + state: inout State, + ) throws -> Bool { + switch character { + case "#": + scanComment(&state) + return true + case "-" where isBlockEntry(state.reader): + scanFixedToken(.blockEntry, state: &state) + return true + case "?" where isIndicatorBoundary(state.reader.peek(offset: 1)): + scanFixedToken(.mappingKey, state: &state) + return true + case ":" where isMappingValueBoundary(state.reader.peek(offset: 1)): + scanFixedToken(.mappingValue, state: &state) + return true + default: + return false + } + } + + func scanFlowToken( + _ character: Character, + state: inout State, + ) -> Bool { + switch character { + case "[": + state.flowDepth += 1 + scanFixedToken(.flowSequenceStart, state: &state) + return true + case "]": + state.flowDepth = max(0, state.flowDepth - 1) + scanFixedToken(.flowSequenceEnd, state: &state) + return true + case "{": + state.flowDepth += 1 + scanFixedToken(.flowMappingStart, state: &state) + return true + case "}": + state.flowDepth = max(0, state.flowDepth - 1) + scanFixedToken(.flowMappingEnd, state: &state) + return true + case ",": + scanFixedToken(.flowEntry, state: &state) + return true + default: + return false + } + } + + func scanNodePropertyToken( + _ character: Character, + state: inout State, + ) throws -> Bool { + switch character { + case "&": + try scanNamedToken( + kind: PureYAML.Parsing.TokenKind.anchor, + missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName, + state: &state, + ) + return true + case "*": + try scanNamedToken( + kind: PureYAML.Parsing.TokenKind.alias, + missingNameError: PureYAML.Parsing.ParseError.expectedAliasName, + state: &state, + ) + return true + case "!": + try scanTag(&state) + return true + default: + return false + } + } + + func scanQuotedOrBlockScalarToken( + _ character: Character, + state: inout State, + ) throws -> Bool { + switch character { + case "\"": + try scanQuotedScalar(.doubleQuoted, state: &state) + return true + case "'": + try scanQuotedScalar(.singleQuoted, state: &state) + return true + case "|": + scanBlockScalarHeader(.literal, state: &state) + return true + case ">": + scanBlockScalarHeader(.folded, state: &state) + return true + default: + return false + } + } + + func skipSeparation(_ state: inout State) { + while state.reader.peek() == " " { + state.reader.advance() + } + } + + func scanFixedToken( + _ kind: PureYAML.Parsing.TokenKind, + state: inout State, + ) { + let start = state.reader.mark + state.reader.advance() + state.append(kind, mark: start, endMark: state.reader.mark) + } + + func scanComment(_ state: inout State) { + let start = state.reader.mark + state.reader.advance() + if state.reader.peek() == " " { + state.reader.advance() + } + let value = state.reader.consume { character in + !isLineBreak(character) + } + state.append(.comment(value), mark: start, endMark: state.reader.mark) + } + + func scanQuotedScalar( + _ style: PureYAML.Parsing.ScalarStyle, + state: inout State, + ) throws { + let start = state.reader.mark + let quote: Character = style == .doubleQuoted ? "\"" : "'" + state.reader.advance() + var value = "" + var previousWasEscape = false + while let character = state.reader.peek() { + if style == .singleQuoted, character == quote, state.reader.peek(offset: 1) == quote { + value.append(character) + state.reader.advance() + value.append(character) + state.reader.advance() + continue + } + if character == quote, !previousWasEscape { + break + } + value.append(character) + previousWasEscape = character == "\\" && style == .doubleQuoted && !previousWasEscape + state.reader.advance() + if character != "\\" { + previousWasEscape = false + } + } + guard state.reader.peek() == quote else { + throw PureYAML.Parsing.ParseError.unterminatedQuotedString(line: start.line) + } + state.reader.advance() + appendMappingKeyIfNeeded(start: start, state: &state) + state.append(.scalar(value: value, style: style), mark: start, endMark: state.reader.mark) + } + + func scanBlockScalarHeader( + _ style: PureYAML.Parsing.BlockScalarStyle, + state: inout State, + ) { + let start = state.reader.mark + state.reader.advance() + while let character = state.reader.peek(), isBlockScalarHeaderCharacter(character) { + state.reader.advance() + } + state.append(.blockScalarHeader(style: style), mark: start, endMark: state.reader.mark) + } + + func scanNamedToken( + kind: (String) -> PureYAML.Parsing.TokenKind, + missingNameError: (Int) -> PureYAML.Parsing.ParseError, + state: inout State, + ) throws { + let start = state.reader.mark + state.reader.advance() + let name = state.reader.consume { character in + !isTokenTerminator(character) + } + guard !name.isEmpty else { + throw missingNameError(start.line) + } + state.append(kind(name), mark: start, endMark: state.reader.mark) + } + + func scanTag(_ state: inout State) throws { + let start = state.reader.mark + state.reader.advance() + var value = "!" + if state.reader.peek() == "<" { + value.append(state.reader.advance() ?? "<") + var didClose = false + while let character = state.reader.peek() { + value.append(character) + state.reader.advance() + if character == ">" { + didClose = true + break + } + } + guard didClose else { + throw PureYAML.Parsing.ParseError.unterminatedTag(line: start.line) + } + } else { + value += state.reader.consume { character in + !isTokenTerminator(character) + } + } + state.append(.tag(value), mark: start, endMark: state.reader.mark) + } + + func scanPlainScalar(_ state: inout State) { + let start = state.reader.mark + var value = "" + while let character = state.reader.peek() { + if shouldEndPlainScalar(character, state: state, accumulated: value) { + break + } + value.append(character) + state.reader.advance() + } + let trimmed = trimTrailingSpaces(value) + appendMappingKeyIfNeeded(start: start, state: &state) + state.append(.scalar(value: trimmed, style: .plain), mark: start, endMark: state.reader.mark) + } + + func appendMappingKeyIfNeeded( + start: PureYAML.Parsing.Mark, + state: inout State, + ) { + guard state.reader.peek() == ":", isMappingValueBoundary(state.reader.peek(offset: 1)) else { + return + } + state.append(.mappingKey, mark: start, endMark: start) + } +} diff --git a/Sources/Parsing/Token.swift b/Sources/Parsing/Token.swift new file mode 100644 index 0000000..7e34648 --- /dev/null +++ b/Sources/Parsing/Token.swift @@ -0,0 +1,98 @@ +extension PureYAML.Parsing { + struct Token: Equatable, CustomStringConvertible { + var kind: TokenKind + var mark: Mark + var endMark: Mark + + var description: String { + "\(kind) @\(mark)..\(endMark)" + } + } +} + +extension PureYAML.Parsing { + enum TokenKind: Equatable { + case streamStart + case streamEnd + case indent(width: Int) + case dedent(width: Int) + case blockEntry + case mappingKey + case mappingValue + case flowSequenceStart + case flowSequenceEnd + case flowMappingStart + case flowMappingEnd + case flowEntry + case scalar(value: String, style: ScalarStyle) + case blockScalarHeader(style: BlockScalarStyle) + case anchor(String) + case alias(String) + case tag(String) + case comment(String) + } +} + +extension PureYAML.Parsing.TokenKind: CustomStringConvertible { + var description: String { + switch self { + case .streamStart: + "streamStart" + case .streamEnd: + "streamEnd" + case let .indent(width): + "indent width=\(width)" + case let .dedent(width): + "dedent width=\(width)" + case .blockEntry: + "blockEntry" + case .mappingKey: + "mappingKey" + case .mappingValue: + "mappingValue" + case .flowSequenceStart: + "flowSequenceStart" + case .flowSequenceEnd: + "flowSequenceEnd" + case .flowMappingStart: + "flowMappingStart" + case .flowMappingEnd: + "flowMappingEnd" + case .flowEntry: + "flowEntry" + case let .scalar(value, style): + "scalar value=\"\(escape(value))\" style=\(style)" + case let .blockScalarHeader(style): + "blockScalarHeader style=\(style)" + case let .anchor(value): + "anchor name=\(value)" + case let .alias(value): + "alias name=\(value)" + case let .tag(value): + "tag value=\(value)" + case let .comment(value): + "comment value=\"\(escape(value))\"" + } + } + + private func escape(_ value: String) -> String { + var output = "" + for character in value { + switch character { + case "\\": + output += "\\\\" + case "\"": + output += "\\\"" + case "\n": + output += "\\n" + case "\r": + output += "\\r" + case "\t": + output += "\\t" + default: + output.append(character) + } + } + return output + } +} diff --git a/Tests/DumperTests.swift b/Tests/DumperTests.swift index 5a29004..65e0103 100644 --- a/Tests/DumperTests.swift +++ b/Tests/DumperTests.swift @@ -4,7 +4,7 @@ import Testing @Suite("Dumper") struct DumperTests { @Test("Dumps scalar mapping values") - func dumpsScalarMappingValues() { + func test_scalarMappingValues() { let value = PureYAML.Model.Value.mapping(.init([ .init(key: "missing", value: .null), .init(key: "enabled", value: .bool(true)), @@ -24,7 +24,7 @@ struct DumperTests { } @Test("Escapes scalar strings and mapping keys") - func escapesScalarStringsAndMappingKeys() { + func test_escapesScalarStringsAndMappingKeys() { let value = PureYAML.Model.Value.mapping(.init([ .init(key: "a:b", value: .string("quote \" slash \\ tab\t")), .init(key: "hash#key", value: .string("line\nnext")), @@ -35,10 +35,12 @@ struct DumperTests { "hash#key": "line\\nnext" """) + #expect(!PureYAML.dump(value).contains("quote \" slash \\ tab\t")) + #expect(!PureYAML.dump(value).contains("line\nnext")) } @Test("Dumps nested sequences and mappings") - func dumpsNestedSequencesAndMappings() { + func test_nestedSequencesAndMappings() { let value = PureYAML.Model.Value.mapping(.init([ .init(key: "servers", value: .sequence([ .mapping(.init([ @@ -58,7 +60,7 @@ struct DumperTests { } @Test("Dumps top-level sequences") - func dumpsTopLevelSequences() { + func test_topLevelSequences() { let value = PureYAML.Model.Value.sequence([ .string("one"), .int(2), @@ -74,7 +76,7 @@ struct DumperTests { } @Test("Dumps round-trippable YAML") - func dumpsRoundTrippableYaml() throws { + func test_roundTrippableYaml() throws { let original = PureYAML.Model.Value.mapping(.init([ .init(key: "paths", value: .mapping(.init([ .init(key: "/users", value: .mapping(.init([ @@ -88,4 +90,10 @@ struct DumperTests { #expect(try PureYAML.parse(PureYAML.dump(original)) == original) } + + @Test("Dumps empty collections") + func test_emptyCollections() { + #expect(PureYAML.dump(.mapping(.init())) == "\n") + #expect(PureYAML.dump(.sequence([])) == "\n") + } } diff --git a/Tests/ModelTests.swift b/Tests/ModelTests.swift index e480506..286e18d 100644 --- a/Tests/ModelTests.swift +++ b/Tests/ModelTests.swift @@ -4,7 +4,7 @@ import Testing @Suite("Model") struct ModelTests { @Test("Mapping preserves insertion order") - func mappingPreservesInsertionOrder() { + func test_mappingPreservesInsertionOrder() { let mapping = PureYAML.Model.Mapping([ .init(key: "b", value: .int(2)), .init(key: "a", value: .int(1)), @@ -14,7 +14,7 @@ struct ModelTests { } @Test("Mapping subscript returns first matching key") - func mappingSubscriptReturnsFirstMatchingKey() { + func test_mappingSubscriptReturnsFirstMatchingKey() { let mapping = PureYAML.Model.Mapping([ .init(key: "name", value: .string("first")), .init(key: "name", value: .string("second")), @@ -22,4 +22,13 @@ struct ModelTests { #expect(mapping["name"] == .string("first")) } + + @Test("Mapping subscript returns nil for missing key") + func test_mappingSubscriptReturnsNilForMissingKey() { + let mapping = PureYAML.Model.Mapping([ + .init(key: "name", value: .string("first")), + ]) + + #expect(mapping["missing"] == nil) + } } diff --git a/Tests/ParsingEventTests.swift b/Tests/ParsingEventTests.swift index 9849402..83d4427 100644 --- a/Tests/ParsingEventTests.swift +++ b/Tests/ParsingEventTests.swift @@ -4,7 +4,7 @@ import Testing @Suite("Parsing Events") struct ParsingEventTests { @Test("Event model represents aliases anchors tags and flow style") - func eventModelRepresentsAliasesAnchorsTagsAndFlowStyle() { + func test_eventModelRepresentsAliasesAnchorsTagsAndFlowStyle() { let mark = PureYAML.Parsing.Mark(line: 2, column: 3, index: 12) let events: [PureYAML.Parsing.Event] = [ .scalar(value: "value", anchor: "root", tag: "!Thing", style: .singleQuoted, mark: mark), @@ -20,7 +20,7 @@ struct ParsingEventTests { } @Test("Emits golden events for mapping scalars and scalar styles") - func emitsGoldenEventsForMappingScalarsAndScalarStyles() throws { + func test_mappingScalarsAndScalarStyles() throws { let events = try PureYAML.Parsing.Parser().parseEvents( """ name: "Example" @@ -46,7 +46,7 @@ struct ParsingEventTests { } @Test("Emits golden events for nested sequences and mappings") - func emitsGoldenEventsForNestedSequencesAndMappings() throws { + func test_nestedSequencesAndMappings() throws { let events = try PureYAML.Parsing.Parser().parseEvents( """ servers: @@ -72,5 +72,19 @@ struct ParsingEventTests { "documentEnd @3:25@44", "streamEnd @3:25@44", ]) + #expect(!events.map(\.description).contains("scalar value=\"\" anchor=- tag=- style=plain @1:1@0")) + } + + @Test("Reports parse-event errors") + func test_errorReporting() { + expectError(PureYAML.Parsing.ParseError.emptyDocument) { + try PureYAML.Parsing.Parser().parseEvents("") + } + expectError(PureYAML.Parsing.ParseError.unexpectedIndentation(line: 3)) { + try PureYAML.Parsing.Parser().parseEvents("root:\n child: yes\n wrong: no") + } + expectError(PureYAML.Parsing.ParseError.expectedMappingKey(line: 3)) { + try PureYAML.Parsing.Parser().parseEvents("root:\n child: value\n dangling") + } } } diff --git a/Tests/ParsingTests.swift b/Tests/ParsingTests.swift index a57a196..ef49802 100644 --- a/Tests/ParsingTests.swift +++ b/Tests/ParsingTests.swift @@ -9,7 +9,7 @@ struct ParsingTests { } @Test("Parses block mappings with common scalars") - func parsesBlockMappingWithCommonScalars() throws { + func test_blockMappingWithCommonScalars() throws { let root = try requireMapping(PureYAML.parse( """ openapi: 3.1.0 @@ -27,6 +27,7 @@ struct ParsingTests { #expect(root?["retries"] == .int(3)) #expect(root?["ratio"] == .double(3.1)) #expect(root?["missing"] == .null) + #expect(root?["unknown"] == nil) } @Test("Parses scalar spellings", arguments: [ @@ -39,14 +40,14 @@ struct ParsingTests { ScalarCase(yaml: "value: 1e3", expected: .double(1000)), ScalarCase(yaml: "value: plain text", expected: .string("plain text")), ]) - func parsesScalarSpellings(testCase: ScalarCase) throws { + func test_scalarSpellings(testCase: ScalarCase) throws { let root = try requireMapping(PureYAML.parse(testCase.yaml)) #expect(root?["value"] == testCase.expected) } @Test("Parses quoted strings and escapes") - func parsesQuotedStringsAndEscapes() throws { + func test_quotedStringsAndEscapes() throws { let root = try requireMapping(PureYAML.parse( """ double: "line\\nnext\\tend" @@ -61,7 +62,7 @@ struct ParsingTests { } @Test("Ignores comments outside quoted strings") - func ignoresCommentsOutsideQuotedStrings() throws { + func test_commentsOutsideQuotedStrings() throws { let root = try requireMapping(PureYAML.parse( """ title: "Keep # inside" # remove outside @@ -73,10 +74,11 @@ struct ParsingTests { #expect(root?["title"] == .string("Keep # inside")) #expect(root?["single"] == .string("Keep # inside")) #expect(root?["plain"] == .string("value#kept")) + #expect(root?["plain"] != .string("value")) } @Test("Parses nested mappings and sequences") - func parsesNestedMappingsAndSequences() throws { + func test_nestedMappingsAndSequences() throws { let root = try requireMapping(PureYAML.parse( """ paths: @@ -101,7 +103,7 @@ struct ParsingTests { } @Test("Parses sequences of mappings") - func parsesSequenceOfMappings() throws { + func test_sequenceOfMappings() throws { let root = try requireMapping(PureYAML.parse( """ servers: @@ -129,7 +131,7 @@ struct ParsingTests { } @Test("Parses top-level sequences") - func parsesTopLevelSequences() throws { + func test_topLevelSequences() throws { let value = try PureYAML.parse( """ - one @@ -146,7 +148,7 @@ struct ParsingTests { } @Test("Reports exact parser errors") - func reportsExactParserErrors() { + func test_errorReporting() { expectParseError("", .emptyDocument) expectParseError("# comment only", .emptyDocument) expectParseError("\tkey: value", .tabIndentation(line: 1)) @@ -156,4 +158,17 @@ struct ParsingTests { expectParseError("name: \"open", .unterminatedQuotedString(line: 1)) expectParseError("name: 'open", .unterminatedQuotedString(line: 1)) } + + @Test("Parser errors describe returned failures") + func test_errorDescriptions() { + #expect(PureYAML.Parsing.ParseError.emptyDocument.description == "document is empty") + #expect(PureYAML.Parsing.ParseError.tabIndentation(line: 2).description == "tabs are not allowed for indentation at line 2") + #expect(PureYAML.Parsing.ParseError.unexpectedIndentation(line: 3).description == "unexpected indentation at line 3") + #expect(PureYAML.Parsing.ParseError.mixedCollectionStyles(line: 4).description == "mapping and sequence entries are mixed at line 4") + #expect(PureYAML.Parsing.ParseError.expectedMappingKey(line: 5).description == "expected a mapping key at line 5") + #expect(PureYAML.Parsing.ParseError.expectedAnchorName(line: 6).description == "expected an anchor name at line 6") + #expect(PureYAML.Parsing.ParseError.expectedAliasName(line: 7).description == "expected an alias name at line 7") + #expect(PureYAML.Parsing.ParseError.unterminatedTag(line: 8).description == "unterminated tag at line 8") + #expect(PureYAML.Parsing.ParseError.unterminatedQuotedString(line: 9).description == "unterminated quoted string at line 9") + } } diff --git a/Tests/ScannerTests.swift b/Tests/ScannerTests.swift new file mode 100644 index 0000000..cba4648 --- /dev/null +++ b/Tests/ScannerTests.swift @@ -0,0 +1,300 @@ +@testable import PureYAML +import Testing + +@Suite("Scanner") +struct ScannerTests { + @Test("Scans comments indentation block entries and mapping keys") + func test_commentsIndentationBlockEntriesAndMappingKeys() throws { + let tokens = try scanKinds( + """ + # leading + root: + - name: "one" # inline + enabled: true + """, + ) + + expectKinds(tokens, [ + "streamStart", + "comment value=\"leading\"", + "mappingKey", + "scalar value=\"root\" style=plain", + "mappingValue", + "indent width=2", + "blockEntry", + "mappingKey", + "scalar value=\"name\" style=plain", + "mappingValue", + "scalar value=\"one\" style=doubleQuoted", + "comment value=\"inline\"", + "indent width=4", + "mappingKey", + "scalar value=\"enabled\" style=plain", + "mappingValue", + "scalar value=\"true\" style=plain", + "dedent width=2", + "dedent width=0", + "streamEnd", + ]) + expectDoesNotContain(tokens, [ + "flowSequenceStart", + "flowMappingStart", + "tag value=!Thing", + ]) + } + + @Test("Scans explicit mapping keys and lower-indent dedents") + func test_explicitMappingKeysAndLowerIndentDedents() throws { + let tokens = try scanKinds( + """ + root: + nested: + value: yes + ? explicit + : answer + """, + ) + + expectKinds(tokens, [ + "streamStart", + "mappingKey", + "scalar value=\"root\" style=plain", + "mappingValue", + "indent width=2", + "mappingKey", + "scalar value=\"nested\" style=plain", + "mappingValue", + "indent width=4", + "mappingKey", + "scalar value=\"value\" style=plain", + "mappingValue", + "scalar value=\"yes\" style=plain", + "dedent width=2", + "dedent width=0", + "mappingKey", + "scalar value=\"explicit\" style=plain", + "mappingValue", + "scalar value=\"answer\" style=plain", + "streamEnd", + ]) + expectContains(tokens, [ + "dedent width=2", + "dedent width=0", + "mappingKey", + "mappingValue", + ]) + } + + @Test("Scans flow delimiters quoted scalars anchors aliases tags and block scalar headers") + func test_flowDelimitersQuotedScalarsAnchorsAliasesTagsAndBlockScalarHeaders() throws { + let tokens = try scanKinds( + """ + flow: [!Thing &item "value", *item, {tagged: ! 'it''s'}] + text: |+ + line + folded: >- + line + """, + ) + + expectFlowAndBlockScalarKinds(tokens) + expectFlowAndBlockScalarRequiredKinds(tokens) + expectFlowAndBlockScalarForbiddenKinds(tokens) + } + + @Test("Plain scalars keep non-indicator hash and colon characters") + func test_plainScalarsKeepNonIndicatorHashAndColonCharacters() throws { + let tokens = try scanKinds( + """ + url: https://example.com/a:b + fragment: value#kept # removed + """, + ) + + expectKinds(tokens, [ + "streamStart", + "mappingKey", + "scalar value=\"url\" style=plain", + "mappingValue", + "scalar value=\"https://example.com/a:b\" style=plain", + "mappingKey", + "scalar value=\"fragment\" style=plain", + "mappingValue", + "scalar value=\"value#kept\" style=plain", + "comment value=\"removed\"", + "streamEnd", + ]) + expectContains(tokens, [ + "scalar value=\"https://example.com/a:b\" style=plain", + "scalar value=\"value#kept\" style=plain", + "comment value=\"removed\"", + ]) + expectDoesNotContain(tokens, [ + "scalar value=\"https\" style=plain", + "comment value=\"kept # removed\"", + ]) + } + + @Test("Tracks UTF-8 source marks") + func test_utf8SourceMarks() throws { + let tokens = try PureYAML.Parsing.Scanner().scan("caf\u{00E9}: yes") + guard let key = tokens.first(where: { + $0.kind == .scalar(value: "caf\u{00E9}", style: .plain) + }) else { + recordIssue("expected cafe scalar") + return + } + guard let value = tokens.first(where: { + $0.kind == .scalar(value: "yes", style: .plain) + }) else { + recordIssue("expected yes scalar") + return + } + + expectMarks(key, start: .init(line: 1, column: 1, index: 0), end: .init(line: 1, column: 5, index: 5)) + expectMarks(value, start: .init(line: 1, column: 7, index: 7), end: .init(line: 1, column: 10, index: 10)) + } + + @Test("Reader advances UTF-8 marks across line breaks") + func test_readerAdvancesUTF8MarksAcrossLineBreaks() { + var reader = PureYAML.Parsing.Reader("a\u{00E9}\n\u{03B2}\r\nz") + + expectReaderMark(&reader, line: 1, column: 1, index: 0) + expectReaderMark(&reader, line: 1, column: 2, index: 1) + expectReaderMark(&reader, line: 1, column: 3, index: 3) + expectReaderMark(&reader, line: 2, column: 1, index: 4) + expectReaderMark(&reader, line: 2, column: 2, index: 6) + expectReaderMark(&reader, line: 3, column: 1, index: 8) + expectReaderMark(&reader, line: 3, column: 2, index: 9, advanceAfterCheck: false) + } + + @Test("Reports scanner errors") + func test_errorReporting() { + expectScannerError("\tname: value", .tabIndentation(line: 1)) + expectScannerError("name: \"open", .unterminatedQuotedString(line: 1)) + expectScannerError("name: 'open", .unterminatedQuotedString(line: 1)) + expectScannerError("anchor: &", .expectedAnchorName(line: 1)) + expectScannerError("alias: *", .expectedAliasName(line: 1)) + expectScannerError("tag: ! [String] { + try PureYAML.Parsing.Scanner().scan(yaml).map(\.kind.description) +} + +private func expectKinds( + _ actual: [String], + _ expected: [String], +) { + #expect(actual == expected) +} + +private func expectContains( + _ actual: [String], + _ required: [String], +) { + for item in required { + #expect(actual.contains(item)) + } +} + +private func expectDoesNotContain( + _ actual: [String], + _ forbidden: [String], +) { + for item in forbidden { + #expect(!actual.contains(item)) + } +} + +private func expectMarks( + _ token: PureYAML.Parsing.Token, + start: PureYAML.Parsing.Mark, + end: PureYAML.Parsing.Mark, +) { + #expect(token.mark == start) + #expect(token.endMark == end) +} + +private func expectFlowAndBlockScalarKinds(_ tokens: [String]) { + expectKinds(tokens, [ + "streamStart", + "mappingKey", + "scalar value=\"flow\" style=plain", + "mappingValue", + "flowSequenceStart", + "tag value=!Thing", + "anchor name=item", + "scalar value=\"value\" style=doubleQuoted", + "flowEntry", + "alias name=item", + "flowEntry", + "flowMappingStart", + "mappingKey", + "scalar value=\"tagged\" style=plain", + "mappingValue", + "tag value=!", + "scalar value=\"it''s\" style=singleQuoted", + "flowMappingEnd", + "flowSequenceEnd", + "mappingKey", + "scalar value=\"text\" style=plain", + "mappingValue", + "blockScalarHeader style=literal", + "indent width=2", + "scalar value=\"line\" style=plain", + "dedent width=0", + "mappingKey", + "scalar value=\"folded\" style=plain", + "mappingValue", + "blockScalarHeader style=folded", + "indent width=2", + "scalar value=\"line\" style=plain", + "dedent width=0", + "streamEnd", + ]) +} + +private func expectFlowAndBlockScalarRequiredKinds(_ tokens: [String]) { + expectContains(tokens, [ + "flowSequenceStart", + "flowSequenceEnd", + "flowMappingStart", + "flowMappingEnd", + "tag value=!Thing", + "anchor name=item", + "alias name=item", + "blockScalarHeader style=literal", + "blockScalarHeader style=folded", + ]) +} + +private func expectFlowAndBlockScalarForbiddenKinds(_ tokens: [String]) { + expectDoesNotContain(tokens, [ + "comment value=\"removed\"", + "scalar value=\"tag:example.com,2026:thing\" style=plain", + ]) +} + +private func expectReaderMark( + _ reader: inout PureYAML.Parsing.Reader, + line: Int, + column: Int, + index: Int, + advanceAfterCheck: Bool = true, +) { + #expect(reader.mark == PureYAML.Parsing.Mark(line: line, column: column, index: index)) + if advanceAfterCheck { + reader.advance() + } +} + +private func expectScannerError( + _ yaml: String, + _ expected: PureYAML.Parsing.ParseError, +) { + expectError(expected) { + try PureYAML.Parsing.Scanner().scan(yaml) + } +} diff --git a/Tests/TestSupport.swift b/Tests/TestSupport.swift index 7e23d5e..c56e92f 100644 --- a/Tests/TestSupport.swift +++ b/Tests/TestSupport.swift @@ -35,10 +35,19 @@ func expectParseError( _ yaml: String, _ expected: PureYAML.Parsing.ParseError, ) { - do { + expectError(expected) { _ = try PureYAML.parse(yaml) - recordIssue("expected parse error \(expected)") - } catch let error as PureYAML.Parsing.ParseError { + } +} + +func expectError( + _ expected: ExpectedError, + operation: () throws -> some Any, +) { + do { + _ = try operation() + recordIssue("expected error \(expected)") + } catch let error as ExpectedError { #expect(error == expected) } catch { recordIssue("unexpected error \(error)") diff --git a/Tests/ValidationTests.swift b/Tests/ValidationTests.swift index aee3ba5..2a8fb65 100644 --- a/Tests/ValidationTests.swift +++ b/Tests/ValidationTests.swift @@ -4,7 +4,7 @@ import Testing @Suite("Validation") struct ValidationTests { @Test("Validates documents with default rules") - func validatesDocumentsWithDefaultRules() throws { + func test_documentsWithDefaultRules() throws { let value = try PureYAML.parse( """ title: Example @@ -16,16 +16,13 @@ struct ValidationTests { } @Test("Reports duplicate mapping keys") - func reportsDuplicateMappingKeys() throws { + func test_duplicateMappingKeyError() { let value = PureYAML.Model.Value.mapping(.init([ .init(key: "title", value: .string("First")), .init(key: "title", value: .string("Second")), ])) - do { - try PureYAML.validate(value) - recordIssue("expected duplicate key validation failure") - } catch let collection as PureYAML.Validation.Issue.Collection { + expectValidationError(value) { collection in #expect(collection.issues == [ PureYAML.Validation.Issue( severity: .error, @@ -33,13 +30,11 @@ struct ValidationTests { path: .init([.key("title")]), ), ]) - } catch { - recordIssue("unexpected error \(error)") } } @Test("Reports nested validation paths") - func reportsNestedValidationPaths() throws { + func test_nestedValidationPaths() throws { let value = try PureYAML.parse( """ routes: @@ -54,7 +49,7 @@ struct ValidationTests { } @Test("Blank validator disables default rules") - func blankValidatorDisablesDefaultRules() throws { + func test_blankValidatorDisablesDefaultRules() throws { let value = PureYAML.Model.Value.mapping(.init([ .init(key: "title", value: .string("First")), .init(key: "title", value: .string("Second")), @@ -64,7 +59,7 @@ struct ValidationTests { } @Test("Custom validation rules can be added") - func customValidationRulesCanBeAdded() throws { + func test_customValidationRuleError() throws { let value = try PureYAML.parse( """ mode: legacy @@ -72,34 +67,24 @@ struct ValidationTests { ) let validator = PureYAML.Validation.Validator.blank.validating(legacyModeRule(severity: .error)) - do { - try PureYAML.validate(value, using: validator) - recordIssue("expected custom validation failure") - } catch let collection as PureYAML.Validation.Issue.Collection { + expectValidationError(value, using: validator) { collection in #expect(collection.issues.map(\.path.description) == ["$.mode"]) #expect(collection.issues.map(\.reason) == ["Legacy mode is not allowed"]) - } catch { - recordIssue("unexpected error \(error)") } } @Test("Strict validation treats warnings as failures") - func strictValidationTreatsWarningsAsFailures() throws { + func test_strictValidationTreatsWarningsAsFailures() throws { let value = try PureYAML.parse("mode: legacy") let validator = PureYAML.Validation.Validator.blank.validating(legacyModeRule(severity: .warning)) - do { - try PureYAML.validate(value, using: validator) - recordIssue("expected warning to fail strict validation") - } catch let collection as PureYAML.Validation.Issue.Collection { + expectValidationError(value, using: validator) { collection in #expect(collection.issues.map(\.severity) == [.warning]) - } catch { - recordIssue("unexpected error \(error)") } } @Test("Non-strict validation returns warnings") - func nonStrictValidationReturnsWarnings() throws { + func test_nonStrictValidationReturnsWarnings() throws { let value = try PureYAML.parse("mode: legacy") let validator = PureYAML.Validation.Validator.blank.validating(legacyModeRule(severity: .warning)) @@ -115,7 +100,7 @@ struct ValidationTests { } @Test("Validation result separates errors and warnings") - func validationResultSeparatesErrorsAndWarnings() { + func test_validationResultSeparatesErrorsAndWarnings() { let result = PureYAML.Validation.Result([ .init(severity: .error, reason: "error"), .init(severity: .warning, reason: "warning"), @@ -127,7 +112,7 @@ struct ValidationTests { } @Test("Validation paths describe roots, keys, and indexes") - func validationPathsDescribeRootsKeysAndIndexes() { + func test_validationPathsDescribeRootsKeysAndIndexes() { let path = PureYAML.Validation.Path.root .appending(.key("routes")) .appending(.index(1)) @@ -139,7 +124,7 @@ struct ValidationTests { } @Test("Validation issue descriptions include severity reason and path") - func validationIssueDescriptionsIncludeSeverityReasonAndPath() { + func test_validationIssueDescriptionsIncludeSeverityReasonAndPath() { let issue = PureYAML.Validation.Issue( severity: .error, reason: "Duplicate mapping key 'title'", @@ -149,6 +134,19 @@ struct ValidationTests { #expect(issue.description == "error: Duplicate mapping key 'title' at $.title") } + @Test("Validation issue collection describes all failures") + func test_validationIssueCollectionDescription() { + let collection = PureYAML.Validation.Issue.Collection([ + .init(severity: .error, reason: "first", path: .root), + .init(severity: .warning, reason: "second", path: .init([.key("name")])), + ]) + + #expect(collection.description == """ + error: first at root + warning: second at $.name + """) + } + private func legacyModeRule(severity: PureYAML.Validation.Severity) -> PureYAML.Validation.Rule { PureYAML.Validation.Rule(description: "Mode must not be legacy") { context in guard case .string("legacy") = context.subject else { @@ -164,3 +162,18 @@ struct ValidationTests { } } } + +private func expectValidationError( + _ value: PureYAML.Model.Value, + using validator: PureYAML.Validation.Validator = .init(), + check: (PureYAML.Validation.Issue.Collection) -> Void, +) { + do { + try PureYAML.validate(value, using: validator) + recordIssue("expected validation failure") + } catch let collection as PureYAML.Validation.Issue.Collection { + check(collection) + } catch { + recordIssue("unexpected error \(error)") + } +} diff --git a/docs/DESIGN.md b/docs/DESIGN.md index cba96cd..2337b74 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -39,6 +39,9 @@ PureYAML │ └── Pair ├── Parsing │ ├── Parser +│ ├── Reader +│ ├── Scanner +│ ├── Token │ ├── Event │ ├── Mark │ ├── Line @@ -76,6 +79,14 @@ marks plus scalar and collection styles. Anchors and tags are carried as event metadata so the later scanner/composer work can add YAML feature coverage without changing the event shape. +The parser layer now also has an internal reader/scanner contract. +`Parsing.Reader` advances through Swift `String` input while tracking UTF-8 byte +indexes, lines, and columns. `Parsing.Scanner` emits lexical `Parsing.Token` +values for comments, indentation, block entries, mapping indicators, flow +delimiters, quoted scalar starts, block scalar headers, anchors, aliases, tags, +and source mark ranges. The scanner is not yet wired into `parse(_:)`; the next +roadmap slice composes scanner tokens into parser events. + The initial dumper emits block-style YAML from the model. The initial validator traverses parsed YAML values and reports path-aware diff --git a/docs/rules/testing.md b/docs/rules/testing.md index 6c58e18..5fb1bd2 100644 --- a/docs/rules/testing.md +++ b/docs/rules/testing.md @@ -8,6 +8,27 @@ The `withDependencies` overrides shown in the patterns below assume the Point-Fr ## Core rules +### Rule 0: OpenAPIKit discipline, Swift Testing syntax + +PureYAML tests must follow the OpenAPIKit testing approach, translated to Swift +Testing instead of XCTest: + +- Use focused `test_...` method names inside `@Suite` groups. +- Use direct equality checks for concrete values, rendered output, token/event + descriptions, paths, and error values. +- Use helper functions for repeated assertions and round-trip/error checks. +- Cover success and expected-failure behavior for every supported surface. +- For parser/scanner/emitter/model tests, assert all relevant dimensions: + what output must be present, what output must be absent, what must succeed, + what must fail, and the exact value or error returned. +- Error tests must pin concrete error cases and descriptions where the error is + user-facing; do not stop at "throws". +- Keep table-driven/argument tests for families of related cases. +- Keep tests fixture-like and realistic enough that a stub implementation cannot + satisfy them accidentally. +- Do not add XCTest-style `XCTestCase`, `XCTAssert...`, or mixed XCTest helpers + to new PureYAML tests. + ### Rule 1: Swift Testing framework usage Use modern Swift Testing: @@ -28,7 +49,8 @@ Control dependencies in tests: Structure tests clearly: - MUST use `@Suite` for logical grouping -- MUST name tests descriptively +- MUST name test methods with the `test_...` convention used by OpenAPIKit +- MUST keep `@Test("...")` descriptions clear and user-readable - MUST test one behavior per test - MUST NOT create kitchen sink tests