From bee7683ee26743e379773c98e7c2c5ee3870200f Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Fri, 5 Jun 2026 09:56:18 +0200 Subject: [PATCH] feat(parsing): parse token stream into events --- CHANGELOG.md | 2 + README.md | 10 +- Sources/Parsing/EventState.swift | 7 - Sources/Parsing/NodeKind.swift | 8 + Sources/Parsing/NodeProperties.swift | 11 + Sources/Parsing/NodeResult.swift | 6 + Sources/Parsing/ParseError.swift | 6 + Sources/Parsing/Parser.Events.swift | 211 +----------------- Sources/Parsing/Scanner.swift | 3 + Sources/Parsing/TokenCursor.swift | 91 ++++++++ Sources/Parsing/TokenEventParser.Block.swift | 126 +++++++++++ Sources/Parsing/TokenEventParser.Flow.swift | 83 +++++++ .../Parsing/TokenEventParser.Helpers.swift | 117 ++++++++++ Sources/Parsing/TokenEventParser.swift | 89 ++++++++ Tests/ParsingEventTests.swift | 92 +++++++- Tests/ScannerTests.swift | 1 + docs/DESIGN.md | 17 +- 17 files changed, 654 insertions(+), 226 deletions(-) delete mode 100644 Sources/Parsing/EventState.swift create mode 100644 Sources/Parsing/NodeKind.swift create mode 100644 Sources/Parsing/NodeProperties.swift create mode 100644 Sources/Parsing/NodeResult.swift create mode 100644 Sources/Parsing/TokenCursor.swift create mode 100644 Sources/Parsing/TokenEventParser.Block.swift create mode 100644 Sources/Parsing/TokenEventParser.Flow.swift create mode 100644 Sources/Parsing/TokenEventParser.Helpers.swift create mode 100644 Sources/Parsing/TokenEventParser.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 154e3da..90c9f19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 comments, indentation, block entries, mapping indicators, flow delimiters, quoted scalars, block scalar headers, anchors, aliases, tags, and source marks. +- Add the internal scanner-token event parser for block collections, flow + collections, aliases, anchors, tags, and line/column parser failures. - Expand tests into focused parsing, dumping, model, validation, and parse-error suites. - Add macOS, Linux, and WASM verification gates and CI jobs, plus a local Linux diff --git a/README.md b/README.md index 48f18c4..1e12e2e 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ flowchart TB classDef todo fill:#f2f4f7,stroke:#8e8e93,color:#111827 Issue2["#2 Event Model and Golden Tests"]:::done Issue3["#3 UTF-8 Reader and Scanner"]:::done - Issue4["#4 Token Stream to Events"]:::todo + Issue4["#4 Token Stream to Events"]:::done Issue5["#5 Events to PureYAML Values"]:::todo Issue6["#6 Scalars Tags and Aliases"]:::todo Issue7["#7 macOS Linux and WASM Hardening"]:::todo @@ -77,9 +77,11 @@ block sequences, ordered mappings, common scalars, quoted strings, comments, and a matching dumper. It also includes path-aware validation for structural YAML checks such as duplicate mapping keys. -It is not yet a full YAML 1.2 implementation. Anchors, aliases, tags, flow -collections, folded scalars, literal scalars, directives, and custom decoding -are planned work. +It is not yet a full YAML 1.2 implementation. The internal event parser now +recognizes anchors, aliases, tags, and flow collections from scanner tokens, but +the public value parser does not compose those events into `PureYAML.Model.Value` +yet. Folded scalars, literal scalars, directives, and custom decoding are +planned work. ## Attribution diff --git a/Sources/Parsing/EventState.swift b/Sources/Parsing/EventState.swift deleted file mode 100644 index 3cf5097..0000000 --- a/Sources/Parsing/EventState.swift +++ /dev/null @@ -1,7 +0,0 @@ -extension PureYAML.Parsing { - struct EventState { - var lines: [Line] - var index: Int - var events: [Event] - } -} diff --git a/Sources/Parsing/NodeKind.swift b/Sources/Parsing/NodeKind.swift new file mode 100644 index 0000000..1b5438c --- /dev/null +++ b/Sources/Parsing/NodeKind.swift @@ -0,0 +1,8 @@ +extension PureYAML.Parsing { + enum NodeKind: Equatable { + case alias + case mapping + case scalar + case sequence + } +} diff --git a/Sources/Parsing/NodeProperties.swift b/Sources/Parsing/NodeProperties.swift new file mode 100644 index 0000000..ff94a78 --- /dev/null +++ b/Sources/Parsing/NodeProperties.swift @@ -0,0 +1,11 @@ +extension PureYAML.Parsing { + struct NodeProperties: Equatable { + var anchor: String? + var tag: String? + var mark: Mark? + + static var none: Self { + Self(anchor: nil, tag: nil, mark: nil) + } + } +} diff --git a/Sources/Parsing/NodeResult.swift b/Sources/Parsing/NodeResult.swift new file mode 100644 index 0000000..34333cc --- /dev/null +++ b/Sources/Parsing/NodeResult.swift @@ -0,0 +1,6 @@ +extension PureYAML.Parsing { + struct NodeResult: Equatable { + var kind: NodeKind + var endMark: Mark + } +} diff --git a/Sources/Parsing/ParseError.swift b/Sources/Parsing/ParseError.swift index c36eb73..eebd534 100644 --- a/Sources/Parsing/ParseError.swift +++ b/Sources/Parsing/ParseError.swift @@ -8,8 +8,10 @@ public extension PureYAML.Parsing { case expectedMappingKey(line: Int) case expectedAnchorName(line: Int) case expectedAliasName(line: Int) + case expectedNode(line: Int, column: Int) case unterminatedTag(line: Int) case unterminatedQuotedString(line: Int) + case unexpectedToken(expected: String, actual: String, line: Int, column: Int) public var description: String { switch self { @@ -27,10 +29,14 @@ public extension PureYAML.Parsing { "expected an anchor name at line \(line)" case let .expectedAliasName(line): "expected an alias name at line \(line)" + case let .expectedNode(line, column): + "expected a YAML node at line \(line), column \(column)" case let .unterminatedTag(line): "unterminated tag at line \(line)" case let .unterminatedQuotedString(line): "unterminated quoted string at line \(line)" + case let .unexpectedToken(expected, actual, line, column): + "expected \(expected) at line \(line), column \(column), found \(actual)" } } } diff --git a/Sources/Parsing/Parser.Events.swift b/Sources/Parsing/Parser.Events.swift index 8243119..2e7a8d2 100644 --- a/Sources/Parsing/Parser.Events.swift +++ b/Sources/Parsing/Parser.Events.swift @@ -1,212 +1,7 @@ extension PureYAML.Parsing.Parser { func parseEvents(_ yaml: String) throws -> [PureYAML.Parsing.Event] { - let lines = try preprocess(yaml) - guard !lines.isEmpty else { - throw PureYAML.Parsing.ParseError.emptyDocument - } - - var state = PureYAML.Parsing.EventState( - lines: lines, - index: 0, - events: [ - .streamStart(mark: .start), - .documentStart(mark: .start), - ], - ) - try emitBlockEvents(&state, indent: lines[0].indent) - if state.index < lines.count { - throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: lines[state.index].number) - } - let endMark = lines.last?.endMark ?? .start - state.events.append(.documentEnd(mark: endMark)) - state.events.append(.streamEnd(mark: endMark)) - return state.events - } -} - -extension PureYAML.Parsing.Parser { - func emitBlockEvents( - _ state: inout PureYAML.Parsing.EventState, - indent: Int, - ) throws { - guard state.index < state.lines.count else { - state.events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: .start)) - return - } - let line = state.lines[state.index] - if line.indent < indent { - state.events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: line.mark)) - return - } - guard line.indent == indent else { - throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: line.number) - } - if line.content.hasPrefix("-") { - try emitSequenceEvents(&state, indent: indent) - } else if splitMappingEntryWithOffsets(line.content) != nil { - try emitMappingEvents(&state, indent: indent) - } else { - state.index += 1 - try state.events.append(scalarEvent(line.content, mark: line.mark, line: line.number)) - } - } - - func emitSequenceEvents( - _ state: inout PureYAML.Parsing.EventState, - indent: Int, - ) throws { - let startMark = state.lines[state.index].mark - state.events.append(.sequenceStart(anchor: nil, tag: nil, style: .block, mark: startMark)) - while state.index < state.lines.count { - let line = state.lines[state.index] - if line.indent < indent { - break - } - guard line.indent == indent else { - throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: line.number) - } - guard line.content == "-" || line.content.hasPrefix("- ") else { - throw PureYAML.Parsing.ParseError.mixedCollectionStyles(line: line.number) - } - - let rest = trimWithOffset(line.content, in: line.content.index(after: line.content.startIndex) ..< line.content.endIndex) - state.index += 1 - if rest.0.isEmpty { - if state.index < state.lines.count, state.lines[state.index].indent > indent { - try emitBlockEvents(&state, indent: state.lines[state.index].indent) - } else { - state.events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: mark(in: line, offset: rest.1))) - } - } else if let entry = splitMappingEntryWithOffsets(rest.0) { - try emitInlineMappingEvents( - entry, - line: line, - baseOffset: rest.1, - state: &state, - parentIndent: indent, - ) - } else { - try state.events.append(scalarEvent(rest.0, mark: mark(in: line, offset: rest.1), line: line.number)) - } - } - state.events.append(.sequenceEnd(mark: eventEndMark(state, fallback: startMark))) - } - - func emitMappingEvents( - _ state: inout PureYAML.Parsing.EventState, - indent: Int, - ) throws { - let startMark = state.lines[state.index].mark - state.events.append(.mappingStart(anchor: nil, tag: nil, style: .block, mark: startMark)) - try emitMappingPairEvents(&state, indent: indent) - state.events.append(.mappingEnd(mark: eventEndMark(state, fallback: startMark))) - } - - func emitMappingPairEvents( - _ state: inout PureYAML.Parsing.EventState, - indent: Int, - ) throws { - while state.index < state.lines.count { - let line = state.lines[state.index] - if line.indent < indent { - break - } - guard line.indent == indent else { - throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: line.number) - } - guard let entry = splitMappingEntryWithOffsets(line.content) else { - throw PureYAML.Parsing.ParseError.expectedMappingKey(line: line.number) - } - state.index += 1 - try emitMappingEntry(entry, line: line, baseOffset: 0, state: &state, parentIndent: indent) - } - } - - func emitInlineMappingEvents( - _ entry: PureYAML.Parsing.MappingEntry, - line: PureYAML.Parsing.Line, - baseOffset: Int, - state: inout PureYAML.Parsing.EventState, - parentIndent: Int, - ) throws { - state.events.append(.mappingStart(anchor: nil, tag: nil, style: .block, mark: mark(in: line, offset: baseOffset + entry.keyOffset))) - try emitMappingEntry(entry, line: line, baseOffset: baseOffset, state: &state, parentIndent: parentIndent) - if state.index < state.lines.count, state.lines[state.index].indent > parentIndent { - try emitMappingPairEvents(&state, indent: state.lines[state.index].indent) - } - state.events.append(.mappingEnd(mark: eventEndMark(state, fallback: line.endMark))) - } - - func emitMappingEntry( - _ entry: PureYAML.Parsing.MappingEntry, - line: PureYAML.Parsing.Line, - baseOffset: Int, - state: inout PureYAML.Parsing.EventState, - parentIndent: Int, - ) throws { - state.events.append(.scalar( - value: entry.key, - anchor: nil, - tag: nil, - style: entry.keyStyle, - mark: mark(in: line, offset: baseOffset + entry.keyOffset), - )) - if !entry.value.isEmpty { - try state.events.append(scalarEvent( - entry.value, - mark: mark(in: line, offset: baseOffset + entry.valueOffset), - line: line.number, - )) - } else if state.index < state.lines.count, state.lines[state.index].indent > parentIndent { - try emitBlockEvents(&state, indent: state.lines[state.index].indent) - } else { - state.events.append(.scalar( - value: "", - anchor: nil, - tag: nil, - style: .plain, - mark: mark(in: line, offset: baseOffset + entry.valueOffset), - )) - } - } - - func scalarEvent( - _ text: String, - mark: PureYAML.Parsing.Mark, - line: Int, - ) throws -> PureYAML.Parsing.Event { - let style = scalarStyle(text) - let value: String = switch style { - case .plain: - text - case .singleQuoted: - try parseSingleQuoted(text, line: line) - case .doubleQuoted: - try parseDoubleQuoted(text, line: line) - } - return .scalar(value: value, anchor: nil, tag: nil, style: style, mark: mark) - } - - func mark( - in line: PureYAML.Parsing.Line, - offset: Int, - ) -> PureYAML.Parsing.Mark { - let boundedOffset = max(0, min(offset, line.content.count)) - let index = line.content.index(line.content.startIndex, offsetBy: boundedOffset) - return PureYAML.Parsing.Mark( - line: line.number, - column: line.column + boundedOffset, - index: line.index + line.content[.. PureYAML.Parsing.Mark { - if state.index > 0, state.lines.indices.contains(state.index - 1) { - return state.lines[state.index - 1].endMark - } - return fallback + let tokens = try PureYAML.Parsing.Scanner().scan(yaml) + var parser = PureYAML.Parsing.TokenEventParser(tokens: tokens, scalarParser: self) + return try parser.parse() } } diff --git a/Sources/Parsing/Scanner.swift b/Sources/Parsing/Scanner.swift index 3b98cab..d382f0d 100644 --- a/Sources/Parsing/Scanner.swift +++ b/Sources/Parsing/Scanner.swift @@ -92,6 +92,9 @@ extension PureYAML.Parsing.Scanner { state.indentation.removeLast() state.append(.dedent(width: state.indentation.last ?? 0), mark: end, endMark: end) } + guard width == state.indentation.last else { + throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: end.line) + } } } diff --git a/Sources/Parsing/TokenCursor.swift b/Sources/Parsing/TokenCursor.swift new file mode 100644 index 0000000..0c6655a --- /dev/null +++ b/Sources/Parsing/TokenCursor.swift @@ -0,0 +1,91 @@ +extension PureYAML.Parsing { + struct TokenCursor { + private var tokens: [Token] + private(set) var index: Int + + init(tokens: [Token]) { + self.tokens = tokens.filter { !$0.kind.isComment } + index = 0 + } + + var current: Token? { + guard tokens.indices.contains(index) else { + return nil + } + return tokens[index] + } + + var previous: Token? { + let previousIndex = index - 1 + guard tokens.indices.contains(previousIndex) else { + return nil + } + return tokens[previousIndex] + } + + mutating func advance() -> Token? { + guard let token = current else { + return nil + } + index += 1 + return token + } + } +} + +extension PureYAML.Parsing.TokenKind { + var isBlockEntry: Bool { + if case .blockEntry = self { + return true + } + return false + } + + var isComment: Bool { + if case .comment = self { + return true + } + return false + } + + var isDedent: Bool { + if case .dedent = self { + return true + } + return false + } + + var isFlowTerminator: Bool { + switch self { + case .flowEntry, .flowMappingEnd, .flowSequenceEnd: + true + default: + false + } + } + + var isIndent: Bool { + if case .indent = self { + return true + } + return false + } + + var isMappingKey: Bool { + if case .mappingKey = self { + return true + } + return false + } + + var isStreamEnd: Bool { + if case .streamEnd = self { + return true + } + return false + } + + var isTerminator: Bool { + isDedent || isFlowTerminator || isStreamEnd + } +} diff --git a/Sources/Parsing/TokenEventParser.Block.swift b/Sources/Parsing/TokenEventParser.Block.swift new file mode 100644 index 0000000..3da11a0 --- /dev/null +++ b/Sources/Parsing/TokenEventParser.Block.swift @@ -0,0 +1,126 @@ +extension PureYAML.Parsing.TokenEventParser { + mutating func parseBlockSequence( + properties: PureYAML.Parsing.NodeProperties, + ) throws -> PureYAML.Parsing.NodeResult { + let start = try expect("block sequence entry") { $0.isBlockEntry } + let startMark = properties.mark ?? start.mark + var endMark = start.endMark + events.append(.sequenceStart( + anchor: properties.anchor, + tag: properties.tag, + style: .block, + mark: startMark, + )) + + try parseBlockSequenceItem(after: start, endMark: &endMark) + while cursor.current?.kind.isBlockEntry == true { + let entry = try expect("block sequence entry") { $0.isBlockEntry } + try parseBlockSequenceItem(after: entry, endMark: &endMark) + } + + events.append(.sequenceEnd(mark: endMark)) + return PureYAML.Parsing.NodeResult(kind: .sequence, endMark: endMark) + } + + mutating func parseBlockSequenceItem( + after entry: PureYAML.Parsing.Token, + endMark: inout PureYAML.Parsing.Mark, + ) throws { + if hasNodeOnSameLine(after: entry) { + if cursor.current?.kind.isMappingKey == true { + endMark = try parseBlockMapping( + properties: .none, + includeIndentedContinuation: true, + ).endMark + } else { + endMark = try parseNode().endMark + } + } else if cursor.current?.kind.isIndent == true { + _ = try expect("indent") { $0.isIndent } + endMark = try parseNode().endMark + _ = try expect("dedent") { $0.isDedent } + } else { + let mark = entry.endMark + events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: mark)) + endMark = mark + } + } + + mutating func parseBlockMapping( + properties: PureYAML.Parsing.NodeProperties, + includeIndentedContinuation: Bool, + ) throws -> PureYAML.Parsing.NodeResult { + let startMark = properties.mark ?? cursor.current?.mark ?? .start + var endMark = startMark + events.append(.mappingStart( + anchor: properties.anchor, + tag: properties.tag, + style: .block, + mark: startMark, + )) + while cursor.current?.kind.isMappingKey == true { + endMark = try parseMappingPair().endMark + } + while includeIndentedContinuation, cursor.current?.kind.isIndent == true { + _ = try expect("indent") { $0.isIndent } + guard cursor.current?.kind.isMappingKey == true else { + throw unexpectedToken(expected: "mapping key") + } + while cursor.current?.kind.isMappingKey == true { + endMark = try parseMappingPair().endMark + } + _ = try expect("dedent") { $0.isDedent } + } + if let current = cursor.current, !canEndBlockMapping(at: current) { + throw PureYAML.Parsing.ParseError.unexpectedToken( + expected: "mapping key", + actual: current.kind.description, + line: current.mark.line, + column: current.mark.column, + ) + } + events.append(.mappingEnd(mark: endMark)) + return PureYAML.Parsing.NodeResult(kind: .mapping, endMark: endMark) + } + + mutating func parseBlockScalar( + properties: PureYAML.Parsing.NodeProperties, + ) throws -> PureYAML.Parsing.NodeResult { + let header = try expect("block scalar header") { kind in + if case .blockScalarHeader = kind { + return true + } + return false + } + var lines: [String] = [] + var endMark = header.endMark + if cursor.current?.kind.isIndent == true { + _ = try expect("indent") { $0.isIndent } + while cursor.current != nil, cursor.current?.kind.isDedent != true { + guard case let .scalar(value, style) = cursor.current?.kind else { + throw unexpectedToken(expected: "block scalar content") + } + let token = cursor.advance() ?? header + try lines.append(decodeScalar(value, style: style, line: token.mark.line)) + endMark = token.endMark + } + _ = try expect("dedent") { $0.isDedent } + } + + let value: String = switch header.kind { + case .blockScalarHeader(.folded): + lines.joined(separator: " ") + (lines.isEmpty ? "" : "\n") + default: + lines.joined(separator: "\n") + (lines.isEmpty ? "" : "\n") + } + let mark = properties.mark ?? header.mark + events.append(.scalar( + value: value, + anchor: properties.anchor, + tag: properties.tag, + style: .plain, + mark: mark, + )) + return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: endMark) + } +} diff --git a/Sources/Parsing/TokenEventParser.Flow.swift b/Sources/Parsing/TokenEventParser.Flow.swift new file mode 100644 index 0000000..c384f4b --- /dev/null +++ b/Sources/Parsing/TokenEventParser.Flow.swift @@ -0,0 +1,83 @@ +extension PureYAML.Parsing.TokenEventParser { + mutating func parseFlowSequence( + properties: PureYAML.Parsing.NodeProperties, + ) throws -> PureYAML.Parsing.NodeResult { + let start = try expect("flow sequence start") { kind in + if case .flowSequenceStart = kind { + return true + } + return false + } + let startMark = properties.mark ?? start.mark + var endMark = start.endMark + events.append(.sequenceStart( + anchor: properties.anchor, + tag: properties.tag, + style: .flow, + mark: startMark, + )) + while cursor.current != nil { + if case .flowSequenceEnd = cursor.current?.kind { + let end = try expect("flow sequence end") { kind in + if case .flowSequenceEnd = kind { + return true + } + return false + } + endMark = end.endMark + events.append(.sequenceEnd(mark: endMark)) + return PureYAML.Parsing.NodeResult(kind: .sequence, endMark: endMark) + } + if case .flowEntry = cursor.current?.kind { + _ = cursor.advance() + continue + } + endMark = try parseNode().endMark + if cursor.current?.kind.isFlowTerminator == false { + throw unexpectedToken(expected: "flow entry or flow sequence end") + } + } + throw unexpectedToken(expected: "flow sequence end") + } + + mutating func parseFlowMapping( + properties: PureYAML.Parsing.NodeProperties, + ) throws -> PureYAML.Parsing.NodeResult { + let start = try expect("flow mapping start") { kind in + if case .flowMappingStart = kind { + return true + } + return false + } + let startMark = properties.mark ?? start.mark + var endMark = start.endMark + events.append(.mappingStart( + anchor: properties.anchor, + tag: properties.tag, + style: .flow, + mark: startMark, + )) + while cursor.current != nil { + if case .flowMappingEnd = cursor.current?.kind { + let end = try expect("flow mapping end") { kind in + if case .flowMappingEnd = kind { + return true + } + return false + } + endMark = end.endMark + events.append(.mappingEnd(mark: endMark)) + return PureYAML.Parsing.NodeResult(kind: .mapping, endMark: endMark) + } + if case .flowEntry = cursor.current?.kind { + _ = cursor.advance() + continue + } + endMark = try parseMappingPair().endMark + if cursor.current?.kind.isFlowTerminator == false { + throw unexpectedToken(expected: "flow entry or flow mapping end") + } + } + throw unexpectedToken(expected: "flow mapping end") + } +} diff --git a/Sources/Parsing/TokenEventParser.Helpers.swift b/Sources/Parsing/TokenEventParser.Helpers.swift new file mode 100644 index 0000000..4aff137 --- /dev/null +++ b/Sources/Parsing/TokenEventParser.Helpers.swift @@ -0,0 +1,117 @@ +extension PureYAML.Parsing.TokenEventParser { + mutating func parseMappingPair() throws -> PureYAML.Parsing.NodeResult { + _ = try expect("mapping key") { $0.isMappingKey } + _ = try parseNode() + let valueIndicator = try expect("mapping value") { kind in + if case .mappingValue = kind { + return true + } + return false + } + + if cursor.current?.kind.isIndent == true { + _ = try expect("indent") { $0.isIndent } + let result = try parseNode() + _ = try expect("dedent") { $0.isDedent } + return result + } + if hasNodeOnSameLine(after: valueIndicator) { + return try parseNode() + } + + let mark = valueIndicator.endMark + events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: mark)) + return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: mark) + } + + mutating func consumeProperties() -> PureYAML.Parsing.NodeProperties { + var properties = PureYAML.Parsing.NodeProperties.none + while let token = cursor.current { + switch token.kind { + case let .anchor(anchor): + properties.anchor = anchor + properties.mark = properties.mark ?? token.mark + _ = cursor.advance() + case let .tag(tag): + properties.tag = tag + properties.mark = properties.mark ?? token.mark + _ = cursor.advance() + default: + return properties + } + } + return properties + } + + func decodeScalar( + _ value: String, + style: PureYAML.Parsing.ScalarStyle, + line: Int, + ) throws -> String { + switch style { + case .doubleQuoted: + try scalarParser.parseDoubleQuoted("\"\(value)\"", line: line) + case .plain: + value + case .singleQuoted: + try scalarParser.parseSingleQuoted("'\(value)'", line: line) + } + } + + func hasNodeOnSameLine(after token: PureYAML.Parsing.Token) -> Bool { + guard let current = cursor.current, !current.kind.isTerminator else { + return false + } + return current.mark.line == token.mark.line + } + + func canEndBlockMapping(at token: PureYAML.Parsing.Token) -> Bool { + token.kind.isBlockEntry + || token.kind.isDedent + || token.kind.isFlowTerminator + || token.kind.isStreamEnd + } + + mutating func expect( + _ expected: String, + matching: (PureYAML.Parsing.TokenKind) -> Bool, + ) throws -> PureYAML.Parsing.Token { + guard let token = cursor.current else { + let mark = cursor.previous?.endMark ?? .start + throw PureYAML.Parsing.ParseError.unexpectedToken( + expected: expected, + actual: "end of tokens", + line: mark.line, + column: mark.column, + ) + } + guard matching(token.kind) else { + throw PureYAML.Parsing.ParseError.unexpectedToken( + expected: expected, + actual: token.kind.description, + line: token.mark.line, + column: token.mark.column, + ) + } + _ = cursor.advance() + return token + } + + func unexpectedToken(expected: String) -> PureYAML.Parsing.ParseError { + guard let token = cursor.current else { + let mark = cursor.previous?.endMark ?? .start + return PureYAML.Parsing.ParseError.unexpectedToken( + expected: expected, + actual: "end of tokens", + line: mark.line, + column: mark.column, + ) + } + return PureYAML.Parsing.ParseError.unexpectedToken( + expected: expected, + actual: token.kind.description, + line: token.mark.line, + column: token.mark.column, + ) + } +} diff --git a/Sources/Parsing/TokenEventParser.swift b/Sources/Parsing/TokenEventParser.swift new file mode 100644 index 0000000..c665bf4 --- /dev/null +++ b/Sources/Parsing/TokenEventParser.swift @@ -0,0 +1,89 @@ +extension PureYAML.Parsing { + struct TokenEventParser { + var cursor: TokenCursor + let scalarParser: Parser + var events: [Event] + + init( + tokens: [Token], + scalarParser: Parser, + ) { + cursor = TokenCursor(tokens: tokens) + self.scalarParser = scalarParser + events = [] + } + + mutating func parse() throws -> [Event] { + let streamStart = try expect("stream start") { kind in + if case .streamStart = kind { + return true + } + return false + } + guard cursor.current?.kind.isStreamEnd != true else { + throw ParseError.emptyDocument + } + + events.append(.streamStart(mark: streamStart.mark)) + events.append(.documentStart(mark: streamStart.mark)) + let result = try parseNode() + guard cursor.current?.kind.isStreamEnd == true else { + throw unexpectedToken(expected: "stream end") + } + _ = try expect("stream end") { kind in + if case .streamEnd = kind { + return true + } + return false + } + events.append(.documentEnd(mark: result.endMark)) + events.append(.streamEnd(mark: result.endMark)) + return events + } + } +} + +extension PureYAML.Parsing.TokenEventParser { + mutating func parseNode() throws -> PureYAML.Parsing.NodeResult { + let properties = consumeProperties() + guard let token = cursor.current, !token.kind.isTerminator else { + let mark = properties.mark ?? cursor.current?.mark ?? cursor.previous?.endMark ?? .start + throw PureYAML.Parsing.ParseError.expectedNode(line: mark.line, column: mark.column) + } + + switch token.kind { + case let .alias(anchor): + _ = cursor.advance() + let mark = properties.mark ?? token.mark + events.append(.alias(anchor: anchor, mark: mark)) + return PureYAML.Parsing.NodeResult(kind: .alias, endMark: token.endMark) + case .blockEntry: + return try parseBlockSequence(properties: properties) + case .blockScalarHeader: + return try parseBlockScalar(properties: properties) + case .flowMappingStart: + return try parseFlowMapping(properties: properties) + case .flowSequenceStart: + return try parseFlowSequence(properties: properties) + case .mappingKey: + return try parseBlockMapping( + properties: properties, + includeIndentedContinuation: false, + ) + case let .scalar(value, style): + _ = cursor.advance() + let decoded = try decodeScalar(value, style: style, line: token.mark.line) + let mark = properties.mark ?? token.mark + events.append(.scalar( + value: decoded, + anchor: properties.anchor, + tag: properties.tag, + style: style, + mark: mark, + )) + return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: token.endMark) + default: + throw PureYAML.Parsing.ParseError.expectedNode(line: token.mark.line, column: token.mark.column) + } + } +} diff --git a/Tests/ParsingEventTests.swift b/Tests/ParsingEventTests.swift index 83d4427..3b63cb8 100644 --- a/Tests/ParsingEventTests.swift +++ b/Tests/ParsingEventTests.swift @@ -75,6 +75,65 @@ struct ParsingEventTests { #expect(!events.map(\.description).contains("scalar value=\"\" anchor=- tag=- style=plain @1:1@0")) } + @Test("Emits golden events for flow collections aliases anchors and tags") + func test_flowCollectionsAliasesAnchorsAndTags() throws { + let events = try PureYAML.Parsing.Parser().parseEvents( + """ + flow: [!Thing &item "value", *item, {tagged: ! 'it''s'}] + """, + ) + + let descriptions = events.map(\.description) + #expect(descriptions == [ + "streamStart @1:1@0", + "documentStart @1:1@0", + "mappingStart anchor=- tag=- style=block @1:1@0", + "scalar value=\"flow\" anchor=- tag=- style=plain @1:1@0", + "sequenceStart anchor=- tag=- style=flow @1:7@6", + "scalar value=\"value\" anchor=item tag=!Thing style=doubleQuoted @1:8@7", + "alias anchor=item @1:30@29", + "mappingStart anchor=- tag=- style=flow @1:37@36", + "scalar value=\"tagged\" anchor=- tag=- style=plain @1:38@37", + "scalar value=\"it's\" anchor=- tag=! style=singleQuoted @1:46@45", + "mappingEnd @1:84@83", + "sequenceEnd @1:85@84", + "mappingEnd @1:85@84", + "documentEnd @1:85@84", + "streamEnd @1:85@84", + ]) + #expect(!descriptions.contains("sequenceStart anchor=- tag=- style=block @1:7@6")) + #expect(!descriptions.contains { $0.contains("it''s") }) + } + + @Test("Emits golden events for block scalar headers") + func test_blockScalarHeaders() throws { + let events = try PureYAML.Parsing.Parser().parseEvents( + """ + text: | + one + two + folded: > + one + two + """, + ) + + let descriptions = events.map(\.description) + #expect(descriptions == [ + "streamStart @1:1@0", + "documentStart @1:1@0", + "mappingStart anchor=- tag=- style=block @1:1@0", + "scalar value=\"text\" anchor=- tag=- style=plain @1:1@0", + "scalar value=\"one\\ntwo\\n\" anchor=- tag=- style=plain @1:7@6", + "scalar value=\"folded\" anchor=- tag=- style=plain @4:1@20", + "scalar value=\"one two\\n\" anchor=- tag=- style=plain @4:9@28", + "mappingEnd @6:6@41", + "documentEnd @6:6@41", + "streamEnd @6:6@41", + ]) + #expect(!descriptions.contains { $0.contains("one\\ntwo") && $0.contains("@4:9@28") }) + } + @Test("Reports parse-event errors") func test_errorReporting() { expectError(PureYAML.Parsing.ParseError.emptyDocument) { @@ -83,8 +142,39 @@ struct ParsingEventTests { expectError(PureYAML.Parsing.ParseError.unexpectedIndentation(line: 3)) { try PureYAML.Parsing.Parser().parseEvents("root:\n child: yes\n wrong: no") } - expectError(PureYAML.Parsing.ParseError.expectedMappingKey(line: 3)) { + expectError(PureYAML.Parsing.ParseError.unexpectedToken( + expected: "mapping key", + actual: "scalar value=\"dangling\" style=plain", + line: 3, + column: 3, + )) { try PureYAML.Parsing.Parser().parseEvents("root:\n child: value\n dangling") } + expectParseEventError( + "items: [one, two", + PureYAML.Parsing.ParseError.unexpectedToken( + expected: "flow entry or flow sequence end", + actual: "streamEnd", + line: 1, + column: 17, + ), + description: "expected flow entry or flow sequence end at line 1, column 17, found streamEnd", + ) + } +} + +private func expectParseEventError( + _ yaml: String, + _ expected: PureYAML.Parsing.ParseError, + description: String, +) { + do { + _ = try PureYAML.Parsing.Parser().parseEvents(yaml) + recordIssue("expected error \(expected)") + } catch let error as PureYAML.Parsing.ParseError { + #expect(error == expected) + #expect(error.description == description) + } catch { + recordIssue("unexpected error \(error)") } } diff --git a/Tests/ScannerTests.swift b/Tests/ScannerTests.swift index cba4648..476b26f 100644 --- a/Tests/ScannerTests.swift +++ b/Tests/ScannerTests.swift @@ -171,6 +171,7 @@ struct ScannerTests { @Test("Reports scanner errors") func test_errorReporting() { expectScannerError("\tname: value", .tabIndentation(line: 1)) + expectScannerError("root:\n child: yes\n wrong: no", .unexpectedIndentation(line: 3)) expectScannerError("name: \"open", .unterminatedQuotedString(line: 1)) expectScannerError("name: 'open", .unterminatedQuotedString(line: 1)) expectScannerError("anchor: &", .expectedAnchorName(line: 1)) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 2337b74..291f5b1 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -42,6 +42,8 @@ PureYAML │ ├── Reader │ ├── Scanner │ ├── Token +│ ├── TokenCursor +│ ├── TokenEventParser │ ├── Event │ ├── Mark │ ├── Line @@ -79,13 +81,16 @@ marks plus scalar and collection styles. Anchors and tags are carried as event metadata so the later scanner/composer work can add YAML feature coverage without changing the event shape. -The parser layer now also has an internal reader/scanner contract. +The parser layer now also has an internal reader/scanner/event contract. `Parsing.Reader` advances through Swift `String` input while tracking UTF-8 byte indexes, lines, and columns. `Parsing.Scanner` emits lexical `Parsing.Token` values for comments, indentation, block entries, mapping indicators, flow delimiters, quoted scalar starts, block scalar headers, anchors, aliases, tags, -and source mark ranges. The scanner is not yet wired into `parse(_:)`; the next -roadmap slice composes scanner tokens into parser events. +and source mark ranges. `Parsing.TokenEventParser` consumes that token stream +and emits `Parsing.Event` values for block collections, flow collections, +aliases, anchors, tags, and scalars. The token event parser is not yet wired +into `parse(_:)`; the next roadmap slice composes events into +`PureYAML.Model.Value`. The initial dumper emits block-style YAML from the model. @@ -97,10 +102,10 @@ blank validator plus custom rules for project-specific checks. Compatibility should be added in small, test-backed slices: -1. Flow collections. +1. Compose event streams into `PureYAML.Model.Value`. 2. Literal and folded scalars. -3. Anchors and aliases. -4. Tags. +3. Anchor and alias resolution. +4. Tag-aware scalar and collection handling. 5. Validation rules beyond duplicate-key behavior. 6. Codable-style decoding and encoding. 7. Yams corpus comparison tests in a separate compatibility suite.