diff --git a/CHANGELOG.md b/CHANGELOG.md index d09d256..ddd099f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 dump/parse round trips. - Add path-aware YAML validation with duplicate-key checks, custom rules, and warning collection. +- Add the internal parse event model with marks, scalar styles, collection + styles, alias events, anchor/tag metadata, and golden event tests for current + parser behavior. - Expand tests into focused parsing, dumping, model, validation, and parse-error suites. - Add macOS, Linux, and WASM verification gates and CI jobs, plus a local Linux diff --git a/README.md b/README.md index abac0da..9c410dd 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ flowchart TB classDef review fill:#fff7d6,stroke:#ffcc00,color:#111827 classDef epic fill:#f2e5ff,stroke:#af52de,color:#111827 classDef todo fill:#f2f4f7,stroke:#8e8e93,color:#111827 - Issue2["#2 Event Model and Golden Tests"]:::todo + Issue2["#2 Event Model and Golden Tests"]:::done Issue3["#3 UTF-8 Reader and Scanner"]:::todo Issue4["#4 Token Stream to Events"]:::todo Issue5["#5 Events to PureYAML Values"]:::todo diff --git a/Sources/Parsing/CollectionStyle.swift b/Sources/Parsing/CollectionStyle.swift new file mode 100644 index 0000000..ee3f128 --- /dev/null +++ b/Sources/Parsing/CollectionStyle.swift @@ -0,0 +1,10 @@ +extension PureYAML.Parsing { + enum CollectionStyle: String, Equatable, CustomStringConvertible { + case block + case flow + + var description: String { + rawValue + } + } +} diff --git a/Sources/Parsing/Event.swift b/Sources/Parsing/Event.swift new file mode 100644 index 0000000..7897b17 --- /dev/null +++ b/Sources/Parsing/Event.swift @@ -0,0 +1,66 @@ +extension PureYAML.Parsing { + enum Event: Equatable, CustomStringConvertible { + case streamStart(mark: Mark) + case streamEnd(mark: Mark) + case documentStart(mark: Mark) + case documentEnd(mark: Mark) + case sequenceStart(anchor: String?, tag: String?, style: CollectionStyle, mark: Mark) + case sequenceEnd(mark: Mark) + case mappingStart(anchor: String?, tag: String?, style: CollectionStyle, mark: Mark) + case mappingEnd(mark: Mark) + case scalar(value: String, anchor: String?, tag: String?, style: ScalarStyle, mark: Mark) + case alias(anchor: String, mark: Mark) + + var description: String { + switch self { + case let .streamStart(mark): + "streamStart @\(mark)" + case let .streamEnd(mark): + "streamEnd @\(mark)" + case let .documentStart(mark): + "documentStart @\(mark)" + case let .documentEnd(mark): + "documentEnd @\(mark)" + case let .sequenceStart(anchor, tag, style, mark): + "sequenceStart anchor=\(describe(anchor)) tag=\(describe(tag)) style=\(style) @\(mark)" + case let .sequenceEnd(mark): + "sequenceEnd @\(mark)" + case let .mappingStart(anchor, tag, style, mark): + "mappingStart anchor=\(describe(anchor)) tag=\(describe(tag)) style=\(style) @\(mark)" + case let .mappingEnd(mark): + "mappingEnd @\(mark)" + case let .scalar(value, anchor, tag, style, mark): + "scalar value=\"\(escape(value))\" anchor=\(describe(anchor)) tag=\(describe(tag)) style=\(style) @\(mark)" + case let .alias(anchor, mark): + "alias anchor=\(anchor) @\(mark)" + } + } + } +} + +extension PureYAML.Parsing.Event { + private func describe(_ value: String?) -> String { + value ?? "-" + } + + private func escape(_ value: String) -> String { + var output = "" + for character in value { + switch character { + case "\\": + output += "\\\\" + case "\"": + output += "\\\"" + case "\n": + output += "\\n" + case "\r": + output += "\\r" + case "\t": + output += "\\t" + default: + output.append(character) + } + } + return output + } +} diff --git a/Sources/Parsing/EventState.swift b/Sources/Parsing/EventState.swift new file mode 100644 index 0000000..3cf5097 --- /dev/null +++ b/Sources/Parsing/EventState.swift @@ -0,0 +1,7 @@ +extension PureYAML.Parsing { + struct EventState { + var lines: [Line] + var index: Int + var events: [Event] + } +} diff --git a/Sources/Parsing/Line.swift b/Sources/Parsing/Line.swift index a9137a9..3887cc5 100644 --- a/Sources/Parsing/Line.swift +++ b/Sources/Parsing/Line.swift @@ -2,6 +2,20 @@ extension PureYAML.Parsing { struct Line: Equatable { var number: Int var indent: Int + var column: Int + var index: Int var content: String + + var mark: Mark { + Mark(line: number, column: column, index: index) + } + + var endMark: Mark { + Mark( + line: number, + column: column + content.count, + index: index + content.utf8.count, + ) + } } } diff --git a/Sources/Parsing/MappingEntry.swift b/Sources/Parsing/MappingEntry.swift new file mode 100644 index 0000000..f6e0f07 --- /dev/null +++ b/Sources/Parsing/MappingEntry.swift @@ -0,0 +1,9 @@ +extension PureYAML.Parsing { + struct MappingEntry: Equatable { + var key: String + var keyStyle: ScalarStyle + var keyOffset: Int + var value: String + var valueOffset: Int + } +} diff --git a/Sources/Parsing/Mark.swift b/Sources/Parsing/Mark.swift new file mode 100644 index 0000000..8aa71aa --- /dev/null +++ b/Sources/Parsing/Mark.swift @@ -0,0 +1,25 @@ +extension PureYAML.Parsing { + struct Mark: Equatable, CustomStringConvertible { + var line: Int + var column: Int + var index: Int + + init( + line: Int, + column: Int, + index: Int, + ) { + self.line = line + self.column = column + self.index = index + } + + static var start: Self { + Self(line: 1, column: 1, index: 0) + } + + var description: String { + "\(line):\(column)@\(index)" + } + } +} diff --git a/Sources/Parsing/Parser.Events.swift b/Sources/Parsing/Parser.Events.swift new file mode 100644 index 0000000..8243119 --- /dev/null +++ b/Sources/Parsing/Parser.Events.swift @@ -0,0 +1,212 @@ +extension PureYAML.Parsing.Parser { + func parseEvents(_ yaml: String) throws -> [PureYAML.Parsing.Event] { + let lines = try preprocess(yaml) + guard !lines.isEmpty else { + throw PureYAML.Parsing.ParseError.emptyDocument + } + + var state = PureYAML.Parsing.EventState( + lines: lines, + index: 0, + events: [ + .streamStart(mark: .start), + .documentStart(mark: .start), + ], + ) + try emitBlockEvents(&state, indent: lines[0].indent) + if state.index < lines.count { + throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: lines[state.index].number) + } + let endMark = lines.last?.endMark ?? .start + state.events.append(.documentEnd(mark: endMark)) + state.events.append(.streamEnd(mark: endMark)) + return state.events + } +} + +extension PureYAML.Parsing.Parser { + func emitBlockEvents( + _ state: inout PureYAML.Parsing.EventState, + indent: Int, + ) throws { + guard state.index < state.lines.count else { + state.events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: .start)) + return + } + let line = state.lines[state.index] + if line.indent < indent { + state.events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: line.mark)) + return + } + guard line.indent == indent else { + throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: line.number) + } + if line.content.hasPrefix("-") { + try emitSequenceEvents(&state, indent: indent) + } else if splitMappingEntryWithOffsets(line.content) != nil { + try emitMappingEvents(&state, indent: indent) + } else { + state.index += 1 + try state.events.append(scalarEvent(line.content, mark: line.mark, line: line.number)) + } + } + + func emitSequenceEvents( + _ state: inout PureYAML.Parsing.EventState, + indent: Int, + ) throws { + let startMark = state.lines[state.index].mark + state.events.append(.sequenceStart(anchor: nil, tag: nil, style: .block, mark: startMark)) + while state.index < state.lines.count { + let line = state.lines[state.index] + if line.indent < indent { + break + } + guard line.indent == indent else { + throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: line.number) + } + guard line.content == "-" || line.content.hasPrefix("- ") else { + throw PureYAML.Parsing.ParseError.mixedCollectionStyles(line: line.number) + } + + let rest = trimWithOffset(line.content, in: line.content.index(after: line.content.startIndex) ..< line.content.endIndex) + state.index += 1 + if rest.0.isEmpty { + if state.index < state.lines.count, state.lines[state.index].indent > indent { + try emitBlockEvents(&state, indent: state.lines[state.index].indent) + } else { + state.events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: mark(in: line, offset: rest.1))) + } + } else if let entry = splitMappingEntryWithOffsets(rest.0) { + try emitInlineMappingEvents( + entry, + line: line, + baseOffset: rest.1, + state: &state, + parentIndent: indent, + ) + } else { + try state.events.append(scalarEvent(rest.0, mark: mark(in: line, offset: rest.1), line: line.number)) + } + } + state.events.append(.sequenceEnd(mark: eventEndMark(state, fallback: startMark))) + } + + func emitMappingEvents( + _ state: inout PureYAML.Parsing.EventState, + indent: Int, + ) throws { + let startMark = state.lines[state.index].mark + state.events.append(.mappingStart(anchor: nil, tag: nil, style: .block, mark: startMark)) + try emitMappingPairEvents(&state, indent: indent) + state.events.append(.mappingEnd(mark: eventEndMark(state, fallback: startMark))) + } + + func emitMappingPairEvents( + _ state: inout PureYAML.Parsing.EventState, + indent: Int, + ) throws { + while state.index < state.lines.count { + let line = state.lines[state.index] + if line.indent < indent { + break + } + guard line.indent == indent else { + throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: line.number) + } + guard let entry = splitMappingEntryWithOffsets(line.content) else { + throw PureYAML.Parsing.ParseError.expectedMappingKey(line: line.number) + } + state.index += 1 + try emitMappingEntry(entry, line: line, baseOffset: 0, state: &state, parentIndent: indent) + } + } + + func emitInlineMappingEvents( + _ entry: PureYAML.Parsing.MappingEntry, + line: PureYAML.Parsing.Line, + baseOffset: Int, + state: inout PureYAML.Parsing.EventState, + parentIndent: Int, + ) throws { + state.events.append(.mappingStart(anchor: nil, tag: nil, style: .block, mark: mark(in: line, offset: baseOffset + entry.keyOffset))) + try emitMappingEntry(entry, line: line, baseOffset: baseOffset, state: &state, parentIndent: parentIndent) + if state.index < state.lines.count, state.lines[state.index].indent > parentIndent { + try emitMappingPairEvents(&state, indent: state.lines[state.index].indent) + } + state.events.append(.mappingEnd(mark: eventEndMark(state, fallback: line.endMark))) + } + + func emitMappingEntry( + _ entry: PureYAML.Parsing.MappingEntry, + line: PureYAML.Parsing.Line, + baseOffset: Int, + state: inout PureYAML.Parsing.EventState, + parentIndent: Int, + ) throws { + state.events.append(.scalar( + value: entry.key, + anchor: nil, + tag: nil, + style: entry.keyStyle, + mark: mark(in: line, offset: baseOffset + entry.keyOffset), + )) + if !entry.value.isEmpty { + try state.events.append(scalarEvent( + entry.value, + mark: mark(in: line, offset: baseOffset + entry.valueOffset), + line: line.number, + )) + } else if state.index < state.lines.count, state.lines[state.index].indent > parentIndent { + try emitBlockEvents(&state, indent: state.lines[state.index].indent) + } else { + state.events.append(.scalar( + value: "", + anchor: nil, + tag: nil, + style: .plain, + mark: mark(in: line, offset: baseOffset + entry.valueOffset), + )) + } + } + + func scalarEvent( + _ text: String, + mark: PureYAML.Parsing.Mark, + line: Int, + ) throws -> PureYAML.Parsing.Event { + let style = scalarStyle(text) + let value: String = switch style { + case .plain: + text + case .singleQuoted: + try parseSingleQuoted(text, line: line) + case .doubleQuoted: + try parseDoubleQuoted(text, line: line) + } + return .scalar(value: value, anchor: nil, tag: nil, style: style, mark: mark) + } + + func mark( + in line: PureYAML.Parsing.Line, + offset: Int, + ) -> PureYAML.Parsing.Mark { + let boundedOffset = max(0, min(offset, line.content.count)) + let index = line.content.index(line.content.startIndex, offsetBy: boundedOffset) + return PureYAML.Parsing.Mark( + line: line.number, + column: line.column + boundedOffset, + index: line.index + line.content[.. PureYAML.Parsing.Mark { + if state.index > 0, state.lines.indices.contains(state.index - 1) { + return state.lines[state.index - 1].endMark + } + return fallback + } +} diff --git a/Sources/Parsing/Scalar.swift b/Sources/Parsing/Scalar.swift index 3eba9f3..9ced40e 100644 --- a/Sources/Parsing/Scalar.swift +++ b/Sources/Parsing/Scalar.swift @@ -1,4 +1,14 @@ extension PureYAML.Parsing.Parser { + func scalarStyle(_ text: String) -> PureYAML.Parsing.ScalarStyle { + if text.hasPrefix("\"") { + return .doubleQuoted + } + if text.hasPrefix("'") { + return .singleQuoted + } + return .plain + } + func parseScalar( _ text: String, line: Int, diff --git a/Sources/Parsing/ScalarStyle.swift b/Sources/Parsing/ScalarStyle.swift new file mode 100644 index 0000000..a416952 --- /dev/null +++ b/Sources/Parsing/ScalarStyle.swift @@ -0,0 +1,11 @@ +extension PureYAML.Parsing { + enum ScalarStyle: String, Equatable, CustomStringConvertible { + case plain + case singleQuoted + case doubleQuoted + + var description: String { + rawValue + } + } +} diff --git a/Sources/Parsing/Text.swift b/Sources/Parsing/Text.swift index 2c4a9fa..ef8eca7 100644 --- a/Sources/Parsing/Text.swift +++ b/Sources/Parsing/Text.swift @@ -2,6 +2,7 @@ extension PureYAML.Parsing.Parser { func preprocess(_ yaml: String) throws -> [PureYAML.Parsing.Line] { var lines: [PureYAML.Parsing.Line] = [] let rawLines = yaml.split(separator: "\n", omittingEmptySubsequences: false) + var lineStartIndex = 0 for offset in rawLines.indices { let raw = String(rawLines[offset]) var indent = 0 @@ -14,23 +15,40 @@ extension PureYAML.Parsing.Parser { break } } - let content = trim(stripComment(String(raw.dropFirst(indent)))) + let textAfterIndent = String(raw.dropFirst(indent)) + let stripped = stripComment(textAfterIndent) + let (content, contentOffset) = trimWithOffset(stripped) if !content.isEmpty { - lines.append(PureYAML.Parsing.Line(number: offset + 1, indent: indent, content: content)) + lines.append(PureYAML.Parsing.Line( + number: offset + 1, + indent: indent, + column: indent + contentOffset + 1, + index: lineStartIndex + indent + String(stripped.prefix(contentOffset)).utf8.count, + content: content, + )) } + lineStartIndex += raw.utf8.count + 1 } return lines } func splitMappingEntry(_ text: String) -> (key: String, value: String)? { + guard let entry = splitMappingEntryWithOffsets(text) else { + return nil + } + return (entry.key, entry.value) + } + + func splitMappingEntryWithOffsets(_ text: String) -> PureYAML.Parsing.MappingEntry? { var inSingleQuote = false var inDoubleQuote = false var previousWasEscape = false - let characters = Array(text) - for index in characters.indices { - let character = characters[index] + var index = text.startIndex + while index < text.endIndex { + let character = text[index] if character == "\\", inDoubleQuote { previousWasEscape.toggle() + index = text.index(after: index) continue } if character == "'", !inDoubleQuote { @@ -38,19 +56,28 @@ extension PureYAML.Parsing.Parser { } else if character == "\"", !inSingleQuote, !previousWasEscape { inDoubleQuote.toggle() } else if character == ":", !inSingleQuote, !inDoubleQuote { - let next = characters.index(after: index) - if next == characters.endIndex || characters[next] == " " { - let key = trim(String(characters[.. String { + trimWithOffset(text).0 + } + + func trimWithOffset(_ text: String) -> (String, Int) { + trimWithOffset(text, in: text.startIndex ..< text.endIndex) + } + + func trimWithOffset( + _ text: String, + in range: Range, + ) -> (String, Int) { var start = text.startIndex - var end = text.endIndex + var end = range.upperBound + start = range.lowerBound while start < end, text[start].isWhitespace { start = text.index(after: start) } @@ -109,6 +148,9 @@ extension PureYAML.Parsing.Parser { } end = previous } - return String(text[start ..< end]) + return ( + String(text[start ..< end]), + text.distance(from: text.startIndex, to: start), + ) } } diff --git a/Tests/ParsingEventTests.swift b/Tests/ParsingEventTests.swift new file mode 100644 index 0000000..9849402 --- /dev/null +++ b/Tests/ParsingEventTests.swift @@ -0,0 +1,76 @@ +@testable import PureYAML +import Testing + +@Suite("Parsing Events") +struct ParsingEventTests { + @Test("Event model represents aliases anchors tags and flow style") + func eventModelRepresentsAliasesAnchorsTagsAndFlowStyle() { + let mark = PureYAML.Parsing.Mark(line: 2, column: 3, index: 12) + let events: [PureYAML.Parsing.Event] = [ + .scalar(value: "value", anchor: "root", tag: "!Thing", style: .singleQuoted, mark: mark), + .sequenceStart(anchor: "items", tag: "!seq", style: .flow, mark: mark), + .alias(anchor: "root", mark: mark), + ] + + #expect(events.map(\.description) == [ + "scalar value=\"value\" anchor=root tag=!Thing style=singleQuoted @2:3@12", + "sequenceStart anchor=items tag=!seq style=flow @2:3@12", + "alias anchor=root @2:3@12", + ]) + } + + @Test("Emits golden events for mapping scalars and scalar styles") + func emitsGoldenEventsForMappingScalarsAndScalarStyles() throws { + let events = try PureYAML.Parsing.Parser().parseEvents( + """ + name: "Example" + single: 'it''s' + missing: + """, + ) + + #expect(events.map(\.description) == [ + "streamStart @1:1@0", + "documentStart @1:1@0", + "mappingStart anchor=- tag=- style=block @1:1@0", + "scalar value=\"name\" anchor=- tag=- style=plain @1:1@0", + "scalar value=\"Example\" anchor=- tag=- style=doubleQuoted @1:7@6", + "scalar value=\"single\" anchor=- tag=- style=plain @2:1@16", + "scalar value=\"it's\" anchor=- tag=- style=singleQuoted @2:9@24", + "scalar value=\"missing\" anchor=- tag=- style=plain @3:1@32", + "scalar value=\"\" anchor=- tag=- style=plain @3:9@40", + "mappingEnd @3:9@40", + "documentEnd @3:9@40", + "streamEnd @3:9@40", + ]) + } + + @Test("Emits golden events for nested sequences and mappings") + func emitsGoldenEventsForNestedSequencesAndMappings() throws { + let events = try PureYAML.Parsing.Parser().parseEvents( + """ + servers: + - url: / + description: Default + """, + ) + + #expect(events.map(\.description) == [ + "streamStart @1:1@0", + "documentStart @1:1@0", + "mappingStart anchor=- tag=- style=block @1:1@0", + "scalar value=\"servers\" anchor=- tag=- style=plain @1:1@0", + "sequenceStart anchor=- tag=- style=block @2:3@11", + "mappingStart anchor=- tag=- style=block @2:5@13", + "scalar value=\"url\" anchor=- tag=- style=plain @2:5@13", + "scalar value=\"/\" anchor=- tag=- style=plain @2:10@18", + "scalar value=\"description\" anchor=- tag=- style=plain @3:5@24", + "scalar value=\"Default\" anchor=- tag=- style=plain @3:18@37", + "mappingEnd @3:25@44", + "sequenceEnd @3:25@44", + "mappingEnd @3:25@44", + "documentEnd @3:25@44", + "streamEnd @3:25@44", + ]) + } +} diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 4cb7c8e..cba96cd 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -39,6 +39,8 @@ PureYAML │ └── Pair ├── Parsing │ ├── Parser +│ ├── Event +│ ├── Mark │ ├── Line │ └── ParseError ├── Emitting @@ -68,6 +70,12 @@ The initial parser supports: - quoted strings - comments outside quoted strings +The parser layer also has an internal event contract. `Parsing.Event` can +represent stream, document, scalar, sequence, mapping, and alias events, with +marks plus scalar and collection styles. Anchors and tags are carried as event +metadata so the later scanner/composer work can add YAML feature coverage +without changing the event shape. + The initial dumper emits block-style YAML from the model. The initial validator traverses parsed YAML values and reports path-aware