feat(parsing): parse token stream into events

CHANGELOG.md

@@ -22,6 +22,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   comments, indentation, block entries, mapping indicators, flow delimiters,
   quoted scalars, block scalar headers, anchors, aliases, tags, and source
   marks.
+- Add the internal scanner-token event parser for block collections, flow
+  collections, aliases, anchors, tags, and line/column parser failures.
 - Expand tests into focused parsing, dumping, model, validation, and parse-error
   suites.
 - Add macOS, Linux, and WASM verification gates and CI jobs, plus a local Linux

README.md

@@ -59,7 +59,7 @@ flowchart TB
   classDef todo fill:#f2f4f7,stroke:#8e8e93,color:#111827
   Issue2["#2 Event Model and Golden Tests"]:::done
   Issue3["#3 UTF-8 Reader and Scanner"]:::done
-  Issue4["#4 Token Stream to Events"]:::todo
+  Issue4["#4 Token Stream to Events"]:::done
   Issue5["#5 Events to PureYAML Values"]:::todo
   Issue6["#6 Scalars Tags and Aliases"]:::todo
   Issue7["#7 macOS Linux and WASM Hardening"]:::todo
@@ -77,9 +77,11 @@ block sequences, ordered mappings, common scalars, quoted strings, comments, and
 a matching dumper. It also includes path-aware validation for structural YAML
 checks such as duplicate mapping keys.
 
-It is not yet a full YAML 1.2 implementation. Anchors, aliases, tags, flow
-collections, folded scalars, literal scalars, directives, and custom decoding
-are planned work.
+It is not yet a full YAML 1.2 implementation. The internal event parser now
+recognizes anchors, aliases, tags, and flow collections from scanner tokens, but
+the public value parser does not compose those events into `PureYAML.Model.Value`
+yet. Folded scalars, literal scalars, directives, and custom decoding are
+planned work.
 
 ## Attribution
 

Sources/Parsing/EventState.swift

@@ -1,7 +0,0 @@
-extension PureYAML.Parsing {
-    struct EventState {
-        var lines: [Line]
-        var index: Int
-        var events: [Event]
-    }
-}

Sources/Parsing/NodeKind.swift

@@ -0,0 +1,8 @@
+extension PureYAML.Parsing {
+    enum NodeKind: Equatable {
+        case alias
+        case mapping
+        case scalar
+        case sequence
+    }
+}

Sources/Parsing/NodeProperties.swift

@@ -0,0 +1,11 @@
+extension PureYAML.Parsing {
+    struct NodeProperties: Equatable {
+        var anchor: String?
+        var tag: String?
+        var mark: Mark?
+
+        static var none: Self {
+            Self(anchor: nil, tag: nil, mark: nil)
+        }
+    }
+}

Sources/Parsing/NodeResult.swift

@@ -0,0 +1,6 @@
+extension PureYAML.Parsing {
+    struct NodeResult: Equatable {
+        var kind: NodeKind
+        var endMark: Mark
+    }
+}

Sources/Parsing/ParseError.swift

@@ -8,8 +8,10 @@ public extension PureYAML.Parsing {
         case expectedMappingKey(line: Int)
         case expectedAnchorName(line: Int)
         case expectedAliasName(line: Int)
+        case expectedNode(line: Int, column: Int)
         case unterminatedTag(line: Int)
         case unterminatedQuotedString(line: Int)
+        case unexpectedToken(expected: String, actual: String, line: Int, column: Int)
 
         public var description: String {
             switch self {
@@ -27,10 +29,14 @@ public extension PureYAML.Parsing {
                 "expected an anchor name at line \(line)"
             case let .expectedAliasName(line):
                 "expected an alias name at line \(line)"
+            case let .expectedNode(line, column):
+                "expected a YAML node at line \(line), column \(column)"
             case let .unterminatedTag(line):
                 "unterminated tag at line \(line)"
             case let .unterminatedQuotedString(line):
                 "unterminated quoted string at line \(line)"
+            case let .unexpectedToken(expected, actual, line, column):
+                "expected \(expected) at line \(line), column \(column), found \(actual)"
             }
         }
     }

Sources/Parsing/Parser.Events.swift

@@ -1,212 +1,7 @@
 extension PureYAML.Parsing.Parser {
     func parseEvents(_ yaml: String) throws -> [PureYAML.Parsing.Event] {
-        let lines = try preprocess(yaml)
-        guard !lines.isEmpty else {
-            throw PureYAML.Parsing.ParseError.emptyDocument
-        }
-
-        var state = PureYAML.Parsing.EventState(
-            lines: lines,
-            index: 0,
-            events: [
-                .streamStart(mark: .start),
-                .documentStart(mark: .start),
-            ],
-        )
-        try emitBlockEvents(&state, indent: lines[0].indent)
-        if state.index < lines.count {
-            throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: lines[state.index].number)
-        }
-        let endMark = lines.last?.endMark ?? .start
-        state.events.append(.documentEnd(mark: endMark))
-        state.events.append(.streamEnd(mark: endMark))
-        return state.events
-    }
-}
-
-extension PureYAML.Parsing.Parser {
-    func emitBlockEvents(
-        _ state: inout PureYAML.Parsing.EventState,
-        indent: Int,
-    ) throws {
-        guard state.index < state.lines.count else {
-            state.events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: .start))
-            return
-        }
-        let line = state.lines[state.index]
-        if line.indent < indent {
-            state.events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: line.mark))
-            return
-        }
-        guard line.indent == indent else {
-            throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: line.number)
-        }
-        if line.content.hasPrefix("-") {
-            try emitSequenceEvents(&state, indent: indent)
-        } else if splitMappingEntryWithOffsets(line.content) != nil {
-            try emitMappingEvents(&state, indent: indent)
-        } else {
-            state.index += 1
-            try state.events.append(scalarEvent(line.content, mark: line.mark, line: line.number))
-        }
-    }
-
-    func emitSequenceEvents(
-        _ state: inout PureYAML.Parsing.EventState,
-        indent: Int,
-    ) throws {
-        let startMark = state.lines[state.index].mark
-        state.events.append(.sequenceStart(anchor: nil, tag: nil, style: .block, mark: startMark))
-        while state.index < state.lines.count {
-            let line = state.lines[state.index]
-            if line.indent < indent {
-                break
-            }
-            guard line.indent == indent else {
-                throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: line.number)
-            }
-            guard line.content == "-" || line.content.hasPrefix("- ") else {
-                throw PureYAML.Parsing.ParseError.mixedCollectionStyles(line: line.number)
-            }
-
-            let rest = trimWithOffset(line.content, in: line.content.index(after: line.content.startIndex) ..< line.content.endIndex)
-            state.index += 1
-            if rest.0.isEmpty {
-                if state.index < state.lines.count, state.lines[state.index].indent > indent {
-                    try emitBlockEvents(&state, indent: state.lines[state.index].indent)
-                } else {
-                    state.events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: mark(in: line, offset: rest.1)))
-                }
-            } else if let entry = splitMappingEntryWithOffsets(rest.0) {
-                try emitInlineMappingEvents(
-                    entry,
-                    line: line,
-                    baseOffset: rest.1,
-                    state: &state,
-                    parentIndent: indent,
-                )
-            } else {
-                try state.events.append(scalarEvent(rest.0, mark: mark(in: line, offset: rest.1), line: line.number))
-            }
-        }
-        state.events.append(.sequenceEnd(mark: eventEndMark(state, fallback: startMark)))
-    }
-
-    func emitMappingEvents(
-        _ state: inout PureYAML.Parsing.EventState,
-        indent: Int,
-    ) throws {
-        let startMark = state.lines[state.index].mark
-        state.events.append(.mappingStart(anchor: nil, tag: nil, style: .block, mark: startMark))
-        try emitMappingPairEvents(&state, indent: indent)
-        state.events.append(.mappingEnd(mark: eventEndMark(state, fallback: startMark)))
-    }
-
-    func emitMappingPairEvents(
-        _ state: inout PureYAML.Parsing.EventState,
-        indent: Int,
-    ) throws {
-        while state.index < state.lines.count {
-            let line = state.lines[state.index]
-            if line.indent < indent {
-                break
-            }
-            guard line.indent == indent else {
-                throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: line.number)
-            }
-            guard let entry = splitMappingEntryWithOffsets(line.content) else {
-                throw PureYAML.Parsing.ParseError.expectedMappingKey(line: line.number)
-            }
-            state.index += 1
-            try emitMappingEntry(entry, line: line, baseOffset: 0, state: &state, parentIndent: indent)
-        }
-    }
-
-    func emitInlineMappingEvents(
-        _ entry: PureYAML.Parsing.MappingEntry,
-        line: PureYAML.Parsing.Line,
-        baseOffset: Int,
-        state: inout PureYAML.Parsing.EventState,
-        parentIndent: Int,
-    ) throws {
-        state.events.append(.mappingStart(anchor: nil, tag: nil, style: .block, mark: mark(in: line, offset: baseOffset + entry.keyOffset)))
-        try emitMappingEntry(entry, line: line, baseOffset: baseOffset, state: &state, parentIndent: parentIndent)
-        if state.index < state.lines.count, state.lines[state.index].indent > parentIndent {
-            try emitMappingPairEvents(&state, indent: state.lines[state.index].indent)
-        }
-        state.events.append(.mappingEnd(mark: eventEndMark(state, fallback: line.endMark)))
-    }
-
-    func emitMappingEntry(
-        _ entry: PureYAML.Parsing.MappingEntry,
-        line: PureYAML.Parsing.Line,
-        baseOffset: Int,
-        state: inout PureYAML.Parsing.EventState,
-        parentIndent: Int,
-    ) throws {
-        state.events.append(.scalar(
-            value: entry.key,
-            anchor: nil,
-            tag: nil,
-            style: entry.keyStyle,
-            mark: mark(in: line, offset: baseOffset + entry.keyOffset),
-        ))
-        if !entry.value.isEmpty {
-            try state.events.append(scalarEvent(
-                entry.value,
-                mark: mark(in: line, offset: baseOffset + entry.valueOffset),
-                line: line.number,
-            ))
-        } else if state.index < state.lines.count, state.lines[state.index].indent > parentIndent {
-            try emitBlockEvents(&state, indent: state.lines[state.index].indent)
-        } else {
-            state.events.append(.scalar(
-                value: "",
-                anchor: nil,
-                tag: nil,
-                style: .plain,
-                mark: mark(in: line, offset: baseOffset + entry.valueOffset),
-            ))
-        }
-    }
-
-    func scalarEvent(
-        _ text: String,
-        mark: PureYAML.Parsing.Mark,
-        line: Int,
-    ) throws -> PureYAML.Parsing.Event {
-        let style = scalarStyle(text)
-        let value: String = switch style {
-        case .plain:
-            text
-        case .singleQuoted:
-            try parseSingleQuoted(text, line: line)
-        case .doubleQuoted:
-            try parseDoubleQuoted(text, line: line)
-        }
-        return .scalar(value: value, anchor: nil, tag: nil, style: style, mark: mark)
-    }
-
-    func mark(
-        in line: PureYAML.Parsing.Line,
-        offset: Int,
-    ) -> PureYAML.Parsing.Mark {
-        let boundedOffset = max(0, min(offset, line.content.count))
-        let index = line.content.index(line.content.startIndex, offsetBy: boundedOffset)
-        return PureYAML.Parsing.Mark(
-            line: line.number,
-            column: line.column + boundedOffset,
-            index: line.index + line.content[..<index].utf8.count,
-        )
-    }
-
-    func eventEndMark(
-        _ state: PureYAML.Parsing.EventState,
-        fallback: PureYAML.Parsing.Mark,
-    ) -> PureYAML.Parsing.Mark {
-        if state.index > 0, state.lines.indices.contains(state.index - 1) {
-            return state.lines[state.index - 1].endMark
-        }
-        return fallback
+        let tokens = try PureYAML.Parsing.Scanner().scan(yaml)
+        var parser = PureYAML.Parsing.TokenEventParser(tokens: tokens, scalarParser: self)
+        return try parser.parse()
     }
 }

Sources/Parsing/Scanner.swift

@@ -92,6 +92,9 @@ extension PureYAML.Parsing.Scanner {
                 state.indentation.removeLast()
                 state.append(.dedent(width: state.indentation.last ?? 0), mark: end, endMark: end)
             }
+            guard width == state.indentation.last else {
+                throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: end.line)
+            }
         }
     }
 

Sources/Parsing/TokenCursor.swift

@@ -0,0 +1,91 @@
+extension PureYAML.Parsing {
+    struct TokenCursor {
+        private var tokens: [Token]
+        private(set) var index: Int
+
+        init(tokens: [Token]) {
+            self.tokens = tokens.filter { !$0.kind.isComment }
+            index = 0
+        }
+
+        var current: Token? {
+            guard tokens.indices.contains(index) else {
+                return nil
+            }
+            return tokens[index]
+        }
+
+        var previous: Token? {
+            let previousIndex = index - 1
+            guard tokens.indices.contains(previousIndex) else {
+                return nil
+            }
+            return tokens[previousIndex]
+        }
+
+        mutating func advance() -> Token? {
+            guard let token = current else {
+                return nil
+            }
+            index += 1
+            return token
+        }
+    }
+}
+
+extension PureYAML.Parsing.TokenKind {
+    var isBlockEntry: Bool {
+        if case .blockEntry = self {
+            return true
+        }
+        return false
+    }
+
+    var isComment: Bool {
+        if case .comment = self {
+            return true
+        }
+        return false
+    }
+
+    var isDedent: Bool {
+        if case .dedent = self {
+            return true
+        }
+        return false
+    }
+
+    var isFlowTerminator: Bool {
+        switch self {
+        case .flowEntry, .flowMappingEnd, .flowSequenceEnd:
+            true
+        default:
+            false
+        }
+    }
+
+    var isIndent: Bool {
+        if case .indent = self {
+            return true
+        }
+        return false
+    }
+
+    var isMappingKey: Bool {
+        if case .mappingKey = self {
+            return true
+        }
+        return false
+    }
+
+    var isStreamEnd: Bool {
+        if case .streamEnd = self {
+            return true
+        }
+        return false
+    }
+
+    var isTerminator: Bool {
+        isDedent || isFlowTerminator || isStreamEnd
+    }
+}

Sources/Parsing/TokenEventParser.Block.swift

@@ -0,0 +1,126 @@
+extension PureYAML.Parsing.TokenEventParser {
+    mutating func parseBlockSequence(
+        properties: PureYAML.Parsing.NodeProperties,
+    ) throws -> PureYAML.Parsing.NodeResult {
+        let start = try expect("block sequence entry") { $0.isBlockEntry }
+        let startMark = properties.mark ?? start.mark
+        var endMark = start.endMark
+        events.append(.sequenceStart(
+            anchor: properties.anchor,
+            tag: properties.tag,
+            style: .block,
+            mark: startMark,
+        ))
+
+        try parseBlockSequenceItem(after: start, endMark: &endMark)
+        while cursor.current?.kind.isBlockEntry == true {
+            let entry = try expect("block sequence entry") { $0.isBlockEntry }
+            try parseBlockSequenceItem(after: entry, endMark: &endMark)
+        }
+
+        events.append(.sequenceEnd(mark: endMark))
+        return PureYAML.Parsing.NodeResult(kind: .sequence, endMark: endMark)
+    }
+
+    mutating func parseBlockSequenceItem(
+        after entry: PureYAML.Parsing.Token,
+        endMark: inout PureYAML.Parsing.Mark,
+    ) throws {
+        if hasNodeOnSameLine(after: entry) {
+            if cursor.current?.kind.isMappingKey == true {
+                endMark = try parseBlockMapping(
+                    properties: .none,
+                    includeIndentedContinuation: true,
+                ).endMark
+            } else {
+                endMark = try parseNode().endMark
+            }
+        } else if cursor.current?.kind.isIndent == true {
+            _ = try expect("indent") { $0.isIndent }
+            endMark = try parseNode().endMark
+            _ = try expect("dedent") { $0.isDedent }
+        } else {
+            let mark = entry.endMark
+            events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: mark))
+            endMark = mark
+        }
+    }
+
+    mutating func parseBlockMapping(
+        properties: PureYAML.Parsing.NodeProperties,
+        includeIndentedContinuation: Bool,
+    ) throws -> PureYAML.Parsing.NodeResult {
+        let startMark = properties.mark ?? cursor.current?.mark ?? .start
+        var endMark = startMark
+        events.append(.mappingStart(
+            anchor: properties.anchor,
+            tag: properties.tag,
+            style: .block,
+            mark: startMark,
+        ))
+        while cursor.current?.kind.isMappingKey == true {
+            endMark = try parseMappingPair().endMark
+        }
+        while includeIndentedContinuation, cursor.current?.kind.isIndent == true {
+            _ = try expect("indent") { $0.isIndent }
+            guard cursor.current?.kind.isMappingKey == true else {
+                throw unexpectedToken(expected: "mapping key")
+            }
+            while cursor.current?.kind.isMappingKey == true {
+                endMark = try parseMappingPair().endMark
+            }
+            _ = try expect("dedent") { $0.isDedent }
+        }
+        if let current = cursor.current, !canEndBlockMapping(at: current) {
+            throw PureYAML.Parsing.ParseError.unexpectedToken(
+                expected: "mapping key",
+                actual: current.kind.description,
+                line: current.mark.line,
+                column: current.mark.column,
+            )
+        }
+        events.append(.mappingEnd(mark: endMark))
+        return PureYAML.Parsing.NodeResult(kind: .mapping, endMark: endMark)
+    }
+
+    mutating func parseBlockScalar(
+        properties: PureYAML.Parsing.NodeProperties,
+    ) throws -> PureYAML.Parsing.NodeResult {
+        let header = try expect("block scalar header") { kind in
+            if case .blockScalarHeader = kind {
+                return true
+            }
+            return false
+        }
+        var lines: [String] = []
+        var endMark = header.endMark
+        if cursor.current?.kind.isIndent == true {
+            _ = try expect("indent") { $0.isIndent }
+            while cursor.current != nil, cursor.current?.kind.isDedent != true {
+                guard case let .scalar(value, style) = cursor.current?.kind else {
+                    throw unexpectedToken(expected: "block scalar content")
+                }
+                let token = cursor.advance() ?? header
+                try lines.append(decodeScalar(value, style: style, line: token.mark.line))
+                endMark = token.endMark
+            }
+            _ = try expect("dedent") { $0.isDedent }
+        }
+
+        let value: String = switch header.kind {
+        case .blockScalarHeader(.folded):
+            lines.joined(separator: " ") + (lines.isEmpty ? "" : "\n")
+        default:
+            lines.joined(separator: "\n") + (lines.isEmpty ? "" : "\n")
+        }
+        let mark = properties.mark ?? header.mark
+        events.append(.scalar(
+            value: value,
+            anchor: properties.anchor,
+            tag: properties.tag,
+            style: .plain,
+            mark: mark,
+        ))
+        return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: endMark)
+    }
+}

Sources/Parsing/TokenEventParser.Flow.swift

@@ -0,0 +1,83 @@
+extension PureYAML.Parsing.TokenEventParser {
+    mutating func parseFlowSequence(
+        properties: PureYAML.Parsing.NodeProperties,
+    ) throws -> PureYAML.Parsing.NodeResult {
+        let start = try expect("flow sequence start") { kind in
+            if case .flowSequenceStart = kind {
+                return true
+            }
+            return false
+        }
+        let startMark = properties.mark ?? start.mark
+        var endMark = start.endMark
+        events.append(.sequenceStart(
+            anchor: properties.anchor,
+            tag: properties.tag,
+            style: .flow,
+            mark: startMark,
+        ))
+        while cursor.current != nil {
+            if case .flowSequenceEnd = cursor.current?.kind {
+                let end = try expect("flow sequence end") { kind in
+                    if case .flowSequenceEnd = kind {
+                        return true
+                    }
+                    return false
+                }
+                endMark = end.endMark
+                events.append(.sequenceEnd(mark: endMark))
+                return PureYAML.Parsing.NodeResult(kind: .sequence, endMark: endMark)
+            }
+            if case .flowEntry = cursor.current?.kind {
+                _ = cursor.advance()
+                continue
+            }
+            endMark = try parseNode().endMark
+            if cursor.current?.kind.isFlowTerminator == false {
+                throw unexpectedToken(expected: "flow entry or flow sequence end")
+            }
+        }
+        throw unexpectedToken(expected: "flow sequence end")
+    }
+
+    mutating func parseFlowMapping(
+        properties: PureYAML.Parsing.NodeProperties,
+    ) throws -> PureYAML.Parsing.NodeResult {
+        let start = try expect("flow mapping start") { kind in
+            if case .flowMappingStart = kind {
+                return true
+            }
+            return false
+        }
+        let startMark = properties.mark ?? start.mark
+        var endMark = start.endMark
+        events.append(.mappingStart(
+            anchor: properties.anchor,
+            tag: properties.tag,
+            style: .flow,
+            mark: startMark,
+        ))
+        while cursor.current != nil {
+            if case .flowMappingEnd = cursor.current?.kind {
+                let end = try expect("flow mapping end") { kind in
+                    if case .flowMappingEnd = kind {
+                        return true
+                    }
+                    return false
+                }
+                endMark = end.endMark
+                events.append(.mappingEnd(mark: endMark))
+                return PureYAML.Parsing.NodeResult(kind: .mapping, endMark: endMark)
+            }
+            if case .flowEntry = cursor.current?.kind {
+                _ = cursor.advance()
+                continue
+            }
+            endMark = try parseMappingPair().endMark
+            if cursor.current?.kind.isFlowTerminator == false {
+                throw unexpectedToken(expected: "flow entry or flow mapping end")
+            }
+        }
+        throw unexpectedToken(expected: "flow mapping end")
+    }
+}

Sources/Parsing/TokenEventParser.Helpers.swift

@@ -0,0 +1,117 @@
+extension PureYAML.Parsing.TokenEventParser {
+    mutating func parseMappingPair() throws -> PureYAML.Parsing.NodeResult {
+        _ = try expect("mapping key") { $0.isMappingKey }
+        _ = try parseNode()
+        let valueIndicator = try expect("mapping value") { kind in
+            if case .mappingValue = kind {
+                return true
+            }
+            return false
+        }
+
+        if cursor.current?.kind.isIndent == true {
+            _ = try expect("indent") { $0.isIndent }
+            let result = try parseNode()
+            _ = try expect("dedent") { $0.isDedent }
+            return result
+        }
+        if hasNodeOnSameLine(after: valueIndicator) {
+            return try parseNode()
+        }
+
+        let mark = valueIndicator.endMark
+        events.append(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: mark))
+        return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: mark)
+    }
+
+    mutating func consumeProperties() -> PureYAML.Parsing.NodeProperties {
+        var properties = PureYAML.Parsing.NodeProperties.none
+        while let token = cursor.current {
+            switch token.kind {
+            case let .anchor(anchor):
+                properties.anchor = anchor
+                properties.mark = properties.mark ?? token.mark
+                _ = cursor.advance()
+            case let .tag(tag):
+                properties.tag = tag
+                properties.mark = properties.mark ?? token.mark
+                _ = cursor.advance()
+            default:
+                return properties
+            }
+        }
+        return properties
+    }
+
+    func decodeScalar(
+        _ value: String,
+        style: PureYAML.Parsing.ScalarStyle,
+        line: Int,
+    ) throws -> String {
+        switch style {
+        case .doubleQuoted:
+            try scalarParser.parseDoubleQuoted("\"\(value)\"", line: line)
+        case .plain:
+            value
+        case .singleQuoted:
+            try scalarParser.parseSingleQuoted("'\(value)'", line: line)
+        }
+    }
+
+    func hasNodeOnSameLine(after token: PureYAML.Parsing.Token) -> Bool {
+        guard let current = cursor.current, !current.kind.isTerminator else {
+            return false
+        }
+        return current.mark.line == token.mark.line
+    }
+
+    func canEndBlockMapping(at token: PureYAML.Parsing.Token) -> Bool {
+        token.kind.isBlockEntry
+            || token.kind.isDedent
+            || token.kind.isFlowTerminator
+            || token.kind.isStreamEnd
+    }
+
+    mutating func expect(
+        _ expected: String,
+        matching: (PureYAML.Parsing.TokenKind) -> Bool,
+    ) throws -> PureYAML.Parsing.Token {
+        guard let token = cursor.current else {
+            let mark = cursor.previous?.endMark ?? .start
+            throw PureYAML.Parsing.ParseError.unexpectedToken(
+                expected: expected,
+                actual: "end of tokens",
+                line: mark.line,
+                column: mark.column,
+            )
+        }
+        guard matching(token.kind) else {
+            throw PureYAML.Parsing.ParseError.unexpectedToken(
+                expected: expected,
+                actual: token.kind.description,
+                line: token.mark.line,
+                column: token.mark.column,
+            )
+        }
+        _ = cursor.advance()
+        return token
+    }
+
+    func unexpectedToken(expected: String) -> PureYAML.Parsing.ParseError {
+        guard let token = cursor.current else {
+            let mark = cursor.previous?.endMark ?? .start
+            return PureYAML.Parsing.ParseError.unexpectedToken(
+                expected: expected,
+                actual: "end of tokens",
+                line: mark.line,
+                column: mark.column,
+            )
+        }
+        return PureYAML.Parsing.ParseError.unexpectedToken(
+            expected: expected,
+            actual: token.kind.description,
+            line: token.mark.line,
+            column: token.mark.column,
+        )
+    }
+}

Sources/Parsing/TokenEventParser.swift

@@ -0,0 +1,89 @@
+extension PureYAML.Parsing {
+    struct TokenEventParser {
+        var cursor: TokenCursor
+        let scalarParser: Parser
+        var events: [Event]
+
+        init(
+            tokens: [Token],
+            scalarParser: Parser,
+        ) {
+            cursor = TokenCursor(tokens: tokens)
+            self.scalarParser = scalarParser
+            events = []
+        }
+
+        mutating func parse() throws -> [Event] {
+            let streamStart = try expect("stream start") { kind in
+                if case .streamStart = kind {
+                    return true
+                }
+                return false
+            }
+            guard cursor.current?.kind.isStreamEnd != true else {
+                throw ParseError.emptyDocument
+            }
+
+            events.append(.streamStart(mark: streamStart.mark))
+            events.append(.documentStart(mark: streamStart.mark))
+            let result = try parseNode()
+            guard cursor.current?.kind.isStreamEnd == true else {
+                throw unexpectedToken(expected: "stream end")
+            }
+            _ = try expect("stream end") { kind in
+                if case .streamEnd = kind {
+                    return true
+                }
+                return false
+            }
+            events.append(.documentEnd(mark: result.endMark))
+            events.append(.streamEnd(mark: result.endMark))
+            return events
+        }
+    }
+}
+
+extension PureYAML.Parsing.TokenEventParser {
+    mutating func parseNode() throws -> PureYAML.Parsing.NodeResult {
+        let properties = consumeProperties()
+        guard let token = cursor.current, !token.kind.isTerminator else {
+            let mark = properties.mark ?? cursor.current?.mark ?? cursor.previous?.endMark ?? .start
+            throw PureYAML.Parsing.ParseError.expectedNode(line: mark.line, column: mark.column)
+        }
+
+        switch token.kind {
+        case let .alias(anchor):
+            _ = cursor.advance()
+            let mark = properties.mark ?? token.mark
+            events.append(.alias(anchor: anchor, mark: mark))
+            return PureYAML.Parsing.NodeResult(kind: .alias, endMark: token.endMark)
+        case .blockEntry:
+            return try parseBlockSequence(properties: properties)
+        case .blockScalarHeader:
+            return try parseBlockScalar(properties: properties)
+        case .flowMappingStart:
+            return try parseFlowMapping(properties: properties)
+        case .flowSequenceStart:
+            return try parseFlowSequence(properties: properties)
+        case .mappingKey:
+            return try parseBlockMapping(
+                properties: properties,
+                includeIndentedContinuation: false,
+            )
+        case let .scalar(value, style):
+            _ = cursor.advance()
+            let decoded = try decodeScalar(value, style: style, line: token.mark.line)
+            let mark = properties.mark ?? token.mark
+            events.append(.scalar(
+                value: decoded,
+                anchor: properties.anchor,
+                tag: properties.tag,
+                style: style,
+                mark: mark,
+            ))
+            return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: token.endMark)
+        default:
+            throw PureYAML.Parsing.ParseError.expectedNode(line: token.mark.line, column: token.mark.column)
+        }
+    }
+}

Tests/ParsingEventTests.swift

@@ -75,6 +75,65 @@ struct ParsingEventTests {
         #expect(!events.map(\.description).contains("scalar value=\"\" anchor=- tag=- style=plain @1:1@0"))
     }
 
+    @Test("Emits golden events for flow collections aliases anchors and tags")
+    func test_flowCollectionsAliasesAnchorsAndTags() throws {
+        let events = try PureYAML.Parsing.Parser().parseEvents(
+            """
+            flow: [!Thing &item "value", *item, {tagged: !<tag:example.com,2026:thing> 'it''s'}]
+            """,
+        )
+
+        let descriptions = events.map(\.description)
+        #expect(descriptions == [
+            "streamStart @1:1@0",
+            "documentStart @1:1@0",
+            "mappingStart anchor=- tag=- style=block @1:1@0",
+            "scalar value=\"flow\" anchor=- tag=- style=plain @1:1@0",
+            "sequenceStart anchor=- tag=- style=flow @1:7@6",
+            "scalar value=\"value\" anchor=item tag=!Thing style=doubleQuoted @1:8@7",
+            "alias anchor=item @1:30@29",
+            "mappingStart anchor=- tag=- style=flow @1:37@36",
+            "scalar value=\"tagged\" anchor=- tag=- style=plain @1:38@37",
+            "scalar value=\"it's\" anchor=- tag=!<tag:example.com,2026:thing> style=singleQuoted @1:46@45",
+            "mappingEnd @1:84@83",
+            "sequenceEnd @1:85@84",
+            "mappingEnd @1:85@84",
+            "documentEnd @1:85@84",
+            "streamEnd @1:85@84",
+        ])
+        #expect(!descriptions.contains("sequenceStart anchor=- tag=- style=block @1:7@6"))
+        #expect(!descriptions.contains { $0.contains("it''s") })
+    }
+
+    @Test("Emits golden events for block scalar headers")
+    func test_blockScalarHeaders() throws {
+        let events = try PureYAML.Parsing.Parser().parseEvents(
+            """
+            text: |
+              one
+              two
+            folded: >
+              one
+              two
+            """,
+        )
+
+        let descriptions = events.map(\.description)
+        #expect(descriptions == [
+            "streamStart @1:1@0",
+            "documentStart @1:1@0",
+            "mappingStart anchor=- tag=- style=block @1:1@0",
+            "scalar value=\"text\" anchor=- tag=- style=plain @1:1@0",
+            "scalar value=\"one\\ntwo\\n\" anchor=- tag=- style=plain @1:7@6",
+            "scalar value=\"folded\" anchor=- tag=- style=plain @4:1@20",
+            "scalar value=\"one two\\n\" anchor=- tag=- style=plain @4:9@28",
+            "mappingEnd @6:6@41",
+            "documentEnd @6:6@41",
+            "streamEnd @6:6@41",
+        ])
+        #expect(!descriptions.contains { $0.contains("one\\ntwo") && $0.contains("@4:9@28") })
+    }
+
     @Test("Reports parse-event errors")
     func test_errorReporting() {
         expectError(PureYAML.Parsing.ParseError.emptyDocument) {
@@ -83,8 +142,39 @@ struct ParsingEventTests {
         expectError(PureYAML.Parsing.ParseError.unexpectedIndentation(line: 3)) {
             try PureYAML.Parsing.Parser().parseEvents("root:\n    child: yes\n  wrong: no")
         }
-        expectError(PureYAML.Parsing.ParseError.expectedMappingKey(line: 3)) {
+        expectError(PureYAML.Parsing.ParseError.unexpectedToken(
+            expected: "mapping key",
+            actual: "scalar value=\"dangling\" style=plain",
+            line: 3,
+            column: 3,
+        )) {
             try PureYAML.Parsing.Parser().parseEvents("root:\n  child: value\n  dangling")
         }
+        expectParseEventError(
+            "items: [one, two",
+            PureYAML.Parsing.ParseError.unexpectedToken(
+                expected: "flow entry or flow sequence end",
+                actual: "streamEnd",
+                line: 1,
+                column: 17,
+            ),
+            description: "expected flow entry or flow sequence end at line 1, column 17, found streamEnd",
+        )
+    }
+}
+
+private func expectParseEventError(
+    _ yaml: String,
+    _ expected: PureYAML.Parsing.ParseError,
+    description: String,
+) {
+    do {
+        _ = try PureYAML.Parsing.Parser().parseEvents(yaml)
+        recordIssue("expected error \(expected)")
+    } catch let error as PureYAML.Parsing.ParseError {
+        #expect(error == expected)
+        #expect(error.description == description)
+    } catch {
+        recordIssue("unexpected error \(error)")
     }
 }

Tests/ScannerTests.swift

@@ -171,6 +171,7 @@ struct ScannerTests {
     @Test("Reports scanner errors")
     func test_errorReporting() {
         expectScannerError("\tname: value", .tabIndentation(line: 1))
+        expectScannerError("root:\n    child: yes\n  wrong: no", .unexpectedIndentation(line: 3))
         expectScannerError("name: \"open", .unterminatedQuotedString(line: 1))
         expectScannerError("name: 'open", .unterminatedQuotedString(line: 1))
         expectScannerError("anchor: &", .expectedAnchorName(line: 1))

docs/DESIGN.md

@@ -42,6 +42,8 @@ PureYAML
 │   ├── Reader
 │   ├── Scanner
 │   ├── Token
+│   ├── TokenCursor
+│   ├── TokenEventParser
 │   ├── Event
 │   ├── Mark
 │   ├── Line
@@ -79,13 +81,16 @@ marks plus scalar and collection styles. Anchors and tags are carried as event
 metadata so the later scanner/composer work can add YAML feature coverage
 without changing the event shape.
 
-The parser layer now also has an internal reader/scanner contract.
+The parser layer now also has an internal reader/scanner/event contract.
 `Parsing.Reader` advances through Swift `String` input while tracking UTF-8 byte
 indexes, lines, and columns. `Parsing.Scanner` emits lexical `Parsing.Token`
 values for comments, indentation, block entries, mapping indicators, flow
 delimiters, quoted scalar starts, block scalar headers, anchors, aliases, tags,
-and source mark ranges. The scanner is not yet wired into `parse(_:)`; the next
-roadmap slice composes scanner tokens into parser events.
+and source mark ranges. `Parsing.TokenEventParser` consumes that token stream
+and emits `Parsing.Event` values for block collections, flow collections,
+aliases, anchors, tags, and scalars. The token event parser is not yet wired
+into `parse(_:)`; the next roadmap slice composes events into
+`PureYAML.Model.Value`.
 
 The initial dumper emits block-style YAML from the model.
 
@@ -97,10 +102,10 @@ blank validator plus custom rules for project-specific checks.
 
 Compatibility should be added in small, test-backed slices:
 
-1. Flow collections.
+1. Compose event streams into `PureYAML.Model.Value`.
 2. Literal and folded scalars.
-3. Anchors and aliases.
-4. Tags.
+3. Anchor and alias resolution.
+4. Tag-aware scalar and collection handling.
 5. Validation rules beyond duplicate-key behavior.
 6. Codable-style decoding and encoding.
 7. Yams corpus comparison tests in a separate compatibility suite.