diff --git a/Sources/Parsing/NodeProperties.swift b/Sources/Parsing/NodeProperties.swift deleted file mode 100644 index f53a175..0000000 --- a/Sources/Parsing/NodeProperties.swift +++ /dev/null @@ -1,17 +0,0 @@ -extension PureYAML.Parsing { - struct NodeProperties: Equatable { - var anchor: String? - var tag: String? - var mark: Mark? - - static var none: Self { - Self(anchor: nil, tag: nil, mark: nil) - } - - mutating func mergeUnset(from other: Self) { - anchor = anchor ?? other.anchor - tag = tag ?? other.tag - mark = mark ?? other.mark - } - } -} diff --git a/Sources/Parsing/NodeResult.swift b/Sources/Parsing/NodeResult.swift deleted file mode 100644 index 34333cc..0000000 --- a/Sources/Parsing/NodeResult.swift +++ /dev/null @@ -1,6 +0,0 @@ -extension PureYAML.Parsing { - struct NodeResult: Equatable { - var kind: NodeKind - var endMark: Mark - } -} diff --git a/Sources/Parsing/Scanner.BlockScalar.swift b/Sources/Parsing/Scanner.BlockScalar.swift deleted file mode 100644 index 3d932fd..0000000 --- a/Sources/Parsing/Scanner.BlockScalar.swift +++ /dev/null @@ -1,224 +0,0 @@ -extension PureYAML.Parsing.Scanner { - /// Scans a literal (`|`) or folded (`>`) block scalar in full, consuming the - /// header line and every content line, then emits a single header token whose - /// associated value is the final folded and chomped scalar text. - /// - /// This mirrors `yaml_parser_scan_block_scalar` / `yaml_parser_scan_block_scalar_breaks` - /// from the reference C scanner: block indentation auto-detection, YAML 1.2 §8.1.3 - /// line folding (single break between equally indented non-empty lines becomes a - /// space, blank lines stay literal, more-indented lines keep their breaks), tab - /// preservation inside content, and clip/strip/keep chomping. - func scanBlockScalarHeader( - _ style: PureYAML.Parsing.BlockScalarStyle, - state: inout State, - ) throws { - let start = state.reader.mark - let header = scanBlockScalarIndicators(&state) - skipBlockScalarHeaderTrailer(&state) - - let literal = style == .literal - // Mirror libyaml's `parser->indent` as the enclosing block indent: an explicit - // increment is added to `max(parser->indent, 0)` (scanner.c:2863), so a root - // block scalar (block indent -1) measures from column 0, not column -1. - let enclosingIndent = max(state.blockIndent, 0) - var indent = header.increment.map { enclosingIndent + $0 } ?? 0 - - var string = "" - var leadingBreak = "" - var trailingBreaks = "" - var leadingBlank = false - - try scanBlockScalarBreaks(indent: &indent, breaks: &trailingBreaks, state: &state) - - while blockScalarColumn(state) == indent, !state.reader.isAtEnd { - // A zero-indent block scalar (its body sits at the document root's column - // 0) ends at a `---`/`...` document marker, which closes the document the - // scalar belongs to. libyaml relies on its indent floor never reaching 0; - // PureYAML lets a root body indent to 0 so the body is captured, so the - // boundary is checked explicitly here. A marker indented past column 0 is - // ordinary content, so this only fires when the body is at the root. - if indent == 0, state.reader.mark.column == 1, isDocumentMarkerAtProbe(state.reader) { - break - } - let trailingBlank = isBlank(state.reader.peek()) - - if !literal, leadingBreak.first == "\n", !leadingBlank, !trailingBlank { - if trailingBreaks.isEmpty { - string.append(" ") - } - leadingBreak = "" - } else { - string += leadingBreak - leadingBreak = "" - } - string += trailingBreaks - trailingBreaks = "" - - leadingBlank = isBlank(state.reader.peek()) - - while let character = state.reader.peek(), !isLineBreak(character) { - string.append(character) - state.reader.advance() - } - - leadingBreak = readBlockScalarLineBreak(&state) - try scanBlockScalarBreaks(indent: &indent, breaks: &trailingBreaks, state: &state) - } - - chompBlockScalarTail( - into: &string, - chomping: header.chomping, - leadingBreak: leadingBreak, - trailingBreaks: trailingBreaks, - ) - - state.append( - .blockScalarHeader(style: style, chomping: header.chomping, value: string), - mark: start, - endMark: state.reader.mark, - ) - state.isAtLineStart = true - } - - /// Applies clip/strip/keep chomping to the accumulated content tail. - private func chompBlockScalarTail( - into string: inout String, - chomping: PureYAML.Parsing.BlockScalarChomping, - leadingBreak: String, - trailingBreaks: String, - ) { - switch chomping { - case .strip: - break - case .clip: - // Clip keeps a single trailing line break. When the final content line is - // not terminated by a break (end of input), the line is still logically - // line-terminated, so a single newline is appended for non-empty content. - if !leadingBreak.isEmpty { - string += leadingBreak - } else if !string.isEmpty { - string += "\n" - } - case .keep: - string += leadingBreak - string += trailingBreaks - } - } - - private func scanBlockScalarIndicators( - _ state: inout State, - ) -> (chomping: PureYAML.Parsing.BlockScalarChomping, increment: Int?) { - var chomping = PureYAML.Parsing.BlockScalarChomping.clip - var increment: Int? - state.reader.advance() - while let character = state.reader.peek(), isBlockScalarHeaderCharacter(character) { - if character == "-" { - chomping = .strip - } else if character == "+" { - chomping = .keep - } else if let digit = character.wholeNumberValue, digit > 0 { - increment = digit - } - state.reader.advance() - } - return (chomping, increment) - } - - /// Eats trailing blanks and an optional comment up to the end of the header line, - /// then consumes the header's line break. - private func skipBlockScalarHeaderTrailer(_ state: inout State) { - while isBlank(state.reader.peek()) { - state.reader.advance() - } - if state.reader.peek() == "#" { - while let character = state.reader.peek(), !isLineBreak(character) { - state.reader.advance() - } - } - if isLineBreak(state.reader.peek()) { - state.reader.advance() - } - } - - /// Mirrors `yaml_parser_scan_block_scalar_breaks`: skips indentation spaces up to - /// `indent` columns (auto-detecting `indent` from the deepest blank-line indentation - /// when it has not been set), collecting the line breaks of empty lines into `breaks`. - /// - /// Unlike the reference C scanner, this re-runs the surrounding indentation logic on - /// the line where it stops, so it must not leave the reader partway through a - /// non-content line's leading whitespace. It therefore only consumes the block - /// indentation of the stopping line when that line actually belongs to the block - /// (its indentation reaches `indent`); otherwise it restores the reader to the line - /// start so the outer scanner re-reads the line's indentation. - private func scanBlockScalarBreaks( - indent: inout Int, - breaks: inout String, - state: inout State, - ) throws { - var maxIndent = 0 - while true { - let lineStart = state.reader - while indent == 0 || blockScalarColumn(state) < indent { - guard state.reader.peek() == " " else { - break - } - state.reader.advance() - } - try rejectBlockScalarIndentationTab(indent: indent, state: state) - if blockScalarColumn(state) > maxIndent { - maxIndent = blockScalarColumn(state) - } - guard isLineBreak(state.reader.peek()) else { - if indent > 0, blockScalarColumn(state) < indent { - state.reader = lineStart - } - break - } - breaks += readBlockScalarLineBreak(&state) - } - if indent == 0 { - // The auto-detected content indent is at least `parser->indent + 1` - // (scanner.c:3014). At the bare document root the block indent is -1, so - // the floor is 0 and a column-0 body is captured as content (its end is a - // document marker, handled in the content loop); inside any collection the - // floor is at least 1, matching libyaml's additional `< 1` clamp. - indent = max(maxIndent, state.blockIndent + 1) - } - } - - /// Rejects a tab where a block-scalar indentation space is expected, matching - /// scanner.c:2993-2997 ("found a tab character where an indentation space is - /// expected"). The tab abuses indentation only when it would itself supply the - /// block's indent: with a fixed indent, that is any tab below the content column; - /// while auto-detecting (`indent == 0`), a tab at column 0 (no space established a - /// content column) is illegal, but a tab after one or more leading spaces is the - /// first content character of an indented line, so the spaces fix the indent and - /// the tab is preserved as content. - private func rejectBlockScalarIndentationTab(indent: Int, state: State) throws { - guard state.reader.peek() == "\t" else { - return - } - let column = blockScalarColumn(state) - let abusesIndentation = indent > 0 ? column < indent : column == 0 - if abusesIndentation { - throw PureYAML.Parsing.ParseError.tabIndentation(line: state.reader.mark.line) - } - } - - /// Reads a single line break, normalizing CR and CRLF to a single `\n`. - private func readBlockScalarLineBreak(_ state: inout State) -> String { - guard isLineBreak(state.reader.peek()) else { - return "" - } - state.reader.advance() - return "\n" - } - - private func blockScalarColumn(_ state: State) -> Int { - state.reader.mark.column - 1 - } - - private func isBlank(_ character: Character?) -> Bool { - character == " " || character == "\t" - } -} diff --git a/Sources/Parsing/Scanner.Directives.swift b/Sources/Parsing/Scanner.Directives.swift deleted file mode 100644 index c94ce59..0000000 --- a/Sources/Parsing/Scanner.Directives.swift +++ /dev/null @@ -1,248 +0,0 @@ -extension PureYAML.Parsing.Scanner { - func scanDirectiveOrDocumentMarker(_ state: inout State) throws -> Bool { - if state.flowDepth > 0, state.reader.mark.column == 1, isDocumentMarkerHere(state) { - // A `---` or `...` document marker at column 0 is recognized regardless - // of flow nesting (scanner.c:908-924 keys only on `mark.column == 0`), - // but a flow collection may not span a document boundary. libyaml reports - // the unterminated flow at the parser layer; reject it here. - throw PureYAML.Parsing.ParseError.unexpectedToken( - expected: "flow collection close before document marker", - actual: "document marker", - line: state.reader.mark.line, - column: state.reader.mark.column, - ) - } - guard state.indentation.last == 0, state.flowDepth == 0 else { - return false - } - if state.reader.peek() == "%" { - // A directive section is only valid at the start of the stream or right - // after a `...` document-end suffix. Once a document has started (via - // `---`), a `%YAML`/`%TAG` line that itself introduces a new explicit - // document (its next line is a `---`/`...` marker) begins a second - // directive section that the YAML grammar's l-document-suffix rule - // forbids without an intervening `...`. This is the MUS6/01 case - // (`%YAML`/`---`/`%YAML`/`---`), distinct from a `%`-prefixed line that - // is part of the current document body. - if state.hasDocumentStartMarker, isDirectiveLineHere(state), directiveLineIntroducesDocument(state) { - throw PureYAML.Parsing.ParseError.directiveWithoutDocument(line: state.reader.mark.line) - } - // `%` introduces a directive only in directive position: before any - // document content and before the `---` that starts a document. After - // content, after `---`, or anywhere inside a node, `%` is plain text, - // so defer to ordinary token scanning by returning false. This mirrors - // the C scanner, which only enters `yaml_parser_scan_directive` from - // the start-of-line directive context, never inside a node or scalar. - guard !state.hasDocumentContent, !state.hasDocumentStartMarker else { - return false - } - try scanDirective(&state) - return true - } - if state.reader.peek() == "#" { - return false - } - if startsWith("---", state.reader), isIndicatorBoundary(state.reader.peek(offset: 3)) { - scanDocumentMarker(.documentStart, state: &state) - resetForDocumentStart(&state) - return true - } - if startsWith("...", state.reader), isIndicatorBoundary(state.reader.peek(offset: 3)) { - // A `...` end marker terminates the directive section it follows just - // as a `---` would, except that no document was started, so a pending - // directive here was never attached to a document and is rejected. - if state.hasPendingDirective { - throw PureYAML.Parsing.ParseError.directiveWithoutDocument(line: state.reader.mark.line) - } - scanDocumentMarker(.documentEnd, state: &state) - resetForDocumentEnd(&state) - return true - } - return false - } - - /// True when, after the directive line under the reader, the following line at - /// column 0 is a `---`/`...` document marker, i.e. the directive line opens a new - /// explicit document. Used to tell a genuine second directive section (MUS6/01) - /// apart from a `%`-prefixed line that belongs to the current document body. - func directiveLineIntroducesDocument(_ state: State) -> Bool { - var probe = state.reader - while let character = probe.peek(), !isLineBreak(character) { - probe.advance() - } - guard isLineBreak(probe.peek()) else { - return false - } - probe.advance() - guard probe.mark.column == 1 else { - return false - } - return (startsWith3(probe, "-") || startsWith3(probe, ".")) - && isIndicatorBoundary(probe.peek(offset: 3)) - } - - private func startsWith3(_ reader: PureYAML.Parsing.Reader, _ character: Character) -> Bool { - reader.peek(offset: 0) == character - && reader.peek(offset: 1) == character - && reader.peek(offset: 2) == character - } - - /// True when the reader sits on a recognized directive line (`%YAML` or `%TAG`). - /// Reserved/unknown `%...` directives are not treated as a hard error here; only - /// the two grammar-significant directives gate the document-suffix rule. - func isDirectiveLineHere(_ state: State) -> Bool { - startsWith("%YAML", state.reader) || startsWith("%TAG", state.reader) - } - - /// True when the reader sits on a `---` or `...` document marker, i.e. three - /// identical marker characters followed by a blank, line break, or end of input. - func isDocumentMarkerHere(_ state: State) -> Bool { - (startsWith("---", state.reader) || startsWith("...", state.reader)) - && isIndicatorBoundary(state.reader.peek(offset: 3)) - } - - func scanDocumentMarker( - _ kind: PureYAML.Parsing.TokenKind, - state: inout State, - ) { - let start = state.reader.mark - state.reader.advance() - state.reader.advance() - state.reader.advance() - state.append(kind, mark: start, endMark: state.reader.mark) - // A `---` start marker may be followed by the document's root node on the - // same line (`--- foo`, `--- |`, `--- [1, 2]`), matching libyaml, which scans - // the marker as a token and keeps scanning the line. Skip the inline spaces - // and, when real content follows, hand it to the ordinary token scanner; a - // trailing comment or empty remainder is skipped as before. A `...` end - // marker carries no node, so its line (only an optional comment) is skipped. - guard case .documentStart = kind else { - skipLine(&state) - return - } - skipSeparation(&state) - guard let character = state.reader.peek(), !isLineBreak(character), character != "#" else { - skipLine(&state) - return - } - state.isAtLineStart = false - } - - func resetForDocumentStart(_ state: inout State) { - state.hasDocumentContent = false - state.hasDocumentStartMarker = true - state.isDocumentClosed = false - state.sawYAMLDirective = false - state.hasPendingDirective = false - resetBlockIndent(&state) - } - - func resetForDocumentEnd(_ state: inout State) { - state.hasDocumentContent = false - state.hasDocumentStartMarker = false - state.isDocumentClosed = true - state.sawYAMLDirective = false - state.hasPendingDirective = false - resetBlockIndent(&state) - } - - /// Mirror libyaml unrolling the block indent to -1 at a document boundary so a - /// new document's root scalar starts with no enclosing block collection. - func resetBlockIndent(_ state: inout State) { - state.blockIndent = -1 - state.blockIndents.removeAll(keepingCapacity: true) - } - - func scanDirective(_ state: inout State) throws { - let start = state.reader.mark - let line = state.reader.consume { character in - !isLineBreak(character) - } - let fields = directiveFields(line) - guard let name = fields.first else { - return - } - switch name { - case "%YAML": - try scanYAMLDirective(fields: fields, start: start, state: &state) - case "%TAG": - scanTagDirective(fields: fields, state: &state) - default: - // Reserved or unknown directives are ignored with a warning per the - // spec; PureYAML accepts and discards them rather than erroring. - break - } - state.hasPendingDirective = true - } - - /// Splits a directive line into its `%name` and parameter fields, dropping - /// any trailing comment. Fields are separated by spaces or tabs, matching the - /// C scanner's blank handling around directive values. - private func directiveFields(_ line: String) -> [String] { - var fields: [String] = [] - var current = "" - for character in line { - // A `#` at a field boundary (i.e. preceded by whitespace or the line - // start) begins a trailing comment that ends the directive's fields. - if character == "#", current.isEmpty { - break - } - if character == " " || character == "\t" { - if !current.isEmpty { - fields.append(current) - current = "" - } - continue - } - current.append(character) - } - if !current.isEmpty { - fields.append(current) - } - return fields - } - - private func scanYAMLDirective( - fields: [String], - start: PureYAML.Parsing.Mark, - state: inout State, - ) throws { - // A `%YAML` directive carries exactly one version field of the form - // `major.minor`. A duplicate within one document, extra trailing fields, - // or an unsupported major version is an error; a higher 1.x minor is - // accepted (treated as 1.2 with a warning) so future minors keep parsing. - guard !state.sawYAMLDirective else { - throw PureYAML.Parsing.ParseError.incompatibleYAMLDirective(line: start.line) - } - guard fields.count == 2, isSupportedYAMLVersion(fields[1]) else { - throw PureYAML.Parsing.ParseError.incompatibleYAMLDirective(line: start.line) - } - state.sawYAMLDirective = true - } - - private func isSupportedYAMLVersion(_ version: String) -> Bool { - let components = version.split(separator: ".", omittingEmptySubsequences: false) - guard components.count == 2 else { - return false - } - let major = components[0] - let minor = components[1] - guard !major.isEmpty, !minor.isEmpty else { - return false - } - guard major.allSatisfy(\.isNumber), minor.allSatisfy(\.isNumber) else { - return false - } - return major == "1" - } - - private func scanTagDirective( - fields: [String], - state: inout State, - ) { - guard fields.count >= 3 else { - return - } - state.tagHandles[fields[1]] = fields[2] - } -} diff --git a/Sources/Parsing/Scanner.FlowContext.swift b/Sources/Parsing/Scanner.FlowContext.swift deleted file mode 100644 index d86bb80..0000000 --- a/Sources/Parsing/Scanner.FlowContext.swift +++ /dev/null @@ -1,22 +0,0 @@ -extension PureYAML.Parsing { - enum ScannerFlowContextKind { - case sequence - case mapping - } - - struct ScannerFlowContext { - var kind: ScannerFlowContextKind - var isExpectingKey: Bool - } - - /// A position that may be promoted to a flow-mapping key, mirroring libyaml's - /// `yaml_simple_key_t`. When a `:` value indicator is later found, the scanner - /// inserts a ``TokenKind/mappingKey`` token at ``tokenIndex`` so a node of any - /// kind (plain or quoted scalar, or a nested `[...]`/`{...}` collection) that - /// precedes the `:` becomes that pair's key. Only used inside flow collections; - /// block context promotes keys with a forward lookahead instead. - struct ScannerFlowSimpleKey { - var tokenIndex: Int - var mark: PureYAML.Parsing.Mark - } -} diff --git a/Sources/Parsing/Scanner.FlowSimpleKey.swift b/Sources/Parsing/Scanner.FlowSimpleKey.swift deleted file mode 100644 index a15e99b..0000000 --- a/Sources/Parsing/Scanner.FlowSimpleKey.swift +++ /dev/null @@ -1,105 +0,0 @@ -extension PureYAML.Parsing.Scanner { - func scanFlowToken( - _ character: Character, - state: inout State, - ) -> Bool { - switch character { - case "[": - openFlowCollection(kind: .sequence, token: .flowSequenceStart, state: &state) - return true - case "]": - closeFlowCollection(token: .flowSequenceEnd, state: &state) - return true - case "{": - openFlowCollection(kind: .mapping, token: .flowMappingStart, state: &state) - return true - case "}": - closeFlowCollection(token: .flowMappingEnd, state: &state) - return true - case ",": - // A flow entry separator abandons any pending simple key and reopens the - // door for a fresh one on the next entry (libyaml fetch_flow_entry). - removeFlowSimpleKey(&state) - scanFixedToken(.flowEntry, state: &state) - state.simpleKeyAllowed = true - if state.flowContexts.last?.kind == .mapping { - state.flowContexts[state.flowContexts.count - 1].isExpectingKey = true - } - return true - default: - return false - } - } - - /// Opens a flow collection (`[` or `{`). The opener itself may be a simple key - /// at the enclosing level, so a key is saved before the flow level rises; a - /// fresh simple-key slot is then pushed and a key is allowed inside. - func openFlowCollection( - kind: PureYAML.Parsing.ScannerFlowContextKind, - token: PureYAML.Parsing.TokenKind, - state: inout State, - ) { - saveFlowSimpleKey(&state) - state.flowDepth += 1 - state.flowContexts.append(.init(kind: kind, isExpectingKey: kind == .mapping)) - state.flowSimpleKeys.append(nil) - scanFixedToken(token, state: &state) - state.simpleKeyAllowed = true - } - - /// Closes a flow collection (`]` or `}`): the pending simple key on this level is - /// dropped, the level is popped, and no simple key may immediately follow the - /// closing indicator (libyaml fetch_flow_collection_end). - func closeFlowCollection( - token: PureYAML.Parsing.TokenKind, - state: inout State, - ) { - removeFlowSimpleKey(&state) - state.flowDepth = max(0, state.flowDepth - 1) - scanFixedToken(token, state: &state) - _ = state.flowContexts.popLast() - _ = state.flowSimpleKeys.popLast() - state.simpleKeyAllowed = false - } - - /// Records a potential simple key at the current reader position for the - /// innermost flow level, if a simple key is allowed there. Mirrors libyaml's - /// `yaml_parser_save_simple_key`: the saved key remembers the index of the - /// next token to be appended, so a later `:` can splice a ``mappingKey`` in - /// front of the node that starts here. No-op outside flow context. - func saveFlowSimpleKey(_ state: inout State) { - guard state.flowDepth > 0, state.simpleKeyAllowed, !state.flowSimpleKeys.isEmpty else { - return - } - state.flowSimpleKeys[state.flowSimpleKeys.count - 1] = PureYAML.Parsing.ScannerFlowSimpleKey( - tokenIndex: state.tokens.count, - mark: state.reader.mark, - ) - } - - /// Discards any potential simple key on the innermost flow level. Mirrors - /// libyaml's `yaml_parser_remove_simple_key` for the flow context, where a - /// simple key is never *required*, so abandoning it is not an error. - func removeFlowSimpleKey(_ state: inout State) { - guard !state.flowSimpleKeys.isEmpty else { - return - } - state.flowSimpleKeys[state.flowSimpleKeys.count - 1] = nil - } - - /// Promotes a pending flow simple key to an actual mapping key by inserting a - /// ``TokenKind/mappingKey`` token at the saved index, mirroring libyaml's - /// `fetch_value` splicing a KEY token before a simple key. Returns true when a - /// key was inserted so the caller can avoid emitting a redundant one. - @discardableResult - func promoteFlowSimpleKey(_ state: inout State) -> Bool { - guard !state.flowSimpleKeys.isEmpty, let key = state.flowSimpleKeys[state.flowSimpleKeys.count - 1] else { - return false - } - let token = PureYAML.Parsing.Token(kind: .mappingKey, mark: key.mark, endMark: key.mark) - let index = min(key.tokenIndex, state.tokens.count) - state.tokens.insert(token, at: index) - state.flowSimpleKeys[state.flowSimpleKeys.count - 1] = nil - return true - } -} diff --git a/Sources/Parsing/Scanner.Helpers.swift b/Sources/Parsing/Scanner.Helpers.swift deleted file mode 100644 index 9f46146..0000000 --- a/Sources/Parsing/Scanner.Helpers.swift +++ /dev/null @@ -1,345 +0,0 @@ -extension PureYAML.Parsing.Scanner { - func startsWith( - _ prefix: String, - _ reader: PureYAML.Parsing.Reader, - ) -> Bool { - for (offset, character) in prefix.enumerated() { - guard reader.peek(offset: offset) == character else { - return false - } - } - return true - } - - func isLineBreak(_ character: Character?) -> Bool { - character == "\n" || character == "\r" || character == "\r\n" - } - - func isBlockEntry(_ reader: PureYAML.Parsing.Reader) -> Bool { - reader.peek() == "-" && isIndicatorBoundary(reader.peek(offset: 1)) - } - - func isIndicatorBoundary(_ character: Character?) -> Bool { - guard let character else { - return true - } - // libyaml's IS_BLANKZ: an indicator (`-`, `?`, `:`) is recognized when it - // is followed by a space, tab, line break, or end of input. A trailing tab - // is then rejected separately because it may not provide indentation. - return character == " " || character == "\t" || isLineBreak(character) - } - - func isMappingValueBoundary(_ character: Character?) -> Bool { - guard let character else { - return true - } - return character == " " - || character == "\t" - || isLineBreak(character) - || character == "," - || character == "]" - || character == "}" - } - - func isBlockMappingValueBoundary(_ character: Character?) -> Bool { - guard let character else { - return true - } - // A `:` is a block mapping value indicator when followed by IS_BLANKZ - // (space, tab, line break, or end of input), matching libyaml's - // IS_BLANKZ_AT check in yaml_parser_fetch_value. - return character == " " || character == "\t" || isLineBreak(character) - } - - func isTokenTerminator(_ character: Character) -> Bool { - character == " " - || isLineBreak(character) - || character == "," - || character == "[" - || character == "]" - || character == "{" - || character == "}" - || character == "#" - } - - /// True for characters that may appear inside an anchor or alias name, - /// matching libyaml's `IS_ALPHA`: ASCII letters, digits, `-`, and `_`. - /// The name terminates at the first character outside this set, so a - /// trailing `:`, flow indicator, or whitespace ends the name rather than - /// being consumed into it. - func isAnchorNameCharacter(_ character: Character) -> Bool { - character.isASCII - && (character.isLetter || character.isNumber || character == "-" || character == "_") - } - - func canStartAnchorName(_ character: Character?) -> Bool { - guard let character else { - return false - } - return !isTokenTerminator(character) - && character != "*" - && character != "&" - && character != "!" - && character != "\"" - && character != "'" - } - - func hasNodeOnSameLine(afterCurrentPositionIn reader: PureYAML.Parsing.Reader) -> Bool { - var offset = 0 - while reader.peek(offset: offset) == " " { - offset += 1 - } - guard let character = reader.peek(offset: offset) else { - return false - } - return !isLineBreak(character) && character != "#" - } - - func isFlowDelimiter(_ character: Character) -> Bool { - character == "," - || character == "[" - || character == "]" - || character == "{" - || character == "}" - } - - func isBlockScalarHeaderCharacter(_ character: Character) -> Bool { - character == "+" - || character == "-" - || character.isNumber - } - - func trimTrailingSpaces(_ value: String) -> String { - var end = value.endIndex - while end > value.startIndex { - let previous = value.index(before: end) - guard value[previous] == " " else { - break - } - end = previous - } - return String(value[.. String { - var value = firstLine - while true { - var probe = state.reader - guard isLineBreak(probe.peek()) else { - return value - } - - probe.advance() - let blankLineCount = skipBlankContinuationLines(&probe) - - _ = probe.consume { character in - character == " " - } - let indentationWidth = probe.mark.column - 1 - - guard !probe.isAtEnd, !isLineBreak(probe.peek()), probe.peek() != "#" else { - return value - } - - // In flow context a plain scalar ends when its next line begins with a - // flow indicator (`,` `]` `}`) or the `:` value indicator. The line break - // that would otherwise fold to a space is trailing whitespace and is - // dropped (YAML 1.2 section 7.3.3), so return the value without folding. - if state.flowDepth > 0, let next = probe.peek(), next == "," || next == "]" || next == "}" || next == ":" { - return value - } - - // libyaml ends a plain scalar at a `---` or `...` document marker that - // begins at column 0 and is followed by a blank, line break, or end of - // input (scanner.c:3425-3433). A root scalar now folds column-0 lines, so - // this boundary must be checked explicitly rather than relying on indent. - if probe.mark.column == 1, isDocumentMarkerAtProbe(probe) { - return value - } - - let baseIndentation = plainScalarContinuationBaseIndentation(state: state) - guard indentationWidth > baseIndentation else { - return value - } - if isBlockEntry(probe), state.validImplicitIndentation.contains(indentationWidth) { - return value - } - - var continuationState = state - continuationState.reader = probe - let continuation = scanPlainScalarContinuationLine(&continuationState) - // In block context a continuation line that ends at a `:` is the next - // mapping key, not part of this scalar, so stop folding. In flow context - // a multi-line plain key folds in full and the trailing `:` then makes - // the folded text the key (libyaml folds first, promotes the key after). - guard state.flowDepth > 0 || !isAtMappingValueIndicator(state: continuationState) else { - return value - } - - state = continuationState - if blankLineCount == 0 { - value += " " - } else { - value += String(repeating: "\n", count: blankLineCount) - } - value += continuation - } - } - - /// Advances `probe` over any wholly blank lines that follow the current line - /// break, returning how many were skipped so the caller can fold them to that - /// many newlines. Leaves the probe at the start of the next non-blank line. - func skipBlankContinuationLines(_ probe: inout PureYAML.Parsing.Reader) -> Int { - var blankLineCount = 0 - while true { - var blankProbe = probe - _ = blankProbe.consume { character in - character == " " - } - guard isLineBreak(blankProbe.peek()) else { - break - } - blankProbe.advance() - probe = blankProbe - blankLineCount += 1 - } - return blankLineCount - } - - /// True when the reader sits on a `---` or `...` document marker: three identical - /// marker characters followed by a blank, line break, or end of input. - func isDocumentMarkerAtProbe(_ reader: PureYAML.Parsing.Reader) -> Bool { - (startsWith("---", reader) || startsWith("...", reader)) - && isIndicatorBoundary(reader.peek(offset: 3)) - } - - func plainScalarContinuationBaseIndentation(state: State) -> Int { - // libyaml folds a plain-scalar continuation line that is indented more than - // the enclosing block collection (`parser->indent + 1`). At the bare document - // root the block indent is -1, so a column-0 continuation folds; inside a - // collection it is the collection's column, so a sibling line at the same - // indentation ends the scalar instead of folding. - state.blockIndent - } - - func scanPlainScalarContinuationLine(_ state: inout State) -> String { - var value = "" - while let character = state.reader.peek(), !isLineBreak(character) { - if character == " ", state.reader.peek(offset: 1) == "#" { - break - } - if character == "#", value.isEmpty || value.last == " " { - break - } - if character == ":", shouldEndPlainScalarAtMappingValue(state: state) { - break - } - if state.flowDepth > 0, isFlowDelimiter(character) { - break - } - value.append(character) - state.reader.advance() - } - return trimTrailingSpaces(value) - } - - func skipLine(_ state: inout State) { - _ = state.reader.consume { character in - !isLineBreak(character) - } - } - - func expandTag( - _ tag: String, - state: State, - ) -> String { - let handles = state.tagHandles.keys.sorted { first, second in - first.count > second.count - } - for handle in handles where tag.hasPrefix(handle) { - let suffix = tag.dropFirst(handle.count) - return (state.tagHandles[handle] ?? handle) + suffix - } - return tag - } - - func shouldEndPlainScalar( - _ character: Character, - state: State, - accumulated: String, - ) -> Bool { - if isLineBreak(character) { - return true - } - if character == " ", state.reader.peek(offset: 1) == "#" { - return true - } - if character == "#", accumulated.isEmpty || accumulated.last == " " { - return true - } - if character == ":", shouldEndPlainScalarAtMappingValue(state: state) { - return true - } - if state.flowDepth > 0, isFlowDelimiter(character) { - return true - } - return false - } - - func shouldEndPlainScalarAtMappingValue(state: State) -> Bool { - if state.flowDepth > 0 { - return state.isExpectingFlowMappingKey - || isMappingValueBoundary(state.reader.peek(offset: 1)) - } - return isBlockMappingValueBoundary(state.reader.peek(offset: 1)) - } - - func isAtMappingValueIndicator(state: State) -> Bool { - state.reader.peek() == ":" && shouldEndPlainScalarAtMappingValue(state: state) - } - - /// True when, after the just-scanned key scalar, optional inline spaces are - /// followed by a `:` mapping-value indicator. A YAML implicit key may carry - /// trailing spaces before its `:` (`"top1" :`), so the key probe looks past - /// those spaces while the value indicator itself is scanned later, after - /// `skipSeparation`. Only safe for already-terminated scalars (quoted or - /// explicit-`?` keys); plain scalars consume their own trailing spaces. - func isAtSpacedMappingValueIndicator(state: State) -> Bool { - var offset = 0 - while state.reader.peek(offset: offset) == " " { - offset += 1 - } - guard state.reader.peek(offset: offset) == ":" else { - return false - } - let following = state.reader.peek(offset: offset + 1) - if state.flowDepth > 0 { - return state.isExpectingFlowMappingKey || isMappingValueBoundary(following) - } - return isBlockMappingValueBoundary(following) - } - - @discardableResult - func appendMappingKeyIfNeeded( - start: PureYAML.Parsing.Mark, - state: inout State, - ) -> Bool { - // Flow-context keys are promoted by the simple-key mechanism (a `:` value - // indicator splices the `mappingKey` in retroactively), so the block-only - // forward lookahead must not also fire here. - guard state.flowDepth == 0 else { - return false - } - guard isAtMappingValueIndicator(state: state) || isAtSpacedMappingValueIndicator(state: state) else { - return false - } - // Mirror libyaml's `fetch_value` rolling the block mapping indent to the - // implicit key's column when the simple key is promoted to a mapping key. - rollBlockIndent(toColumn: start.column - 1, state: &state) - state.append(.mappingKey, mark: start, endMark: start) - return true - } -} diff --git a/Sources/Parsing/Scanner.Indentation.swift b/Sources/Parsing/Scanner.Indentation.swift deleted file mode 100644 index bd289b8..0000000 --- a/Sources/Parsing/Scanner.Indentation.swift +++ /dev/null @@ -1,132 +0,0 @@ -extension PureYAML.Parsing.Scanner { - func scanIndentation(_ state: inout State) throws { - let start = state.reader.mark - let width = try scanIndentationWidth(&state) - guard shouldProcessIndentation(state) else { - return - } - - // Mirror libyaml unrolling the block indent to the line's column before any - // token on the line is scanned, so a dedented line restores the enclosing - // collection's indent. A continuation of a plain scalar never reaches here - // (it is consumed inside the scalar scanner), so this only fires at genuine - // block-line starts. - unrollBlockIndent(toColumn: width, state: &state) - - let end = state.reader.mark - let current = state.indentation.last ?? 0 - - if width > current { - applyIncreasedIndentation(width: width, start: start, end: end, state: &state) - } else if width < current { - try applyDedentation(width: width, start: start, end: end, state: &state) - } - - pruneImplicitIndentation(&state) - } - - func scanIndentationWidth(_ state: inout State) throws -> Int { - var width = 0 - while state.reader.peek() == " " { - width += 1 - state.reader.advance() - } - // Block indentation (s-indent) is spaces only: a tab is never indentation - // (scanner.c's scan_to_next_token eats only spaces unconditionally). The tab - // is left under the reader so the token scanner can decide whether it is legal - // in-line separation (before a scalar value or flow node) or an illegal tab - // that supplies the indentation of a nested block node; `rejectTabSeparation` - // and the blank-line handling in `scan` make that call. A whitespace-only line - // (spaces and/or tabs followed by a break) is a blank line and carries no - // indentation token. - return width - } - - func shouldProcessIndentation(_ state: State) -> Bool { - guard !state.reader.isAtEnd, !isLineBreak(state.reader.peek()) else { - return false - } - // A leading '#' starts a comment line, which must not drive indent/dedent - // processing. Block scalar content (including lines that begin with '#') is - // consumed in full by the block scalar scanner before control returns here, - // so this check no longer needs to special-case block scalars. - if state.reader.peek() == "#" { - return false - } - return state.flowDepth == 0 - } - - func applyIncreasedIndentation( - width: Int, - start: PureYAML.Parsing.Mark, - end: PureYAML.Parsing.Mark, - state: inout State, - ) { - state.indentation.append(width) - state.append(.indent(width: width), mark: start, endMark: end) - } - - func applyDedentation( - width: Int, - start: PureYAML.Parsing.Mark, - end: PureYAML.Parsing.Mark, - state: inout State, - ) throws { - while state.indentation.count > 1, width < (state.indentation.last ?? 0) { - state.indentation.removeLast() - state.append(.dedent(width: state.indentation.last ?? 0), mark: end, endMark: end) - } - if canUseImplicitIndentation(width: width, state: state) { - state.indentation.append(width) - state.append(.indent(width: width), mark: start, endMark: end) - } - guard width == state.indentation.last else { - throw PureYAML.Parsing.ParseError.unexpectedIndentation(line: end.line) - } - } - - func canUseImplicitIndentation( - width: Int, - state: State, - ) -> Bool { - width != state.indentation.last - && width > (state.indentation.last ?? 0) - && state.validImplicitIndentation.contains(width) - } - - func pruneImplicitIndentation(_ state: inout State) { - let current = state.indentation.last ?? 0 - state.validImplicitIndentation = state.validImplicitIndentation.filter { $0 <= current } - } - - /// Mirrors libyaml's `yaml_parser_roll_indent`: when a block collection opens at - /// `column` (0-based) deeper than the current block indent, push the old indent - /// and adopt the new one. No-op in flow context, matching the C scanner. - func rollBlockIndent(toColumn column: Int, state: inout State) { - guard state.flowDepth == 0, state.blockIndent < column else { - return - } - state.blockIndents.append(state.blockIndent) - state.blockIndent = column - } - - /// Mirrors libyaml's `yaml_parser_unroll_indent`: pop block indents until the - /// current block indent is at or below `column`. Called at each block-context - /// line start so the block indent tracks the line's actual nesting. - func unrollBlockIndent(toColumn column: Int, state: inout State) { - guard state.flowDepth == 0 else { - return - } - while state.blockIndent > column { - state.blockIndent = state.blockIndents.popLast() ?? -1 - } - } - - func closeIndentation(_ state: inout State) { - let mark = state.reader.mark - while state.indentation.count > 1 { - state.indentation.removeLast() - state.append(.dedent(width: state.indentation.last ?? 0), mark: mark, endMark: mark) - } - } -} diff --git a/Sources/Parsing/Scanner.PropertyTokens.swift b/Sources/Parsing/Scanner.PropertyTokens.swift deleted file mode 100644 index e1b6ce7..0000000 --- a/Sources/Parsing/Scanner.PropertyTokens.swift +++ /dev/null @@ -1,56 +0,0 @@ -extension PureYAML.Parsing.Scanner { - func scanNamedToken( - kind: (String) -> PureYAML.Parsing.TokenKind, - missingNameError: (Int) -> PureYAML.Parsing.ParseError, - state: inout State, - ) throws { - let start = state.reader.mark - // An anchor or alias may itself begin a flow simple key, so record the - // position before the token is appended; the following node then does not - // start a new key, leaving `&anchor key :` anchored at the `&` (libyaml's - // fetch_anchor saves the simple key before clearing simple_key_allowed). - saveFlowSimpleKey(&state) - state.reader.advance() - // An anchor or alias name is a run of name characters that ends at the - // first non-name character, mirroring `yaml_parser_scan_anchor`: the - // name must stop at whitespace, `:`, `?`, flow indicators, and the other - // indicator characters so that `*alias:` reads the alias `alias` - // followed by a `:` mapping indicator rather than swallowing the colon. - let name = state.reader.consume { character in - isAnchorNameCharacter(character) - } - guard !name.isEmpty else { - throw missingNameError(start.line) - } - state.append(kind(name), mark: start, endMark: state.reader.mark) - } - - func scanTag(_ state: inout State) throws { - let start = state.reader.mark - // A tag may itself begin a flow simple key (see `scanNamedToken`). - saveFlowSimpleKey(&state) - state.reader.advance() - var value = "!" - if state.reader.peek() == "<" { - value.append(state.reader.advance() ?? "<") - var didClose = false - while let character = state.reader.peek() { - value.append(character) - state.reader.advance() - if character == ">" { - didClose = true - break - } - } - guard didClose else { - throw PureYAML.Parsing.ParseError.unterminatedTag(line: start.line) - } - } else { - value += state.reader.consume { character in - !isTokenTerminator(character) - } - value = expandTag(value, state: state) - } - state.append(.tag(value), mark: start, endMark: state.reader.mark) - } -} diff --git a/Sources/Parsing/Scanner.Step.swift b/Sources/Parsing/Scanner.Step.swift deleted file mode 100644 index 4f8812c..0000000 --- a/Sources/Parsing/Scanner.Step.swift +++ /dev/null @@ -1,85 +0,0 @@ -extension PureYAML.Parsing.Scanner { - /// Result of one scanning iteration over the source. Both the batch ``scan`` - /// loop and the incremental ``PureYAML/Parsing/TokenCursor`` driver share the - /// same step so the two paths cannot diverge (a divergence is what once let a - /// tab-indentation error slip past the streaming parser while the batch scanner - /// caught it). - enum Step { - /// Nothing remains; the caller appends stream end and stops. - case endOfInput - /// A line break, blank line, or comment was consumed; restart the loop. - case advanced - /// A directive or document marker was scanned; restart the loop. - case scannedMarker - /// An ordinary token was scanned. - case scannedToken - } - - /// Performs one scanning iteration: consumes a leading line break or blank line, - /// scans indentation at a line start, then scans either a directive / document - /// marker or one ordinary token. Resets `simpleKeyAllowed` at every block-context - /// line break so a tab at the next line start is recognized as (illegal) - /// indentation rather than (legal) in-line separation. - func scanStep(_ state: inout State) throws -> Step { - if state.reader.isAtEnd { - return .endOfInput - } - if isLineBreak(state.reader.peek()) { - state.reader.advance() - state.isAtLineStart = true - if state.flowDepth == 0 { - state.simpleKeyAllowed = true - } - return .advanced - } - - if state.isAtLineStart { - // A line whose entire content is blanks (spaces and/or tabs) before the - // break is a blank line: it carries no indentation and no token, matching - // scanner.c's scan_to_next_token, which eats the blanks and the break - // before any indent processing. Consuming it here keeps a whitespace-only - // line (e.g. `· \t`) from emitting a spurious indent token or being - // mis-scanned as content. - if consumeBlankLineRemainder(&state) { - return .advanced - } - try scanIndentation(&state) - if state.reader.isAtEnd || isLineBreak(state.reader.peek()) { - return .advanced - } - if try scanDirectiveOrDocumentMarker(&state) { - return .scannedMarker - } - state.isAtLineStart = false - } - - try scanToken(&state) - return .scannedToken - } - - /// When the reader sits on a run of blanks (spaces and/or tabs) that ends the - /// line (a line break or end of input follows), consumes that run plus the line - /// break and reports true: the line was blank. Otherwise leaves the reader where - /// it was and reports false. Used so a whitespace-only line (e.g. `· \t`) is - /// skipped rather than mis-scanned as plain-scalar content. - func consumeBlankLineRemainder(_ state: inout State) -> Bool { - var offset = 0 - while let character = state.reader.peek(offset: offset), character == " " || character == "\t" { - offset += 1 - } - let next = state.reader.peek(offset: offset) - guard next == nil || isLineBreak(next) else { - return false - } - for _ in 0 ..< offset { - state.reader.advance() - } - if isLineBreak(state.reader.peek()) { - state.reader.advance() - if state.flowDepth == 0 { - state.simpleKeyAllowed = true - } - } - return true - } -} diff --git a/Sources/Parsing/Scanner.Tabs.swift b/Sources/Parsing/Scanner.Tabs.swift deleted file mode 100644 index a4078c0..0000000 --- a/Sources/Parsing/Scanner.Tabs.swift +++ /dev/null @@ -1,62 +0,0 @@ -extension PureYAML.Parsing.Scanner { - /// Rejects a tab that appears in separation position only when it supplies - /// indentation for a nested block node. YAML 1.2 allows a tab as in-line - /// separation (s-white) before a plain or quoted scalar value, but never as - /// indentation (s-indent). The two are distinguished by what follows the tab: - /// a block-entry `-`, an explicit-key `?`, or a scalar that is itself a mapping - /// key (a compact nested mapping) all start a nested block node whose column the - /// tab would be indenting, so they are rejected; a plain scalar *value* is - /// ordinary separation and is allowed. - /// - /// Examples (yaml-test-suite): `--1` and `foo: bar` are valid (scalar - /// value follows), while `--`, `?key:`, and `:key:` are rejected - /// (a nested indicator or key follows). - func rejectTabSeparation(_ state: inout State) throws { - guard state.flowDepth == 0, state.simpleKeyAllowed, state.reader.peek() == "\t" else { - return - } - if tabSeparationPrecedesNestedBlockNode(state) { - throw PureYAML.Parsing.ParseError.tabIndentation(line: state.reader.mark.line) - } - } - - /// True when, after the tab(s) and spaces under the reader, the next token starts - /// a nested block node (a `-` block entry, a `?` explicit key, or a plain scalar - /// immediately followed by a `:` mapping-value indicator). - private func tabSeparationPrecedesNestedBlockNode(_ state: State) -> Bool { - var offset = 0 - while state.reader.peek(offset: offset) == "\t" || state.reader.peek(offset: offset) == " " { - offset += 1 - } - guard let character = state.reader.peek(offset: offset) else { - return false - } - if character == "-", isIndicatorBoundary(state.reader.peek(offset: offset + 1)) { - return true - } - if character == "?", isIndicatorBoundary(state.reader.peek(offset: offset + 1)) { - return true - } - return plainScalarIsMappingKey(in: state, startingAt: offset) - } - - /// Scans a prospective plain scalar starting at `offset` (in lookahead only) and - /// reports whether it is immediately followed by a `:` block mapping-value - /// indicator, i.e. the scalar is a nested mapping key rather than a value. - private func plainScalarIsMappingKey(in state: State, startingAt offset: Int) -> Bool { - var cursor = offset - while let character = state.reader.peek(offset: cursor) { - if isLineBreak(character) { - return false - } - if character == ":", isBlockMappingValueBoundary(state.reader.peek(offset: cursor + 1)) { - return true - } - if character == " ", state.reader.peek(offset: cursor + 1) == "#" { - return false - } - cursor += 1 - } - return false - } -} diff --git a/Sources/Parsing/Scanner.swift b/Sources/Parsing/Scanner.swift deleted file mode 100644 index 65d8cd4..0000000 --- a/Sources/Parsing/Scanner.swift +++ /dev/null @@ -1,357 +0,0 @@ -extension PureYAML.Parsing { - struct Scanner { - init() {} - - func scan(_ yaml: String) throws -> [Token] { - var state = State(reader: Reader(yaml)) - state.append(.streamStart) - - loop: while !state.reader.isAtEnd { - switch try scanStep(&state) { - case .endOfInput: - break loop - case .advanced, .scannedMarker, .scannedToken: - continue - } - } - - if state.hasPendingDirective { - throw PureYAML.Parsing.ParseError.directiveWithoutDocument(line: state.reader.mark.line) - } - closeIndentation(&state) - state.append(.streamEnd) - return state.tokens - } - } -} - -extension PureYAML.Parsing.Scanner { - struct State { - var reader: PureYAML.Parsing.Reader - var tokens: [PureYAML.Parsing.Token] = [] - var indentation: [Int] = [0] - var isAtLineStart = true - var hasDocumentContent = false - var hasDocumentStartMarker = false - var isDocumentClosed = false - /// True once a `%YAML` directive has been scanned for the document that - /// the next `---` will start. A second `%YAML` before that `---` is a - /// duplicate and is rejected; the flag resets at every document boundary. - var sawYAMLDirective = false - /// True once any directive has been scanned in the current directive - /// section. A directive section must be terminated by a `---` document - /// start; reaching a `...` end marker or the end of the stream with a - /// directive still pending is an error. - var hasPendingDirective = false - var flowDepth = 0 - /// Mirrors libyaml's `parser->indent`: the 0-based column of the innermost - /// open block collection (sequence or mapping), or -1 at the bare document - /// root. Plain-scalar continuation folding compares a continuation line's - /// indentation against this value, so a root scalar (block indent -1) folds - /// column-0 continuations while a scalar inside a collection only folds lines - /// indented past the collection. Tracked separately from ``indentation`` -- - /// which records every line-indentation level -- because libyaml rolls this - /// only on block-collection indicators, not on every indentation increase. - var blockIndent = -1 - /// Saved ``blockIndent`` values, mirroring libyaml's `parser->indents` stack, - /// so a dedent restores the enclosing collection's indent. - var blockIndents: [Int] = [] - /// Mirrors libyaml's `simple_key_allowed`. In block context a tab may only - /// be consumed as whitespace when a simple key is NOT allowed (i.e. mid-line, - /// after a node has been emitted). It is true at the start of every block - /// line and immediately after a `-`, `?`, or `:` indicator, so a tab in those - /// positions provides block indentation and is rejected. See scanner.c:1948. - var simpleKeyAllowed = true - var flowContexts: [PureYAML.Parsing.ScannerFlowContext] = [] - /// A stack of potential simple keys, one entry per open flow collection, - /// mirroring libyaml's `simple_keys`. The slot for the innermost flow level - /// records the most recent position a `:` could promote to a mapping key. - /// Empty outside flow context; block keys do not use this stack. - var flowSimpleKeys: [PureYAML.Parsing.ScannerFlowSimpleKey?] = [] - var validImplicitIndentation: Set = [] - var tagHandles = [ - "!": "!", - "!!": "tag:yaml.org,2002:", - ] - - mutating func append(_ kind: PureYAML.Parsing.TokenKind) { - let mark = reader.mark - tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: mark)) - markDocumentContentIfNeeded(kind) - } - - mutating func append( - _ kind: PureYAML.Parsing.TokenKind, - mark: PureYAML.Parsing.Mark, - endMark: PureYAML.Parsing.Mark, - ) { - tokens.append(PureYAML.Parsing.Token(kind: kind, mark: mark, endMark: endMark)) - markDocumentContentIfNeeded(kind) - } - - mutating func markDocumentContentIfNeeded(_ kind: PureYAML.Parsing.TokenKind) { - switch kind { - case .alias, .anchor, .blockScalarHeader, .scalar, .tag: - // A node (or one of its properties) has appeared on this line, so a - // simple key is no longer allowed and a following tab is ordinary - // content rather than indentation. Mirrors libyaml setting - // simple_key_allowed = 0 after scanning these tokens. - hasDocumentContent = true - simpleKeyAllowed = false - case .comment, .dedent, .documentEnd, .documentStart, .indent, .streamEnd, .streamStart: - return - default: - hasDocumentContent = true - } - } - - var isExpectingFlowMappingKey: Bool { - flowContexts.last?.kind == .mapping - && flowContexts.last?.isExpectingKey == true - } - } -} - -extension PureYAML.Parsing.Scanner { - func scanToken(_ state: inout State) throws { - skipSpaceSeparation(&state) - try rejectTabSeparation(&state) - skipSeparation(&state) - guard let character = state.reader.peek(), !isLineBreak(character) else { - return - } - - if try scanLineIndicatorToken(character, state: &state) { - return - } - if scanFlowToken(character, state: &state) { - return - } - if try scanQuotedOrBlockScalarToken(character, state: &state) { - return - } - if try scanNodePropertyToken(character, state: &state) { - return - } - scanPlainScalar(&state) - } - - func scanLineIndicatorToken( - _ character: Character, - state: inout State, - ) throws -> Bool { - switch character { - case "#": - scanComment(&state) - return true - case "-" where isBlockEntry(state.reader): - scanFixedToken(.blockEntry, state: &state) - if state.flowDepth == 0 { - state.simpleKeyAllowed = true - } - return true - case "?" where state.flowDepth > 0 || isIndicatorBoundary(state.reader.peek(offset: 1)): - // In flow context a `?` is always an explicit-key indicator (libyaml's - // fetch_more_tokens checks `flow_level || IS_BLANKZ_AT`); a pending - // simple key is abandoned and no implicit key may follow on this entry. - if state.flowDepth > 0 { - removeFlowSimpleKey(&state) - } - scanFixedToken(.mappingKey, state: &state) - if state.flowDepth == 0 { - state.simpleKeyAllowed = true - } else { - state.simpleKeyAllowed = false - } - return true - case ":" where state.flowDepth > 0 || isAtMappingValueIndicator(state: state): - // In flow context a `:` is always a value indicator regardless of what - // follows it (libyaml), so `"k":v` and `{k}:v` are key/value pairs. A - // pending simple key is spliced into a `mappingKey` before its node. - if state.flowDepth > 0 { - promoteFlowSimpleKey(&state) - } - scanFixedToken(.mappingValue, state: &state) - if state.flowDepth == 0 { - state.simpleKeyAllowed = true - } else { - state.simpleKeyAllowed = false - } - return true - default: - return false - } - } - - func scanNodePropertyToken( - _ character: Character, - state: inout State, - ) throws -> Bool { - switch character { - case "&": - try scanNamedToken( - kind: PureYAML.Parsing.TokenKind.anchor, - missingNameError: PureYAML.Parsing.ParseError.expectedAnchorName, - state: &state, - ) - return true - case "*": - try scanNamedToken( - kind: PureYAML.Parsing.TokenKind.alias, - missingNameError: PureYAML.Parsing.ParseError.expectedAliasName, - state: &state, - ) - return true - case "!": - try scanTag(&state) - return true - default: - return false - } - } - - func scanQuotedOrBlockScalarToken( - _ character: Character, - state: inout State, - ) throws -> Bool { - switch character { - case "\"": - try scanQuotedScalar(.doubleQuoted, state: &state) - return true - case "'": - try scanQuotedScalar(.singleQuoted, state: &state) - return true - case "|": - try scanBlockScalarHeader(.literal, state: &state) - return true - case ">": - try scanBlockScalarHeader(.folded, state: &state) - return true - default: - return false - } - } - - func skipSpaceSeparation(_ state: inout State) { - while state.reader.peek() == " " { - state.reader.advance() - } - } - - func skipSeparation(_ state: inout State) { - // In-line separation (s-white) is spaces and tabs. `rejectTabSeparation` has - // already rejected any tab that would supply the indentation of a nested block - // node, so a tab still under the reader here is ordinary separation before a - // scalar value or flow node and is consumed like a space (scanner.c IS_BLANK). - while let character = state.reader.peek(), character == " " || character == "\t" { - state.reader.advance() - } - } - - func scanFixedToken( - _ kind: PureYAML.Parsing.TokenKind, - state: inout State, - ) { - let start = state.reader.mark - // Mirror libyaml rolling the block indent when a block sequence entry (`-`) - // or an explicit `?` mapping key opens a collection at this column. The - // implicit-key case rolls in `appendMappingKeyIfNeeded` at the key column; - // the `:` value indicator does not roll again, because for a simple key the - // mapping indent was already set at the key's (shallower) column and rolling - // to the deeper `:` column would wrongly raise the continuation threshold. - switch kind { - case .blockEntry, .mappingKey: - rollBlockIndent(toColumn: start.column - 1, state: &state) - default: - break - } - state.reader.advance() - state.append(kind, mark: start, endMark: state.reader.mark) - if case .blockEntry = kind, hasNodeOnSameLine(afterCurrentPositionIn: state.reader) { - state.validImplicitIndentation.insert(start.column + 1) - } - if case .mappingValue = kind, state.flowContexts.last?.kind == .mapping { - state.flowContexts[state.flowContexts.count - 1].isExpectingKey = false - } - } - - func scanComment(_ state: inout State) { - let start = state.reader.mark - state.reader.advance() - if state.reader.peek() == " " { - state.reader.advance() - } - let value = state.reader.consume { character in - !isLineBreak(character) - } - state.append(.comment(value), mark: start, endMark: state.reader.mark) - } - - func scanQuotedScalar( - _ style: PureYAML.Parsing.ScalarStyle, - state: inout State, - ) throws { - let start = state.reader.mark - // A quoted scalar may be a flow simple key: record its position before the - // content is consumed so a following `:` can splice a key in front of it. - saveFlowSimpleKey(&state) - let quote: Character = style == .doubleQuoted ? "\"" : "'" - state.reader.advance() - var value = "" - var previousWasEscape = false - while let character = state.reader.peek() { - // A `---` or `...` document marker at column 0 cannot appear inside an - // open quoted scalar: it means the scalar was never closed before a - // document boundary (scanner.c:3058-3070). Reject it as unterminated. - if state.reader.mark.column == 1, isDocumentMarkerHere(state) { - throw PureYAML.Parsing.ParseError.unterminatedQuotedString(line: start.line) - } - if style == .singleQuoted, character == quote, state.reader.peek(offset: 1) == quote { - value.append(character) - state.reader.advance() - value.append(character) - state.reader.advance() - continue - } - if character == quote, !previousWasEscape { - break - } - value.append(character) - previousWasEscape = character == "\\" && style == .doubleQuoted && !previousWasEscape - state.reader.advance() - if character != "\\" { - previousWasEscape = false - } - } - guard state.reader.peek() == quote else { - throw PureYAML.Parsing.ParseError.unterminatedQuotedString(line: start.line) - } - state.reader.advance() - appendMappingKeyIfNeeded(start: start, state: &state) - state.append(.scalar(value: value, style: style), mark: start, endMark: state.reader.mark) - } - - func scanPlainScalar(_ state: inout State) { - let start = state.reader.mark - // A plain scalar may be a flow simple key: record its position before the - // content is consumed so a following `:` can splice a key in front of it. - saveFlowSimpleKey(&state) - var value = "" - while let character = state.reader.peek() { - if shouldEndPlainScalar(character, state: state, accumulated: value) { - break - } - value.append(character) - state.reader.advance() - } - var trimmed = trimTrailingSpaces(value) - let isMappingKey = appendMappingKeyIfNeeded(start: start, state: &state) - if !isMappingKey { - trimmed = scanPlainScalarContinuations( - after: trimmed, - state: &state, - ) - } - state.append(.scalar(value: trimmed, style: .plain), mark: start, endMark: state.reader.mark) - } -} diff --git a/Sources/Parsing/Token.swift b/Sources/Parsing/Token.swift deleted file mode 100644 index ec6c049..0000000 --- a/Sources/Parsing/Token.swift +++ /dev/null @@ -1,108 +0,0 @@ -extension PureYAML.Parsing { - struct Token: Equatable, CustomStringConvertible { - var kind: TokenKind - var mark: Mark - var endMark: Mark - - var description: String { - "\(kind) @\(mark)..\(endMark)" - } - } -} - -extension PureYAML.Parsing { - enum TokenKind: Equatable { - case streamStart - case streamEnd - case documentStart - case documentEnd - case indent(width: Int) - case dedent(width: Int) - case blockEntry - case mappingKey - case mappingValue - case flowSequenceStart - case flowSequenceEnd - case flowMappingStart - case flowMappingEnd - case flowEntry - case scalar(value: String, style: ScalarStyle) - case blockScalarHeader(style: BlockScalarStyle, chomping: BlockScalarChomping, value: String) - case anchor(String) - case alias(String) - case tag(String) - case comment(String) - } -} - -extension PureYAML.Parsing.TokenKind: CustomStringConvertible { - var description: String { - switch self { - case .streamStart: - "streamStart" - case .streamEnd: - "streamEnd" - case .documentStart: - "documentStart" - case .documentEnd: - "documentEnd" - case let .indent(width): - "indent width=\(width)" - case let .dedent(width): - "dedent width=\(width)" - case .blockEntry: - "blockEntry" - case .mappingKey: - "mappingKey" - case .mappingValue: - "mappingValue" - case .flowSequenceStart: - "flowSequenceStart" - case .flowSequenceEnd: - "flowSequenceEnd" - case .flowMappingStart: - "flowMappingStart" - case .flowMappingEnd: - "flowMappingEnd" - case .flowEntry: - "flowEntry" - case let .scalar(value, style): - "scalar value=\"\(escape(value))\" style=\(style)" - case let .blockScalarHeader(style, chomping, value): - if chomping == .clip { - "blockScalarHeader style=\(style) value=\"\(escape(value))\"" - } else { - "blockScalarHeader style=\(style) chomping=\(chomping) value=\"\(escape(value))\"" - } - case let .anchor(value): - "anchor name=\(value)" - case let .alias(value): - "alias name=\(value)" - case let .tag(value): - "tag value=\(value)" - case let .comment(value): - "comment value=\"\(escape(value))\"" - } - } - - private func escape(_ value: String) -> String { - var output = "" - for character in value { - switch character { - case "\\": - output += "\\\\" - case "\"": - output += "\\\"" - case "\n": - output += "\\n" - case "\r": - output += "\\r" - case "\t": - output += "\\t" - default: - output.append(character) - } - } - return output - } -} diff --git a/Sources/Parsing/TokenCursor.swift b/Sources/Parsing/TokenCursor.swift deleted file mode 100644 index 62f632e..0000000 --- a/Sources/Parsing/TokenCursor.swift +++ /dev/null @@ -1,218 +0,0 @@ -extension PureYAML.Parsing { - private final class TokenScannerStateBox { - var state: Scanner.State - - init(state: Scanner.State) { - self.state = state - } - } - - struct TokenCursor { - private var tokens: [Token] - private(set) var index: Int - private var previousToken: Token? - private var scanner: Scanner? - private var scannerBox: TokenScannerStateBox? - - init(tokens: [Token]) { - self.tokens = tokens.filter { !$0.kind.isComment } - index = 0 - previousToken = nil - scanner = nil - scannerBox = nil - } - - init(yaml: String, scanner: Scanner = .init()) { - var state = Scanner.State(reader: Reader(yaml)) - state.append(.streamStart) - tokens = state.tokens.filter { !$0.kind.isComment } - state.tokens.removeAll(keepingCapacity: true) - index = 0 - previousToken = nil - self.scanner = scanner - scannerBox = TokenScannerStateBox(state: state) - } - - var current: Token? { - guard tokens.indices.contains(index) else { - return nil - } - return tokens[index] - } - - var previous: Token? { - previousToken - } - - mutating func peek(offset: Int = 1) throws -> Token? { - try ensureAvailable(offset: offset) - let targetIndex = index + offset - guard tokens.indices.contains(targetIndex) else { - return nil - } - return tokens[targetIndex] - } - - mutating func advance() throws -> Token? { - try ensureAvailable(offset: 0) - guard tokens.indices.contains(index) else { - return nil - } - let token = tokens[index] - previousToken = token - index += 1 - compactConsumedTokens() - try ensureAvailable(offset: 0) - return token - } - - private mutating func ensureAvailable(offset: Int) throws { - while tokens.count <= index + offset, scannerBox != nil { - try scanMoreTokens() - } - } - - private mutating func scanMoreTokens() throws { - guard let scanner, let scannerBox else { - return - } - - // Scan at least one token, and keep scanning without draining while a - // flow collection is open: a flow `:` may splice a `mappingKey` into an - // already-scanned earlier token (the simple-key mechanism), so the whole - // flow collection's tokens must stay in `state.tokens` until it closes - // for the retroactive index to remain valid. - while scannerBox.state.tokens.isEmpty || scannerBox.state.flowDepth > 0 { - if scannerBox.state.reader.isAtEnd { - if scannerBox.state.hasPendingDirective { - throw PureYAML.Parsing.ParseError.directiveWithoutDocument( - line: scannerBox.state.reader.mark.line, - ) - } - scanner.closeIndentation(&scannerBox.state) - scannerBox.state.append(.streamEnd) - drainScannedTokens(from: &scannerBox.state) - self.scannerBox = nil - return - } - - // Drive the same scanning step the batch scanner uses, so the two - // paths share one source of truth for line breaks, blank lines, - // indentation, and `simpleKeyAllowed` handling. - let step = try scanner.scanStep(&scannerBox.state) - if case .scannedMarker = step, !scannerBox.state.tokens.isEmpty { - break - } - } - - drainScannedTokens(from: &scannerBox.state) - } - - private mutating func drainScannedTokens(from state: inout Scanner.State) { - tokens.append(contentsOf: state.tokens.filter { !$0.kind.isComment }) - state.tokens.removeAll(keepingCapacity: true) - } - - private mutating func compactConsumedTokens() { - guard index > 4096 else { - return - } - tokens.removeFirst(index) - index = 0 - } - } -} - -extension PureYAML.Parsing.TokenKind { - var isAlias: Bool { - if case .alias = self { - return true - } - return false - } - - var isNodeProperty: Bool { - switch self { - case .anchor, .tag: - true - default: - false - } - } - - var isBlockEntry: Bool { - if case .blockEntry = self { - return true - } - return false - } - - var isComment: Bool { - if case .comment = self { - return true - } - return false - } - - var isDedent: Bool { - if case .dedent = self { - return true - } - return false - } - - var isDocumentEnd: Bool { - if case .documentEnd = self { - return true - } - return false - } - - var isDocumentStart: Bool { - if case .documentStart = self { - return true - } - return false - } - - var isFlowTerminator: Bool { - switch self { - case .flowEntry, .flowMappingEnd, .flowSequenceEnd: - true - default: - false - } - } - - var isIndent: Bool { - if case .indent = self { - return true - } - return false - } - - var isMappingKey: Bool { - if case .mappingKey = self { - return true - } - return false - } - - var isMappingValue: Bool { - if case .mappingValue = self { - return true - } - return false - } - - var isStreamEnd: Bool { - if case .streamEnd = self { - return true - } - return false - } - - var isTerminator: Bool { - isDedent || isDocumentEnd || isDocumentStart || isFlowTerminator || isStreamEnd - } -} diff --git a/Sources/Parsing/TokenEventParser.Block.swift b/Sources/Parsing/TokenEventParser.Block.swift deleted file mode 100644 index 013c121..0000000 --- a/Sources/Parsing/TokenEventParser.Block.swift +++ /dev/null @@ -1,231 +0,0 @@ -extension PureYAML.Parsing.TokenEventParser { - mutating func parseBlockSequence( - properties: PureYAML.Parsing.NodeProperties, - allowMappingKeyTerminator: Bool = false, - ) throws -> PureYAML.Parsing.NodeResult { - let start = try expect("block sequence entry") { $0.isBlockEntry } - let startMark = properties.mark ?? start.mark - var endMark = start.endMark - try emit(.sequenceStart( - anchor: properties.anchor, - tag: properties.tag, - style: .block, - mark: startMark, - )) - - try parseBlockSequenceItem( - after: start, - endMark: &endMark, - allowMappingKeyTerminator: allowMappingKeyTerminator, - ) - try consumeSequenceItemBoundaryDedent(sequenceIndentWidth: start.mark.column - 1) - try consumeSequenceItemBoundaryIndent(sequenceIndentWidth: start.mark.column - 1) - while cursor.current?.kind.isBlockEntry == true { - let entry = try expect("block sequence entry") { $0.isBlockEntry } - try parseBlockSequenceItem( - after: entry, - endMark: &endMark, - allowMappingKeyTerminator: allowMappingKeyTerminator, - ) - try consumeSequenceItemBoundaryDedent(sequenceIndentWidth: entry.mark.column - 1) - try consumeSequenceItemBoundaryIndent(sequenceIndentWidth: entry.mark.column - 1) - } - if let current = cursor.current, current.kind.isMappingKey, !allowMappingKeyTerminator { - throw PureYAML.Parsing.ParseError.mixedCollectionStyles(line: current.mark.line) - } - - try emit(.sequenceEnd(mark: endMark)) - return PureYAML.Parsing.NodeResult(kind: .sequence, endMark: endMark) - } - - mutating func consumeSequenceItemBoundaryDedent(sequenceIndentWidth: Int) throws { - guard dedentWidth(cursor.current) == sequenceIndentWidth, - try cursor.peek()?.kind.isBlockEntry == true - else { - return - } - _ = try expect("dedent") { $0.isDedent } - } - - mutating func consumeSequenceItemBoundaryIndent(sequenceIndentWidth: Int) throws { - guard indentWidth(cursor.current) == sequenceIndentWidth, - try cursor.peek()?.kind.isBlockEntry == true - else { - return - } - _ = try expect("indent") { $0.isIndent } - } - - mutating func parseBlockSequenceItem( - after entry: PureYAML.Parsing.Token, - endMark: inout PureYAML.Parsing.Mark, - allowMappingKeyTerminator: Bool, - ) throws { - if hasNodeOnSameLine(after: entry) { - let properties = try consumeProperties() - if cursor.current?.kind.isIndent == true { - _ = try expect("indent") { $0.isIndent } - endMark = try parseNode(properties: properties).endMark - _ = try expect("dedent") { $0.isDedent } - return - } - // After consuming anchor/tag properties the only content remaining on - // this entry's line may be gone: a `mappingKey` or sibling `-` now on - // the same line opens a nested collection, but one on a later line is - // the next sibling, leaving this entry an empty scalar that carries - // the properties. - if cursor.current?.kind.isMappingKey == true, hasNodeOnSameLine(after: entry) { - endMark = try parseBlockMapping( - properties: properties, - includeIndentedContinuation: true, - minimumColumn: entry.mark.column + 1, - ).endMark - } else if cursor.current?.kind.isBlockEntry == true, hasNodeOnSameLine(after: entry) { - endMark = try parseBlockSequence( - properties: properties, - allowMappingKeyTerminator: allowMappingKeyTerminator, - ).endMark - } else if canBeginNode(cursor.current), hasNodeOnSameLine(after: entry) { - endMark = try parseNode(properties: properties).endMark - } else { - let mark = properties.mark ?? entry.endMark - try emitEmptyScalar(properties: properties, mark: mark) - endMark = mark - } - } else if cursor.current?.kind.isIndent == true { - _ = try expect("indent") { $0.isIndent } - endMark = try parseNode().endMark - _ = try expect("dedent") { $0.isDedent } - } else { - let mark = entry.endMark - try emit(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: mark)) - endMark = mark - } - } - - mutating func parseBlockMapping( - properties: PureYAML.Parsing.NodeProperties, - includeIndentedContinuation: Bool, - minimumColumn: Int? = nil, - firstKeyProperties: PureYAML.Parsing.NodeProperties = .none, - ) throws -> PureYAML.Parsing.NodeResult { - let startMark = firstKeyProperties.mark ?? properties.mark ?? cursor.current?.mark ?? .start - let tokenColumn = cursor.current?.mark.column ?? startMark.column - let contentColumn = min(startMark.column, tokenColumn) - let boundaryColumn = minimumColumn ?? contentColumn - var endMark = startMark - try emit(.mappingStart( - anchor: properties.anchor, - tag: properties.tag, - style: .block, - mark: startMark, - )) - - try parseBlockMappingPairs( - boundaryColumn: boundaryColumn, - includeIndentedContinuation: includeIndentedContinuation, - endMark: &endMark, - firstKeyProperties: firstKeyProperties, - ) - if let current = cursor.current, !canEndBlockMapping(at: current, minimumColumn: boundaryColumn) { - throw PureYAML.Parsing.ParseError.unexpectedToken( - expected: "mapping key", - actual: current.kind.description, - line: current.mark.line, - column: current.mark.column, - ) - } - try emit(.mappingEnd(mark: endMark)) - return PureYAML.Parsing.NodeResult(kind: .mapping, endMark: endMark) - } - - mutating func parseBlockMappingPairs( - boundaryColumn: Int, - includeIndentedContinuation: Bool, - endMark: inout PureYAML.Parsing.Mark, - firstKeyProperties: PureYAML.Parsing.NodeProperties = .none, - ) throws { - var keyProperties = firstKeyProperties - while canContinueBlockMapping(minimumColumn: boundaryColumn) { - endMark = try parseMappingPairAndBoundaries( - boundaryColumn: boundaryColumn, - includeIndentedContinuation: includeIndentedContinuation, - keyProperties: keyProperties, - ).endMark - keyProperties = .none - } - while includeIndentedContinuation, cursor.current?.kind.isIndent == true { - endMark = try parseIndentedBlockMappingPairs(boundaryColumn: boundaryColumn).endMark - } - } - - mutating func parseIndentedBlockMappingPairs( - boundaryColumn: Int, - ) throws -> PureYAML.Parsing.NodeResult { - _ = try expect("indent") { $0.isIndent } - guard cursor.current?.kind.isMappingKey == true else { - throw unexpectedToken(expected: "mapping key") - } - var endMark = cursor.current?.mark ?? .start - while canContinueBlockMapping(minimumColumn: boundaryColumn) { - endMark = try parseMappingPairAndBoundaries( - boundaryColumn: boundaryColumn, - includeIndentedContinuation: true, - ).endMark - } - if !canEndBlockMapping(at: cursor.current, minimumColumn: boundaryColumn) { - _ = try expect("dedent") { $0.isDedent } - } - return PureYAML.Parsing.NodeResult(kind: .mapping, endMark: endMark) - } - - mutating func parseMappingPairAndBoundaries( - boundaryColumn: Int, - includeIndentedContinuation: Bool, - keyProperties: PureYAML.Parsing.NodeProperties = .none, - ) throws -> PureYAML.Parsing.NodeResult { - let result = try parseMappingPair(keyProperties: keyProperties) - try consumeMappingSiblingBoundaryDedents(minimumColumn: boundaryColumn) - try consumeMappingSiblingBoundaryReindent(minimumColumn: boundaryColumn) - if includeIndentedContinuation { - try consumeDedents(atOrAbove: boundaryColumn) - try consumeMappingSiblingBoundaryDedents(minimumColumn: boundaryColumn) - try consumeMappingSiblingBoundaryReindent(minimumColumn: boundaryColumn) - } - return result - } - - mutating func parseBlockScalar( - properties: PureYAML.Parsing.NodeProperties, - ) throws -> PureYAML.Parsing.NodeResult { - let header = try expect("block scalar header") { kind in - if case .blockScalarHeader = kind { - return true - } - return false - } - guard case let .blockScalarHeader(_, _, value) = header.kind else { - throw unexpectedToken(expected: "block scalar header") - } - let mark = properties.mark ?? header.mark - try emit(.scalar( - value: value, - anchor: properties.anchor, - tag: properties.tag, - style: scalarStyle(for: header.kind), - mark: mark, - )) - return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: header.endMark) - } - - func scalarStyle(for kind: PureYAML.Parsing.TokenKind) -> PureYAML.Parsing.ScalarStyle { - switch kind { - case .blockScalarHeader(.folded, _, _): - .folded - case .blockScalarHeader(.literal, _, _): - .literal - default: - .plain - } - } -} diff --git a/Sources/Parsing/TokenEventParser.EmptyNode.swift b/Sources/Parsing/TokenEventParser.EmptyNode.swift deleted file mode 100644 index 79ffec9..0000000 --- a/Sources/Parsing/TokenEventParser.EmptyNode.swift +++ /dev/null @@ -1,143 +0,0 @@ -extension PureYAML.Parsing.TokenEventParser { - /// Parses the key portion of a block-mapping pair. An explicit `?` key may - /// have no following node (an empty key), and a pair written as plain - /// `: value` has no `?` and no key node at all. Both compose to an empty - /// scalar key, mirroring libyaml's empty-scalar production for an absent key. - mutating func parseMappingPairKey( - preconsumedProperties: PureYAML.Parsing.NodeProperties = .none, - ) throws { - // A key node may carry leading anchor/tag properties, which the scanner - // emits before the implicit `mappingKey` indicator (for example - // `&anchor c: 3` scans as anchor, `?`, `c`). It may also itself be an - // alias (`*b : value`), which has no `mappingKey` indicator at all. - // `preconsumedProperties` are properties the caller already consumed - // because they sat on the same line as the first key of this mapping. - var leadingProperties = preconsumedProperties - try leadingProperties.mergeUnset(from: consumeProperties()) - if leadingProperties != .none { - try parseKeyWithLeadingProperties(leadingProperties) - return - } - if cursor.current?.kind.isAlias == true { - try emitAliasKey() - return - } - guard cursor.current?.kind.isMappingKey == true else { - let mark = cursor.current?.mark ?? cursor.previous?.endMark ?? .start - try emitEmptyScalar(properties: .none, mark: mark) - return - } - let keyIndicator = try expect("mapping key") { $0.isMappingKey } - let properties = try consumeProperties() - // An alias as an explicit `? *alias` key is the key node itself; emit it - // directly so the following `:` is read as this pair's value separator - // rather than re-routing the alias into a nested block mapping. - if properties == .none, cursor.current?.kind.isAlias == true { - try emitAliasKey() - return - } - if properties != .none || canBeginNode(cursor.current) { - _ = try parseNode(properties: properties) - return - } - try emitEmptyScalar(properties: properties, mark: keyIndicator.endMark) - } - - /// Consumes an alias token that stands in mapping-key position and emits the - /// alias event for it. Done here rather than through ``parseNode`` because an - /// alias followed by `:` routes back into a block mapping in ``parseNode``; - /// the key parser owns the alias key directly so it resolves to its value. - mutating func emitAliasKey() throws { - let token = try expect("alias") { $0.isAlias } - guard case let .alias(name) = token.kind else { - throw unexpectedToken(expected: "alias") - } - try emit(.alias(anchor: name, mark: token.mark)) - } - - /// Parses a key node whose anchor/tag properties were already consumed. - /// The scanner emits properties before the implicit `mappingKey` indicator, - /// so the indicator (if present) is skipped here and the remaining content, - /// or an empty scalar, carries the properties. - mutating func parseKeyWithLeadingProperties( - _ properties: PureYAML.Parsing.NodeProperties, - ) throws { - if cursor.current?.kind.isMappingKey == true { - let keyIndicator = try expect("mapping key") { $0.isMappingKey } - if canBeginNode(cursor.current), hasNodeOnSameLine(after: keyIndicator) { - _ = try parseNode(properties: properties) - return - } - try emitEmptyScalar(properties: properties, mark: keyIndicator.endMark) - return - } - if canBeginNode(cursor.current) { - _ = try parseNode(properties: properties) - return - } - let mark = properties.mark ?? cursor.current?.mark ?? cursor.previous?.endMark ?? .start - try emitEmptyScalar(properties: properties, mark: mark) - } - - /// Emits an empty (null) plain scalar carrying any pending anchor or tag. - /// This is the single composition point for an empty node, so `!!null`, - /// `!!str`, an anchored empty scalar, and a bare absent node all resolve - /// through the normal scalar resolver against an empty value. - mutating func emitEmptyScalar( - properties: PureYAML.Parsing.NodeProperties, - mark: PureYAML.Parsing.Mark, - ) throws { - try emit(.scalar( - value: "", - anchor: properties.anchor, - tag: properties.tag, - style: .plain, - mark: properties.mark ?? mark, - )) - } - - /// Emits an empty scalar when a node is grammatically required but the next - /// token cannot begin one. An anchor or tag makes the empty node explicit; - /// without either, the absence is a genuine error, matching libyaml's - /// "did not find expected node content". - mutating func emptyScalarOrThrow( - properties: PureYAML.Parsing.NodeProperties, - ) throws -> PureYAML.Parsing.NodeResult { - let mark = properties.mark ?? cursor.current?.mark ?? cursor.previous?.endMark ?? .start - guard properties.anchor != nil || properties.tag != nil else { - throw PureYAML.Parsing.ParseError.expectedNode(line: mark.line, column: mark.column) - } - try emitEmptyScalar(properties: properties, mark: mark) - return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: mark) - } - - /// True when `token` can begin a YAML node (scalar, alias, or a collection - /// opener). Properties (anchor/tag) are consumed before this is asked. - func canBeginNode(_ token: PureYAML.Parsing.Token?) -> Bool { - guard let token else { - return false - } - switch token.kind { - case .scalar, .alias, .blockScalarHeader, .blockEntry, - .flowMappingStart, .flowSequenceStart, .indent: - return true - default: - return false - } - } - - /// Parses a block node introduced by a leading indent token, carrying the - /// already-consumed node properties through to the indented content. Used - /// when an anchor or tag precedes an indented block collection on the next - /// line, e.g. `key: &anchor` then an indented `!!map` mapping. - mutating func parseIndentedNode( - properties: PureYAML.Parsing.NodeProperties, - ) throws -> PureYAML.Parsing.NodeResult { - _ = try expect("indent") { $0.isIndent } - let result = try parseNode(properties: properties) - if cursor.current?.kind.isDedent == true { - _ = try expect("dedent") { $0.isDedent } - } - return result - } -} diff --git a/Sources/Parsing/TokenEventParser.Flow.swift b/Sources/Parsing/TokenEventParser.Flow.swift deleted file mode 100644 index 7cb237b..0000000 --- a/Sources/Parsing/TokenEventParser.Flow.swift +++ /dev/null @@ -1,203 +0,0 @@ -extension PureYAML.Parsing.TokenEventParser { - /// Parses a flow sequence, mirroring libyaml's `flow_sequence_entry` - /// productions. Each entry is either a flow node or, when introduced by a - /// `mappingKey` (`?`) or recognized as a single-pair `key: value`, a one-entry - /// flow mapping nested directly in the sequence. - mutating func parseFlowSequence( - properties: PureYAML.Parsing.NodeProperties, - ) throws -> PureYAML.Parsing.NodeResult { - let start = try expect("flow sequence start") { kind in - if case .flowSequenceStart = kind { - return true - } - return false - } - let startMark = properties.mark ?? start.mark - var endMark = start.endMark - try emit(.sequenceStart( - anchor: properties.anchor, - tag: properties.tag, - style: .flow, - mark: startMark, - )) - while cursor.current != nil { - if case .flowSequenceEnd = cursor.current?.kind { - let end = try expect("flow sequence end") { kind in - if case .flowSequenceEnd = kind { - return true - } - return false - } - endMark = end.endMark - try emit(.sequenceEnd(mark: endMark)) - return PureYAML.Parsing.NodeResult(kind: .sequence, endMark: endMark) - } - if case .flowEntry = cursor.current?.kind { - _ = try cursor.advance() - continue - } - endMark = try parseFlowSequenceEntry().endMark - if cursor.current?.kind.isFlowTerminator == false { - throw unexpectedToken(expected: "flow entry or flow sequence end") - } - } - throw unexpectedToken(expected: "flow sequence end") - } - - /// Parses a single flow-sequence entry. A `mappingKey` (`?`, or one spliced in - /// by the simple-key mechanism before a `key: value` pair) produces a - /// single-pair flow mapping wrapping the entry; otherwise the entry is a plain - /// flow node. - mutating func parseFlowSequenceEntry() throws -> PureYAML.Parsing.NodeResult { - if cursor.current?.kind.isMappingKey == true { - return try parseFlowSequenceSinglePair() - } - return try parseNode() - } - - /// Wraps an explicit-`?` single-pair entry of a flow sequence in a one-entry - /// flow mapping, mirroring libyaml's `flow_sequence_entry_mapping_*` states. - mutating func parseFlowSequenceSinglePair() throws -> PureYAML.Parsing.NodeResult { - let keyIndicator = try expect("mapping key") { $0.isMappingKey } - try emit(.mappingStart(anchor: nil, tag: nil, style: .flow, mark: keyIndicator.mark)) - try parseFlowSinglePairKeyValue(keyIndicator: keyIndicator) - let mark = cursor.current?.mark ?? keyIndicator.endMark - try emit(.mappingEnd(mark: mark)) - return PureYAML.Parsing.NodeResult(kind: .mapping, endMark: mark) - } - - mutating func parseFlowMapping( - properties: PureYAML.Parsing.NodeProperties, - ) throws -> PureYAML.Parsing.NodeResult { - let start = try expect("flow mapping start") { kind in - if case .flowMappingStart = kind { - return true - } - return false - } - let startMark = properties.mark ?? start.mark - var endMark = start.endMark - try emit(.mappingStart( - anchor: properties.anchor, - tag: properties.tag, - style: .flow, - mark: startMark, - )) - while cursor.current != nil { - if case .flowMappingEnd = cursor.current?.kind { - let end = try expect("flow mapping end") { kind in - if case .flowMappingEnd = kind { - return true - } - return false - } - endMark = end.endMark - try emit(.mappingEnd(mark: endMark)) - return PureYAML.Parsing.NodeResult(kind: .mapping, endMark: endMark) - } - if case .flowEntry = cursor.current?.kind { - _ = try cursor.advance() - continue - } - endMark = try parseFlowMappingPair().endMark - if cursor.current?.kind.isFlowTerminator == false { - throw unexpectedToken(expected: "flow entry or flow mapping end") - } - } - throw unexpectedToken(expected: "flow mapping end") - } - - /// Parses one flow-mapping entry. With a `mappingKey` (`?`) indicator the key - /// follows the explicit form; without one the entry is a `flow_node` that is - /// itself the key (carrying an empty value unless a `:` provides one). - mutating func parseFlowMappingPair() throws -> PureYAML.Parsing.NodeResult { - if cursor.current?.kind.isMappingKey == true { - let keyIndicator = try expect("mapping key") { $0.isMappingKey } - return try parseFlowSinglePairKeyValue(keyIndicator: keyIndicator) - } - try parseFlowMappingImplicitKey() - return try parseFlowMappingValue() - } - - /// Parses the key and value of a flow pair introduced by an explicit `?`, - /// where each side may be absent (composing an empty scalar). Shared by flow - /// mappings and single-pair entries nested in a flow sequence. - @discardableResult - mutating func parseFlowSinglePairKeyValue( - keyIndicator: PureYAML.Parsing.Token, - ) throws -> PureYAML.Parsing.NodeResult { - if flowKeyIsEmpty() { - try emitEmptyScalar(properties: .none, mark: keyIndicator.endMark) - } else { - // The key of a flow pair follows the same rules whether the pair was - // introduced by an explicit `?` or had a `mappingKey` spliced in by the - // simple-key mechanism: a bare tag/anchor before the `:` composes an empty - // tagged scalar key (`{ !!str : bar }`), and an alias before the `:` is the - // key itself. Routing through `parseNode` would instead treat the tag/`:` - // as a nested block mapping, so use the flow key parser shared with the - // implicit form. - try parseFlowMappingImplicitKey() - } - return try parseFlowMappingValue() - } - - /// Parses the key of an implicit flow-mapping entry (`key : value` with no - /// explicit `?`). The key may be absent entirely (`{ : v }`, `{ , : v }`) - /// or be only a tag/anchor with no content, both composing to an empty - /// scalar key rather than routing into a block mapping. - mutating func parseFlowMappingImplicitKey() throws { - let properties = try consumeProperties() - if properties == .none, cursor.current?.kind.isMappingValue == true { - let mark = cursor.current?.mark ?? cursor.previous?.endMark ?? .start - try emitEmptyScalar(properties: .none, mark: mark) - return - } - if properties != .none, cursor.current?.kind.isMappingValue == true || cursor.current?.kind.isFlowTerminator == true { - let mark = cursor.current?.mark ?? cursor.previous?.endMark ?? .start - try emitEmptyScalar(properties: properties, mark: mark) - return - } - // An alias as a flow-mapping key (`{*a: b}`) is emitted directly so it is - // not re-routed into a block mapping by the alias-as-key check in - // `parseNode`, which only applies to block context. - if properties == .none, cursor.current?.kind.isAlias == true, try cursor.peek()?.kind.isMappingValue == true { - try emitAliasKey() - return - } - _ = try parseNode(properties: properties) - } - - /// Parses the value half of a flow-mapping entry. A `:` consumes its node (or an - /// empty scalar when the value is absent); without a `:` the entry has an - /// implicit empty value (libyaml's `flow_mapping_value` with `empty == 1`). - mutating func parseFlowMappingValue() throws -> PureYAML.Parsing.NodeResult { - guard cursor.current?.kind.isMappingValue == true else { - let mark = cursor.current?.mark ?? cursor.previous?.endMark ?? .start - try emitEmptyScalar(properties: .none, mark: mark) - return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: mark) - } - let valueIndicator = try expect("mapping value") { kind in - if case .mappingValue = kind { - return true - } - return false - } - let properties = try consumeProperties() - guard properties != .none || canBeginNode(cursor.current), cursor.current?.kind.isFlowTerminator == false else { - let mark = valueIndicator.endMark - try emitEmptyScalar(properties: properties, mark: mark) - return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: mark) - } - return try parseNode(properties: properties) - } - - /// True when the key position right after a `?` indicator is empty, i.e. the - /// next token is a `:` value indicator, a flow terminator, or the end of input, - /// so an empty scalar key must be composed (libyaml's `flow_mapping_key`). - func flowKeyIsEmpty() -> Bool { - guard let current = cursor.current else { - return true - } - return current.kind.isMappingValue || current.kind.isFlowTerminator - } -} diff --git a/Sources/Parsing/TokenEventParser.Helpers.swift b/Sources/Parsing/TokenEventParser.Helpers.swift deleted file mode 100644 index fdf6426..0000000 --- a/Sources/Parsing/TokenEventParser.Helpers.swift +++ /dev/null @@ -1,344 +0,0 @@ -extension PureYAML.Parsing.TokenEventParser { - mutating func parseMappingPair( - keyProperties: PureYAML.Parsing.NodeProperties = .none, - ) throws -> PureYAML.Parsing.NodeResult { - try parseMappingPairKey(preconsumedProperties: keyProperties) - guard cursor.current?.kind.isMappingValue == true else { - let mark = cursor.current?.mark ?? cursor.previous?.endMark ?? .start - try emitEmptyScalar(properties: .none, mark: mark) - return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: mark) - } - let valueIndicator = try expect("mapping value") { kind in - if case .mappingValue = kind { - return true - } - return false - } - return try parseMappingPairValue(valueIndicator: valueIndicator) - } - - mutating func parseMappingPairValue( - valueIndicator: PureYAML.Parsing.Token, - ) throws -> PureYAML.Parsing.NodeResult { - let properties = try consumeProperties() - if cursor.current?.kind.isIndent == true { - _ = try expect("indent") { $0.isIndent } - let result: PureYAML.Parsing.NodeResult = if cursor.current?.kind.isBlockEntry == true { - try parseBlockSequence( - properties: properties, - allowMappingKeyTerminator: true, - ) - } else { - try parseNode(properties: properties) - } - if cursor.current?.kind.isDedent == true { - _ = try expect("dedent") { $0.isDedent } - } else if cursor.current?.kind.isMappingKey != true { - _ = try expect("dedent") { $0.isDedent } - } - return result - } - if properties == .none, try canParsePlainScalarWithContinuation(after: valueIndicator) { - return try parsePlainScalarWithContinuation(properties: properties) - } - if cursor.current?.kind.isMappingKey == true, hasNodeOnSameLine(after: valueIndicator) { - return try parseBlockMapping( - properties: properties, - includeIndentedContinuation: true, - minimumColumn: cursor.current?.mark.column, - ) - } - // An anchor or tag at value position with no node following on the same - // line (the next token opens a sibling key) marks an empty value that - // still carries the properties, so `a: &anchor` registers `anchor` as - // null rather than swallowing the next key into a nested mapping. - if properties != .none, !hasNodeOnSameLine(after: valueIndicator) { - let mark = properties.mark ?? valueIndicator.endMark - try emitEmptyScalar(properties: properties, mark: mark) - return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: mark) - } - if properties != .none || hasNodeOnSameLine(after: valueIndicator) { - return try parseNode(properties: properties) - } - if cursor.current?.kind.isBlockEntry == true { - return try parseBlockSequence( - properties: properties, - allowMappingKeyTerminator: true, - ) - } - - let mark = valueIndicator.endMark - try emitEmptyScalar(properties: .none, mark: mark) - return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: mark) - } - - mutating func parsePlainScalarWithContinuation( - properties: PureYAML.Parsing.NodeProperties, - ) throws -> PureYAML.Parsing.NodeResult { - let first = try expect("plain scalar") { kind in - if case .scalar(_, .plain) = kind { - return true - } - return false - } - var value = try decodedScalar(from: first) - var endMark = first.endMark - - _ = try expect("indent") { $0.isIndent } - while isScalar(cursor.current) { - let token = try cursor.advance() ?? first - let continuation = try plainContinuationText(from: token) - value += " " + continuation - endMark = token.endMark - } - _ = try expect("dedent") { $0.isDedent } - - let mark = properties.mark ?? first.mark - try emit(.scalar( - value: value, - anchor: properties.anchor, - tag: properties.tag, - style: .plain, - mark: mark, - )) - return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: endMark) - } - - mutating func consumeProperties() throws -> PureYAML.Parsing.NodeProperties { - var properties = PureYAML.Parsing.NodeProperties.none - while let token = cursor.current { - switch token.kind { - case let .anchor(anchor): - properties.anchor = anchor - properties.mark = properties.mark ?? token.mark - _ = try cursor.advance() - case let .tag(tag): - properties.tag = tag - properties.mark = properties.mark ?? token.mark - _ = try cursor.advance() - default: - return properties - } - } - return properties - } - - func decodeScalar( - _ value: String, - style: PureYAML.Parsing.ScalarStyle, - line: Int, - ) throws -> String { - switch style { - case .doubleQuoted: - try scalarParser.parseDoubleQuoted("\"\(value)\"", line: line) - case .folded, .literal: - value - case .plain: - value - case .singleQuoted: - try scalarParser.parseSingleQuoted("'\(value)'", line: line) - } - } - - func decodedScalar(from token: PureYAML.Parsing.Token) throws -> String { - guard case let .scalar(value, style) = token.kind else { - throw unexpectedToken(expected: "scalar") - } - return try decodeScalar(value, style: style, line: token.mark.line) - } - - func plainContinuationText(from token: PureYAML.Parsing.Token) throws -> String { - guard case let .scalar(value, style) = token.kind else { - throw unexpectedToken(expected: "scalar") - } - switch style { - case .doubleQuoted: - return "\"\(value)\"" - case .singleQuoted: - return "'\(value)'" - case .folded, .literal, .plain: - return try decodeScalar(value, style: style, line: token.mark.line) - } - } - - func hasNodeOnSameLine(after token: PureYAML.Parsing.Token) -> Bool { - guard let current = cursor.current, !current.kind.isTerminator else { - return false - } - return current.mark.line == token.mark.line - } - - mutating func canParsePlainScalarWithContinuation(after token: PureYAML.Parsing.Token) throws -> Bool { - guard case .scalar(_, .plain) = cursor.current?.kind else { - return false - } - guard cursor.current?.mark.line == token.mark.line else { - return false - } - guard try cursor.peek()?.kind.isIndent == true else { - return false - } - guard try isScalar(cursor.peek(offset: 2)) else { - return false - } - return true - } - - mutating func canParsePlainScalarContinuation() throws -> Bool { - guard case .scalar(_, .plain) = cursor.current?.kind else { - return false - } - guard try cursor.peek()?.kind.isIndent == true else { - return false - } - guard try isScalar(cursor.peek(offset: 2)) else { - return false - } - return true - } - - func isScalar(_ token: PureYAML.Parsing.Token?) -> Bool { - guard case .scalar = token?.kind else { - return false - } - return true - } - - mutating func consumeDedents(atOrAbove minimumColumn: Int) throws { - while let width = dedentWidth(cursor.current), width >= minimumColumn { - _ = try expect("dedent") { $0.isDedent } - } - } - - mutating func consumeMappingSiblingBoundaryDedents(minimumColumn: Int) throws { - while try canConsumeMappingSiblingBoundaryDedent(minimumColumn: minimumColumn) { - _ = try expect("dedent") { $0.isDedent } - } - } - - mutating func canConsumeMappingSiblingBoundaryDedent(minimumColumn: Int) throws -> Bool { - guard let width = dedentWidth(cursor.current), - let next = try cursor.peek(), - next.kind.isMappingKey, - next.mark.column >= minimumColumn - else { - return false - } - return width == next.mark.column - 1 - } - - mutating func consumeMappingSiblingBoundaryReindent(minimumColumn: Int) throws { - guard let dedent = dedentWidth(cursor.current), - dedent < minimumColumn, - try indentWidth(cursor.peek()) == minimumColumn, - let mappingKey = try cursor.peek(offset: 2), - mappingKey.kind.isMappingKey, - mappingKey.mark.column == minimumColumn + 1 - else { - return - } - _ = try expect("dedent") { $0.isDedent } - _ = try expect("indent") { $0.isIndent } - } - - func dedentWidth(_ token: PureYAML.Parsing.Token?) -> Int? { - guard case let .dedent(width) = token?.kind else { - return nil - } - return width - } - - func indentWidth(_ token: PureYAML.Parsing.Token?) -> Int? { - guard case let .indent(width) = token?.kind else { - return nil - } - return width - } - - func canContinueBlockMapping(minimumColumn: Int?) -> Bool { - // A pair starts at a `mappingKey`/`mappingValue` indicator, or at the - // node properties (anchor/tag) or alias that introduce a key node whose - // own indicator the scanner emits after them. - guard let current = cursor.current, - current.kind.isMappingKey - || current.kind.isMappingValue - || current.kind.isNodeProperty - || current.kind.isAlias - else { - return false - } - guard let minimumColumn else { - return true - } - return current.mark.column >= minimumColumn - } - - func canEndBlockMapping( - at token: PureYAML.Parsing.Token?, - minimumColumn: Int? = nil, - ) -> Bool { - guard let token else { - return true - } - return token.kind.isBlockEntry - || token.kind.isDedent - || token.kind.isDocumentEnd - || token.kind.isDocumentStart - || token.kind.isFlowTerminator - || isLowerColumnMappingKey(token, minimumColumn: minimumColumn) - || token.kind.isStreamEnd - } - - func isLowerColumnMappingKey( - _ token: PureYAML.Parsing.Token?, - minimumColumn: Int?, - ) -> Bool { - guard let token, token.kind.isMappingKey, let minimumColumn else { - return false - } - return token.mark.column < minimumColumn - } - - mutating func expect( - _ expected: String, - matching: (PureYAML.Parsing.TokenKind) -> Bool, - ) throws -> PureYAML.Parsing.Token { - guard let token = cursor.current else { - let mark = cursor.previous?.endMark ?? .start - throw PureYAML.Parsing.ParseError.unexpectedToken( - expected: expected, - actual: "end of tokens", - line: mark.line, - column: mark.column, - ) - } - guard matching(token.kind) else { - throw PureYAML.Parsing.ParseError.unexpectedToken( - expected: expected, - actual: token.kind.description, - line: token.mark.line, - column: token.mark.column, - ) - } - _ = try cursor.advance() - return token - } - - func unexpectedToken(expected: String) -> PureYAML.Parsing.ParseError { - guard let token = cursor.current else { - let mark = cursor.previous?.endMark ?? .start - return PureYAML.Parsing.ParseError.unexpectedToken( - expected: expected, - actual: "end of tokens", - line: mark.line, - column: mark.column, - ) - } - return PureYAML.Parsing.ParseError.unexpectedToken( - expected: expected, - actual: token.kind.description, - line: token.mark.line, - column: token.mark.column, - ) - } -} diff --git a/Sources/Parsing/TokenEventParser.swift b/Sources/Parsing/TokenEventParser.swift deleted file mode 100644 index b2b180d..0000000 --- a/Sources/Parsing/TokenEventParser.swift +++ /dev/null @@ -1,324 +0,0 @@ -extension PureYAML.Parsing { - struct TokenEventParser { - var cursor: TokenCursor - let scalarParser: Parser - var events: [Event]? - var eventSink: ((Event) throws -> Void)? - - /// Current node recursion depth, tracked so deeply nested input fails - /// with a diagnostic instead of overflowing the stack. - var depth = 0 - - /// Maximum node nesting the parser will follow before failing with - /// ``ParseError/nestingTooDeep(line:column:)``. - /// - /// This bound is the single safety invariant for deeply nested input, - /// and it is *necessary*, not merely convenient. A ``Model/Value`` is a - /// recursive tree, so composing it, emitting it, comparing it, hashing - /// it, and -- crucially -- deallocating it all recurse with the nesting - /// depth. Swift tears down a recursive value type recursively, and that - /// teardown cannot be made iterative cheaply, so the only way to keep - /// every operation within the call stack is to bound the depth of the - /// tree where it is built: here, while parsing. - /// - /// Because every parsing entry point routes through ``parseNode``, this - /// one bound caps the depth of any parsed tree, which transitively keeps - /// the parser, the event composers, the emitter, equality, hashing, and - /// deallocation all well inside their limits. Measured on the main - /// thread's 8 MB stack: the parser overflows near depth 225 and the - /// tree operations near depth 50,000, while real-world specs nest below - /// ~40. So 64 accepts every realistic document yet rejects pathological - /// input -- for example a 100 KB `[[[...]]]` -- cleanly and early. - /// - /// The bound cannot simply be raised: above ~225 the recursive parser - /// itself overflows before the guard fires (lifting it would require an - /// iterative parser), and a tree past ~5,000 deep crashes on `deinit` - /// regardless, so a higher limit would trade a parse-time failure for an - /// un-fixable teardown crash. 64 is the largest value that is both safe - /// here and sufficient for real input. - /// - /// Caveat: a tree built *without* the parser -- for example by encoding - /// a pathologically deep `Codable` value -- bypasses this bound, and - /// operations on such a tree are O(depth) in stack like any recursive - /// value type. Keeping those shallow is the caller's responsibility, the - /// same way the standard library cannot prevent unbounded recursion in - /// user code. - static let maxNestingDepth = 64 - - init( - tokens: [Token], - scalarParser: Parser, - ) { - cursor = TokenCursor(tokens: tokens) - self.scalarParser = scalarParser - events = [] - eventSink = nil - } - - mutating func parse() throws -> [Event] { - try parseEvents() - return events ?? [] - } - - init( - tokens: [Token], - scalarParser: Parser, - eventSink: @escaping (Event) throws -> Void, - ) { - cursor = TokenCursor(tokens: tokens) - self.scalarParser = scalarParser - events = nil - self.eventSink = eventSink - } - - init( - yaml: String, - scalarParser: Parser, - eventSink: @escaping (Event) throws -> Void, - ) { - cursor = TokenCursor(yaml: yaml) - self.scalarParser = scalarParser - events = nil - self.eventSink = eventSink - } - - mutating func parseStreaming() throws { - try parseEvents() - } - - mutating func parseEvents() throws { - let streamStart = try expect("stream start") { kind in - if case .streamStart = kind { - return true - } - return false - } - - try emit(.streamStart(mark: streamStart.mark)) - // A stream whose only content is comments or whitespace has zero - // documents (YAML 1.2, matching libyaml and the test suite), so no - // document events are emitted. The value-level API maps an empty - // document stream to a single null value separately. - while cursor.current?.kind.isStreamEnd != true { - try parseDocument() - } - let streamEnd = try expect("stream end") { kind in - if case .streamEnd = kind { - return true - } - return false - } - try emit(.streamEnd(mark: streamEnd.mark)) - } - - mutating func emit(_ event: Event) throws { - if let eventSink { - try eventSink(event) - } else { - events?.append(event) - } - } - } -} - -extension PureYAML.Parsing.TokenEventParser { - mutating func parseDocument() throws { - let (startMark, explicitStart) = try parseDocumentStartMark() - try emit(.documentStart(explicit: explicitStart, mark: startMark)) - - if cursor.current?.kind.isDocumentEnd == true { - let end = try expect("document end") { $0.isDocumentEnd } - try appendEmptyDocument(at: end.mark) - try emit(.documentEnd(explicit: true, mark: end.endMark)) - return - } - - if cursor.current?.kind.isDocumentStart == true || cursor.current?.kind.isStreamEnd == true { - try appendEmptyDocument(at: cursor.current?.mark ?? startMark) - try emit(.documentEnd(explicit: false, mark: cursor.current?.mark ?? startMark)) - return - } - - var endMark = try parseDocumentRootNode().endMark - while cursor.current?.kind.isDedent == true { - let dedent = try expect("dedent") { $0.isDedent } - endMark = dedent.endMark - } - if cursor.current?.kind.isDocumentEnd == true { - // An explicit `...` end marker closes the document outright. Whatever - // follows -- a bare node, an explicit `---`, or the end of the stream - // -- starts a fresh document on the next loop iteration, so emit the - // end here and return rather than demanding a document boundary token. - let end = try expect("document end") { $0.isDocumentEnd } - try emit(.documentEnd(explicit: true, mark: end.endMark)) - return - } - if let current = cursor.current, !canEndDocument(at: current) { - throw PureYAML.Parsing.ParseError.unexpectedToken( - expected: "document boundary or stream end", - actual: current.kind.description, - line: current.mark.line, - column: current.mark.column, - ) - } - try emit(.documentEnd(explicit: false, mark: endMark)) - } - - mutating func parseDocumentRootNode() throws -> PureYAML.Parsing.NodeResult { - guard cursor.current?.kind.isIndent == true else { - return try parseNode() - } - _ = try expect("indent") { $0.isIndent } - let result = try parseNode() - if cursor.current?.kind.isDedent == true { - _ = try expect("dedent") { $0.isDedent } - } - return result - } - - mutating func parseDocumentStartMark() throws -> (mark: PureYAML.Parsing.Mark, explicit: Bool) { - guard cursor.current?.kind.isDocumentStart == true else { - return (cursor.current?.mark ?? cursor.previous?.endMark ?? .start, false) - } - let start = try cursor.advance() - return (start?.mark ?? .start, true) - } - - mutating func appendEmptyDocument(at mark: PureYAML.Parsing.Mark) throws { - try emit(.scalar(value: "", anchor: nil, tag: nil, style: .plain, mark: mark)) - } - - func canEndDocument(at token: PureYAML.Parsing.Token) -> Bool { - token.kind.isDocumentStart || token.kind.isStreamEnd - } - - mutating func parseNode() throws -> PureYAML.Parsing.NodeResult { - let properties = try consumeProperties() - return try parseNode(properties: properties) - } - - /// Reject documents nested past ``maxNestingDepth`` with a diagnostic so a - /// pathologically nested input fails cleanly instead of overflowing the - /// recursive-descent stack. - func enforceNestingLimit(properties: PureYAML.Parsing.NodeProperties) throws { - guard depth > Self.maxNestingDepth else { - return - } - let mark = properties.mark ?? cursor.current?.mark ?? cursor.previous?.endMark ?? .start - throw PureYAML.Parsing.ParseError.nestingTooDeep(line: mark.line, column: mark.column) - } - - mutating func parseNode( - properties initialProperties: PureYAML.Parsing.NodeProperties, - ) throws -> PureYAML.Parsing.NodeResult { - depth += 1 - defer { depth -= 1 } - try enforceNestingLimit(properties: initialProperties) - var properties = initialProperties - try properties.mergeUnset(from: consumeProperties()) - - guard let token = cursor.current, !token.kind.isTerminator else { - return try emptyScalarOrThrow(properties: properties) - } - - switch token.kind { - case let .alias(anchor): - return try parseAliasNode(anchor: anchor, token: token, properties: properties) - case .blockEntry: - return try parseBlockSequence(properties: properties) - case .blockScalarHeader: - return try parseBlockScalar(properties: properties) - case .flowMappingStart: - return try parseFlowMapping(properties: properties) - case .flowSequenceStart: - return try parseFlowSequence(properties: properties) - case .mappingKey, .mappingValue: - return try parseBlockMappingNode(token: token, properties: properties) - case .indent: - return try parseIndentedNode(properties: properties) - case let .scalar(value, style): - return try parseScalarNode(token: token, value: value, style: style, properties: properties) - default: - throw PureYAML.Parsing.ParseError.expectedNode(line: token.mark.line, column: token.mark.column) - } - } - - /// Parses an alias node. An alias immediately followed by a `:` value - /// indicator is the key of a block mapping (`*alias : value`), not a - /// standalone alias, so route into the mapping and let the key parser - /// consume it; otherwise emit the alias event directly. - mutating func parseAliasNode( - anchor: String, - token: PureYAML.Parsing.Token, - properties: PureYAML.Parsing.NodeProperties, - ) throws -> PureYAML.Parsing.NodeResult { - // An alias is a complete reference to an already-anchored node and cannot - // itself carry an anchor or tag. libyaml handles ALIAS as the first branch - // of yaml_parser_parse_node (parser.c:563-570), so the property branches - // are unreachable for an alias; mirror that by rejecting `&b *alias`. The - // rejection applies only when the property sits on the *same line* as the - // alias: a property scanned on the previous line (e.g. `key: &anchor` then an - // indented `*alias : value`) anchors the enclosing mapping, not the alias key. - let hasProperties = properties.anchor != nil || properties.tag != nil - if hasProperties, properties.mark?.line == token.mark.line { - let mark = properties.mark ?? token.mark - throw PureYAML.Parsing.ParseError.expectedNode(line: mark.line, column: mark.column) - } - // An alias immediately followed by a `:` is the first key of a block mapping; - // any property that reached here belongs to the mapping (it was on a prior - // line, per the check above), so route into the mapping carrying it. - if try cursor.peek()?.kind.isMappingValue == true { - return try parseBlockMapping( - properties: properties, - includeIndentedContinuation: false, - ) - } - _ = try cursor.advance() - let mark = properties.mark ?? token.mark - try emit(.alias(anchor: anchor, mark: mark)) - return PureYAML.Parsing.NodeResult(kind: .alias, endMark: token.endMark) - } - - /// Parses a block mapping introduced by a `mappingKey`/`mappingValue` - /// indicator. Properties on the same line as the first key indicator belong - /// to that key node, not to the mapping: `&a a: b` anchors the key `a`, - /// whereas `&a` alone on a line before the mapping anchors the map. - mutating func parseBlockMappingNode( - token: PureYAML.Parsing.Token, - properties: PureYAML.Parsing.NodeProperties, - ) throws -> PureYAML.Parsing.NodeResult { - if properties != .none, let mark = properties.mark, mark.line == token.mark.line, token.kind.isMappingKey { - return try parseBlockMapping( - properties: .none, - includeIndentedContinuation: false, - firstKeyProperties: properties, - ) - } - return try parseBlockMapping( - properties: properties, - includeIndentedContinuation: false, - ) - } - - mutating func parseScalarNode( - token: PureYAML.Parsing.Token, - value: String, - style: PureYAML.Parsing.ScalarStyle, - properties: PureYAML.Parsing.NodeProperties, - ) throws -> PureYAML.Parsing.NodeResult { - if try canParsePlainScalarContinuation() { - return try parsePlainScalarWithContinuation(properties: properties) - } - _ = try cursor.advance() - let decoded = try decodeScalar(value, style: style, line: token.mark.line) - let mark = properties.mark ?? token.mark - try emit(.scalar( - value: decoded, - anchor: properties.anchor, - tag: properties.tag, - style: style, - mark: mark, - )) - return PureYAML.Parsing.NodeResult(kind: .scalar, endMark: token.endMark) - } -} diff --git a/Tests/EventStreamingParserTests.swift b/Tests/EventStreamingParserTests.swift index 56f61d3..e357e26 100644 --- a/Tests/EventStreamingParserTests.swift +++ b/Tests/EventStreamingParserTests.swift @@ -43,18 +43,6 @@ struct EventStreamingParserTests { "error: Duplicate mapping key 'id' at $.flow.id", ]) } - - @Test( - "Lazy token cursor matches full scanner tokens", - arguments: EventStreamingFixtures.singleDocuments + EventStreamingFixtures.streams, - ) - func test_lazyTokenCursorMatchesFullScannerTokens(testCase: EventStreamingFixtures.DocumentCase) throws { - let full = try PureYAML.Parsing.Scanner().scan(testCase.yaml) - .filter { !$0.kind.isComment } - let lazy = try collectLazyTokens(testCase.yaml) - - #expect(lazy == full) - } } enum EventStreamingFixtures { @@ -155,13 +143,3 @@ private func composeStreamWithFullEvents(_ yaml: String) throws -> [PureYAML.Str var composer = PureYAML.Parsing.EventComposer(events: events, scalarParser: parser) return try composer.composeStream() } - -private func collectLazyTokens(_ yaml: String) throws -> [PureYAML.Parsing.Token] { - var cursor = PureYAML.Parsing.TokenCursor(yaml: yaml) - var tokens: [PureYAML.Parsing.Token] = [] - while let token = cursor.current { - tokens.append(token) - _ = try cursor.advance() - } - return tokens -} diff --git a/Tests/PerformancePhaseTests.swift b/Tests/PerformancePhaseTests.swift deleted file mode 100644 index 03bf685..0000000 --- a/Tests/PerformancePhaseTests.swift +++ /dev/null @@ -1,298 +0,0 @@ -import Foundation -@testable import PureYAML -import Testing - -@Suite("Performance Phases") -struct PerformancePhaseTests { - @Test("Profiles parser phases for representative slow corpus seeds") - func test_profilesParserPhasesForRepresentativeSeeds() throws { - guard ProcessInfo.processInfo.environment["PUREYAML_RUN_PHASE_PROFILER"] == "1" else { - return - } - - let measurements = try PerformancePhaseSeed.all.map(profile) - let slowestLazyParse = measurements.map(\.lazyParse.seconds).max() ?? 0 - let slowestMaterializedPipeline = measurements.map(\.materializedPipelineSeconds).max() ?? 0 - let report: [String: Any] = [ - "summary": [ - "seeds": measurements.count, - "slowestLazyParseSeconds": slowestLazyParse, - "slowestMaterializedPipelineSeconds": slowestMaterializedPipeline, - ], - "measurements": measurements.map(\.object), - ] - let artifactURL = try performancePhaseArtifactURL() - let data = try JSONSerialization.data(withJSONObject: report, options: [.prettyPrinted, .sortedKeys]) - try data.write(to: artifactURL) - - print("PureYAML phase profiler seeds: \(measurements.count)") - for measurement in measurements { - print(measurement.summaryLine) - } - print("Report: \(artifactURL.path)") - } -} - -private struct PerformancePhaseSeed { - var id: String - var localPath: String - var iterations: Int - - static let all = [ - PerformancePhaseSeed( - id: "cert-manager-crd", - localPath: "cert-manager-certificate-crd.yaml", - iterations: 20, - ), - PerformancePhaseSeed( - id: "stripe-yaml", - localPath: "api-guru-stripe-openapi.yaml", - iterations: 2, - ), - PerformancePhaseSeed( - id: "zoom-yaml", - localPath: "api-guru-zoom-openapi.yaml", - iterations: 2, - ), - ] -} - -private struct PhaseTiming { - var seconds: TimeInterval - var units: Int - - var object: [String: Any] { - [ - "seconds": seconds, - "units": units, - ] - } -} - -private struct PhaseMeasurement { - var seed: PerformancePhaseSeed - var bytes: Int - var scanner: PhaseTiming - var eventParser: PhaseTiming - var composer: PhaseTiming - var lazyParse: PhaseTiming - var validation: PhaseTiming - var dumping: PhaseTiming - - var materializedPipelineSeconds: TimeInterval { - scanner.seconds + eventParser.seconds + composer.seconds - } - - var object: [String: Any] { - [ - "id": seed.id, - "localPath": seed.localPath, - "bytes": bytes, - "iterations": seed.iterations, - "scanner": scanner.object, - "eventParser": eventParser.object, - "composer": composer.object, - "lazyParse": lazyParse.object, - "validation": validation.object, - "dumping": dumping.object, - "materializedPipelineSeconds": materializedPipelineSeconds, - ] - } - - var summaryLine: String { - [ - seed.id, - "bytes=\(bytes)", - "iter=\(seed.iterations)", - "scan=\(format(scanner.seconds))s", - "events=\(format(eventParser.seconds))s", - "compose=\(format(composer.seconds))s", - "lazy=\(format(lazyParse.seconds))s", - "validate=\(format(validation.seconds))s", - "dump=\(format(dumping.seconds))s", - ].joined(separator: " ") - } -} - -private struct WarmedPerformanceInputs { - var tokens: [PureYAML.Parsing.Token] - var events: [PureYAML.Parsing.Event] - var documents: [PureYAML.Stream.Document] -} - -private func profile(_ seed: PerformancePhaseSeed) throws -> PhaseMeasurement { - let yaml = try String(contentsOf: fixtureURL(for: seed), encoding: .utf8) - let parser = PureYAML.Parsing.Parser() - let warmed = try warmup(yaml: yaml, parser: parser) - let scanner = try timeScanner(yaml: yaml, iterations: seed.iterations) - let eventParser = try timeEventParser(tokens: warmed.tokens, parser: parser, iterations: seed.iterations) - let composer = try timeComposer(events: warmed.events, parser: parser, iterations: seed.iterations) - let lazyParse = try timeLazyParse(yaml: yaml, iterations: seed.iterations) - let validation = timeValidation(documents: warmed.documents, iterations: seed.iterations) - let dumping = timeDumping(documents: warmed.documents, iterations: seed.iterations) - - return PhaseMeasurement( - seed: seed, - bytes: yaml.utf8.count, - scanner: .init(seconds: scanner.seconds, units: scanner.value), - eventParser: .init(seconds: eventParser.seconds, units: eventParser.value), - composer: .init(seconds: composer.seconds, units: composer.value), - lazyParse: .init(seconds: lazyParse.seconds, units: lazyParse.value), - validation: .init(seconds: validation.seconds, units: validation.value), - dumping: .init(seconds: dumping.seconds, units: dumping.value), - ) -} - -private func warmup( - yaml: String, - parser: PureYAML.Parsing.Parser, -) throws -> WarmedPerformanceInputs { - let tokens = try PureYAML.Parsing.Scanner().scan(yaml) - var eventParser = PureYAML.Parsing.TokenEventParser(tokens: tokens, scalarParser: parser) - let events = try eventParser.parse() - var composer = PureYAML.Parsing.EventComposer(events: events, scalarParser: parser) - let documents = try composer.composeStream() - _ = PureYAML.Validation.Validator().collect(documents) - _ = PureYAML.dump(documents) - _ = try PureYAML.parseStream(yaml) - return WarmedPerformanceInputs(tokens: tokens, events: events, documents: documents) -} - -private func timeScanner( - yaml: String, - iterations: Int, -) throws -> (seconds: TimeInterval, value: Int) { - try timed { - var units = 0 - for _ in 0 ..< iterations { - units += try PureYAML.Parsing.Scanner().scan(yaml).count - } - return units - } -} - -private func timeEventParser( - tokens: [PureYAML.Parsing.Token], - parser: PureYAML.Parsing.Parser, - iterations: Int, -) throws -> (seconds: TimeInterval, value: Int) { - try timed { - var units = 0 - for _ in 0 ..< iterations { - var eventParser = PureYAML.Parsing.TokenEventParser(tokens: tokens, scalarParser: parser) - units += try eventParser.parse().count - } - return units - } -} - -private func timeComposer( - events: [PureYAML.Parsing.Event], - parser: PureYAML.Parsing.Parser, - iterations: Int, -) throws -> (seconds: TimeInterval, value: Int) { - try timed { - var units = 0 - for _ in 0 ..< iterations { - var composer = PureYAML.Parsing.EventComposer(events: events, scalarParser: parser) - units += try composer.composeStream().count - } - return units - } -} - -private func timeLazyParse( - yaml: String, - iterations: Int, -) throws -> (seconds: TimeInterval, value: Int) { - try timed { - var units = 0 - for _ in 0 ..< iterations { - units += try PureYAML.parseStream(yaml).count - } - return units - } -} - -private func timeValidation( - documents: [PureYAML.Stream.Document], - iterations: Int, -) -> (seconds: TimeInterval, value: Int) { - timed { - var units = 0 - for _ in 0 ..< iterations { - units += PureYAML.Validation.Validator().collect(documents).issues.count - } - return units - } -} - -private func timeDumping( - documents: [PureYAML.Stream.Document], - iterations: Int, -) -> (seconds: TimeInterval, value: Int) { - timed { - var units = 0 - for _ in 0 ..< iterations { - units += PureYAML.dump(documents).utf8.count - } - return units - } -} - -private func fixtureURL(for seed: PerformancePhaseSeed) throws -> URL { - try #require(Bundle.module.url( - forResource: seed.resourceName, - withExtension: seed.resourceExtension, - subdirectory: "Fixtures/real-yaml", - )) -} - -private func performancePhaseArtifactURL() throws -> URL { - if let path = ProcessInfo.processInfo.environment["PUREYAML_PHASE_PROFILER_ARTIFACT"] { - let url = URL(fileURLWithPath: path) - try FileManager.default.createDirectory( - at: url.deletingLastPathComponent(), - withIntermediateDirectories: true, - ) - return url - } - - let url = URL(fileURLWithPath: FileManager.default.currentDirectoryPath) - .appendingPathComponent(".build") - .appendingPathComponent("pureyaml-artifacts") - .appendingPathComponent("performance") - .appendingPathComponent("phase-profile.json") - try FileManager.default.createDirectory( - at: url.deletingLastPathComponent(), - withIntermediateDirectories: true, - ) - return url -} - -private func timed(_ body: () throws -> Int) rethrows -> (seconds: TimeInterval, value: Int) { - let start = DispatchTime.now().uptimeNanoseconds - let value = try body() - let end = DispatchTime.now().uptimeNanoseconds - return (Double(end - start) / 1_000_000_000, value) -} - -private func format(_ value: Double) -> String { - String(format: "%.3f", value) -} - -private extension PerformancePhaseSeed { - var resourceName: String { - guard let dotIndex = localPath.lastIndex(of: ".") else { - return localPath - } - return String(localPath[.. 'it''s'}] - text: |+ - line - folded: >- - line - """, - ) - - expectFlowAndBlockScalarKinds(tokens) - expectFlowAndBlockScalarRequiredKinds(tokens) - expectFlowAndBlockScalarForbiddenKinds(tokens) - } - - @Test("Plain scalars keep non-indicator hash and colon characters") - func test_plainScalarsKeepNonIndicatorHashAndColonCharacters() throws { - let tokens = try scanKinds( - """ - url: https://example.com/a:b - fragment: value#kept # removed - """, - ) - - expectKinds(tokens, [ - "streamStart", - "mappingKey", - "scalar value=\"url\" style=plain", - "mappingValue", - "scalar value=\"https://example.com/a:b\" style=plain", - "mappingKey", - "scalar value=\"fragment\" style=plain", - "mappingValue", - "scalar value=\"value#kept\" style=plain", - "comment value=\"removed\"", - "streamEnd", - ]) - expectContains(tokens, [ - "scalar value=\"https://example.com/a:b\" style=plain", - "scalar value=\"value#kept\" style=plain", - "comment value=\"removed\"", - ]) - expectDoesNotContain(tokens, [ - "scalar value=\"https\" style=plain", - "comment value=\"kept # removed\"", - ]) - } - - @Test("Scans directives and document markers without value tokens") - func test_directivesAndDocumentMarkers() throws { - let tokens = try scanKinds( - """ - %YAML 1.2 - %TAG !yaml! tag:yaml.org,2002: - --- - value: !yaml!str true - ... - """, - ) - - expectKinds(tokens, [ - "streamStart", - "documentStart", - "mappingKey", - "scalar value=\"value\" style=plain", - "mappingValue", - "tag value=tag:yaml.org,2002:str", - "scalar value=\"true\" style=plain", - "documentEnd", - "streamEnd", - ]) - expectDoesNotContain(tokens, [ - "scalar value=\"%YAML 1.2\" style=plain", - "scalar value=\"---\" style=plain", - "scalar value=\"...\" style=plain", - ]) - } - - @Test("Scans multiple document boundaries as tokens") - func test_multipleDocumentBoundaries() throws { - let tokens = try scanKinds( - """ - first: one - --- - - two - ... - # trailing - --- - """, - ) - - expectKinds(tokens, [ - "streamStart", - "mappingKey", - "scalar value=\"first\" style=plain", - "mappingValue", - "scalar value=\"one\" style=plain", - "documentStart", - "blockEntry", - "scalar value=\"two\" style=plain", - "documentEnd", - "comment value=\"trailing\"", - "documentStart", - "streamEnd", - ]) - expectContains(tokens, [ - "documentStart", - "documentEnd", - "comment value=\"trailing\"", - ]) - expectDoesNotContain(tokens, [ - "scalar value=\"---\" style=plain", - "scalar value=\"...\" style=plain", - ]) - } - - @Test("Tracks UTF-8 source marks") - func test_utf8SourceMarks() throws { - let tokens = try PureYAML.Parsing.Scanner().scan("caf\u{00E9}: yes") - guard let key = tokens.first(where: { - $0.kind == .scalar(value: "caf\u{00E9}", style: .plain) - }) else { - recordIssue("expected cafe scalar") - return - } - guard let value = tokens.first(where: { - $0.kind == .scalar(value: "yes", style: .plain) - }) else { - recordIssue("expected yes scalar") - return - } - - expectMarks(key, start: .init(line: 1, column: 1, index: 0), end: .init(line: 1, column: 5, index: 5)) - expectMarks(value, start: .init(line: 1, column: 7, index: 7), end: .init(line: 1, column: 10, index: 10)) - } - - @Test("Tracks ASCII and extended UTF-8 marks in one scan") - func test_asciiAndExtendedUTF8Marks() throws { - let tokens = try PureYAML.Parsing.Scanner().scan("ascii: abc\nemoji: 😀\nnext: ok") - guard let emoji = tokens.first(where: { - $0.kind == .scalar(value: "😀", style: .plain) - }) else { - recordIssue("expected emoji scalar") - return - } - guard let next = tokens.first(where: { - $0.kind == .scalar(value: "next", style: .plain) - }) else { - recordIssue("expected next scalar") - return - } - - expectMarks(emoji, start: .init(line: 2, column: 8, index: 18), end: .init(line: 2, column: 9, index: 22)) - expectMarks(next, start: .init(line: 3, column: 1, index: 23), end: .init(line: 3, column: 5, index: 27)) - } - - @Test("Reader advances UTF-8 marks across line breaks") - func test_readerAdvancesUTF8MarksAcrossLineBreaks() { - var reader = PureYAML.Parsing.Reader("a\u{00E9}\n\u{03B2}\r\nz") - - expectReaderMark(&reader, line: 1, column: 1, index: 0) - expectReaderMark(&reader, line: 1, column: 2, index: 1) - expectReaderMark(&reader, line: 1, column: 3, index: 3) - expectReaderMark(&reader, line: 2, column: 1, index: 4) - expectReaderMark(&reader, line: 2, column: 2, index: 6) - expectReaderMark(&reader, line: 3, column: 1, index: 8) - expectReaderMark(&reader, line: 3, column: 2, index: 9, advanceAfterCheck: false) - } - - @Test("Reports scanner errors") - func test_errorReporting() { - expectScannerError("root:\n child: yes\n wrong: no", .unexpectedIndentation(line: 3)) - expectScannerError("name: \"open", .unterminatedQuotedString(line: 1)) - expectScannerError("name: 'open", .unterminatedQuotedString(line: 1)) - expectScannerError("anchor: &", .expectedAnchorName(line: 1)) - expectScannerError("alias: *", .expectedAliasName(line: 1)) - expectScannerError("tag: ! [String] { - try PureYAML.Parsing.Scanner().scan(yaml).map(\.kind.description) -} - -private func expectKinds( - _ actual: [String], - _ expected: [String], -) { - #expect(actual == expected) -} - -private func expectContains( - _ actual: [String], - _ required: [String], -) { - for item in required { - #expect(actual.contains(item)) - } -} - -private func expectDoesNotContain( - _ actual: [String], - _ forbidden: [String], -) { - for item in forbidden { - #expect(!actual.contains(item)) - } -} - -private func expectMarks( - _ token: PureYAML.Parsing.Token, - start: PureYAML.Parsing.Mark, - end: PureYAML.Parsing.Mark, -) { - #expect(token.mark == start) - #expect(token.endMark == end) -} - -private func expectFlowAndBlockScalarKinds(_ tokens: [String]) { - expectKinds(tokens, [ - "streamStart", - "mappingKey", - "scalar value=\"flow\" style=plain", - "mappingValue", - "flowSequenceStart", - "tag value=!Thing", - "anchor name=item", - "scalar value=\"value\" style=doubleQuoted", - "flowEntry", - "alias name=item", - "flowEntry", - "flowMappingStart", - "mappingKey", - "scalar value=\"tagged\" style=plain", - "mappingValue", - "tag value=!", - "scalar value=\"it''s\" style=singleQuoted", - "flowMappingEnd", - "flowSequenceEnd", - "mappingKey", - "scalar value=\"text\" style=plain", - "mappingValue", - "blockScalarHeader style=literal chomping=keep value=\"line\\n\"", - "mappingKey", - "scalar value=\"folded\" style=plain", - "mappingValue", - "blockScalarHeader style=folded chomping=strip value=\"line\"", - "streamEnd", - ]) -} - -private func expectFlowAndBlockScalarRequiredKinds(_ tokens: [String]) { - expectContains(tokens, [ - "flowSequenceStart", - "flowSequenceEnd", - "flowMappingStart", - "flowMappingEnd", - "tag value=!Thing", - "anchor name=item", - "alias name=item", - "blockScalarHeader style=literal chomping=keep value=\"line\\n\"", - "blockScalarHeader style=folded chomping=strip value=\"line\"", - ]) -} - -private func expectFlowAndBlockScalarForbiddenKinds(_ tokens: [String]) { - expectDoesNotContain(tokens, [ - "comment value=\"removed\"", - "scalar value=\"tag:example.com,2026:thing\" style=plain", - ]) -} - -private func expectReaderMark( - _ reader: inout PureYAML.Parsing.Reader, - line: Int, - column: Int, - index: Int, - advanceAfterCheck: Bool = true, -) { - #expect(reader.mark == PureYAML.Parsing.Mark(line: line, column: column, index: index)) - if advanceAfterCheck { - reader.advance() - } -} - -private func expectScannerError( - _ yaml: String, - _ expected: PureYAML.Parsing.ParseError, -) { - expectError(expected) { - try PureYAML.Parsing.Scanner().scan(yaml) - } -}