diff --git a/CHANGELOG.md b/CHANGELOG.md index 43ea3a9..0fe6115 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,54 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -Nothing yet. +### Added + +- Broaden the real-seed mutation property suite (issue #6) with three mutation + classes named in that issue but not yet covered: `invalidControlCharacter` + (a raw control byte injected into a line), `mixedTabSpaceIndentation` (a tab + after leading spaces, distinct from the existing pure-tab case), and + `truncatedFlowCollection` (an unterminated `[a, b, c` appended after the real + seed sample, exercising the deep parser's `malformed` error rather than the + preflight scanner). The test now also asserts diagnostic quality, not just + pass/fail: every matching diagnostic's `line` must fall within the mutated + document, and its `column`, when present, must be 1-based. Some `ParseError` + cases (`.tabIndentation`) are line-only by design, so column is checked when + present rather than required. + +- Add a resumable, one-token-at-a-time scanner entry point, `Scanner.scanStep()` + (internal), the first increment of true chunked input streaming (issue #4). + `Reader` gains `isFinal`/`append(_:)`/`markFinal()`: past the buffered end while + `!isFinal`, `peek(_:)` flags `ranDry` instead of asserting true end of input. + `scanStep()` snapshots the scanner before each `fetchNextToken()` and restores it + in full whenever a fetch runs dry, so a fetch spanning a chunk boundary is + retried from scratch after more input arrives rather than misread as an early + end of input. Every existing call site keeps `isFinal: true` (the default), so + `scanAll()` and all batch parsing are unchanged; `ranDry` can never be set when + `isFinal` is true, by construction, not just by test. + A token already queued is not always safe to release: libyaml's simple-key + mechanism can still splice a KEY token in front of it once a later `:` is found, + so `scanStep()` ports libyaml's `yaml_parser_fetch_more_tokens` release + condition, holding the queue front until no possible simple key still targets + that position. + Proof: `Tests/StreamingScannerDifferentialTests.swift` drives `scanStep()` at + several chunk sizes (1, 2, 3, 7, 64, whole-input) over representative cases + always, and over the entire yaml-test-suite manifest (402 cases) behind + `PUREYAML_RUN_YAML_TEST_SUITE=1`, asserting the resulting token sequence (or + thrown error) is byte-identical to `scanAll()`. + That proof surfaced a real bug, now fixed: `lastTokenForbidsInlineSequence()` + (`LibYAML.Scanner.Guards.swift`) read `indentEngine.tokens.last?.type` to check + whether a `key: - a` value indicator bars an inline block sequence. Batch mode + never dequeues, so the queue never empties and `.last` always works; `scanStep()` + releases and discards tokens as soon as they are safe, so the queue can be empty + exactly when this check runs. `IndentEngine` now tracks `lastToken` explicitly, + kept in sync with `tokens.last` after every enqueue/insert/roll/unroll, which is + provably identical to the old `tokens.last` read in batch mode (nothing is ever + dequeued there) and survives dequeuing in streaming mode. + Not yet wired into `PureYAMLStreaming`: `scanStep()` and its supporting types are + internal, and true mid-scalar suspension (a plain/block/flow scalar spanning a + chunk boundary currently re-scans from the last released token rather than + resuming mid-token) is deferred, tracked at + . ## [0.2.0] - 2026-07-05 diff --git a/Sources/Parsing/LibYAML.IndentEngine.swift b/Sources/Parsing/LibYAML.IndentEngine.swift index 1ef725f..b6c05d1 100644 --- a/Sources/Parsing/LibYAML.IndentEngine.swift +++ b/Sources/Parsing/LibYAML.IndentEngine.swift @@ -19,6 +19,15 @@ extension PureYAML.Parsing.LibYAML { /// `parser->tokens_parsed`, the count already dequeued, used to turn an /// absolute token number into an index into the live queue. var tokensParsed = 0 + /// The most recently enqueued or spliced token, kept in sync with + /// `tokens.last` after every mutation below. Batch scanning never dequeues + /// (`tokensParsed` stays 0, `dequeue()` unused), so `tokens.last` alone + /// would suffice there; this field exists for a resumable scanner + /// (`Scanner.scanStep()`), which releases and discards tokens as soon as + /// they are safe to hand out, so `tokens` can be empty exactly when a check + /// like `lastTokenForbidsInlineSequence()` needs to know what the last + /// token was. + private(set) var lastToken: Token? /// Port of `yaml_parser_roll_indent`. In flow context it does nothing. /// When the column is greater than the current indent, it pushes the @@ -44,6 +53,7 @@ extension PureYAML.Parsing.LibYAML { } else { tokens.insert(token, at: number - tokensParsed) } + lastToken = tokens.last } } @@ -59,17 +69,20 @@ extension PureYAML.Parsing.LibYAML { tokens.append(Token(type: .blockEnd, startMark: mark, endMark: mark)) indent = indents.removeLast() } + lastToken = tokens.last ?? lastToken } /// `ENQUEUE`: append a token to the queue. mutating func enqueue(_ token: Token) { tokens.append(token) + lastToken = token } /// `QUEUE_INSERT`: insert a token at an absolute token number, used to /// splice a `KEY` token in front of a node already queued. mutating func insert(_ token: Token, atTokenNumber number: Int) { tokens.insert(token, at: number - tokensParsed) + lastToken = tokens.last } /// Dequeue the front token, advancing `tokensParsed` like libyaml's diff --git a/Sources/Parsing/LibYAML.Reader.swift b/Sources/Parsing/LibYAML.Reader.swift index 611af5b..9522463 100644 --- a/Sources/Parsing/LibYAML.Reader.swift +++ b/Sources/Parsing/LibYAML.Reader.swift @@ -6,20 +6,51 @@ extension PureYAML.Parsing.LibYAML { /// libyaml's 0-based line and column convention; conversion to the package's /// 1-based `Mark` happens at token emission during integration. struct Reader { - private let scalars: [Unicode.Scalar] + private var scalars: [Unicode.Scalar] private(set) var index = 0 private(set) var line = 0 private(set) var column = 0 + /// Whether the buffered `scalars` are the complete input. True for every + /// batch caller (the default), so `ranDry` can never be set and this type's + /// behavior is unchanged for `scanAll()`. A resumable caller passes `false` + /// and later calls `markFinal()` once the last chunk has been appended. + private(set) var isFinal: Bool + /// Set by `peek(_:)` when it was asked for a position past the buffered + /// scalars while `!isFinal`: the query is inconclusive, not a real end of + /// input, and the caller must suspend and retry after `append(_:)`. Batch + /// callers (`isFinal == true`) never set this, since the guard below is + /// unreachable when `isFinal` is true. + private(set) var ranDry = false - init(_ string: String) { + init(_ string: String = "", isFinal: Bool = true) { scalars = Array(string.unicodeScalars) + self.isFinal = isFinal } /// `PEEK_TOKEN` / `CHECK_AT`: the scalar `offset` ahead, or nil past the - /// end (libyaml's `\0`). - func peek(_ offset: Int = 0) -> Unicode.Scalar? { + /// end (libyaml's `\0`). Past the buffered end while more chunks may still + /// arrive, this flags `ranDry` instead of asserting true end of input. + mutating func peek(_ offset: Int = 0) -> Unicode.Scalar? { let position = index + offset - return position < scalars.count ? scalars[position] : nil + if position < scalars.count { + return scalars[position] + } + if !isFinal { + ranDry = true + } + return nil + } + + /// Append a chunk of newly arrived input. Only meaningful before + /// `markFinal()`. + mutating func append(_ chunk: String) { + scalars.append(contentsOf: chunk.unicodeScalars) + } + + /// Declare that no further chunks are coming: a `peek` past the buffered + /// end is now a real end of input, matching the batch reader's behavior. + mutating func markFinal() { + isFinal = true } var isAtEnd: Bool { diff --git a/Sources/Parsing/LibYAML.Scanner.BlockScalar.swift b/Sources/Parsing/LibYAML.Scanner.BlockScalar.swift index 28faf61..d09dd48 100644 --- a/Sources/Parsing/LibYAML.Scanner.BlockScalar.swift +++ b/Sources/Parsing/LibYAML.Scanner.BlockScalar.swift @@ -161,7 +161,7 @@ extension PureYAML.Parsing.LibYAML.Scanner { /// Fold the pending breaks at the start of a content line (folded style joins a /// single break between non-blank lines into a space; otherwise breaks are /// kept). - private func foldBlockLine(_ state: inout BlockState, literal: Bool) { + private mutating func foldBlockLine(_ state: inout BlockState, literal: Bool) { let trailingBlank = Classify.isBlank(reader.peek()) if !literal, state.leadingBreak.first == "\n", !state.leadingBlank, !trailingBlank { if state.trailingBreaks.isEmpty { @@ -184,7 +184,7 @@ extension PureYAML.Parsing.LibYAML.Scanner { /// Whether a `---` or `...` document marker (three repeats then a blank, break, /// or end of input) starts at the current position. - private func documentMarkerAhead() -> Bool { + private mutating func documentMarkerAhead() -> Bool { let first = reader.peek() guard first == "-" || first == "." else { return false diff --git a/Sources/Parsing/LibYAML.Scanner.Fetch.swift b/Sources/Parsing/LibYAML.Scanner.Fetch.swift index 6db7da3..b9426c5 100644 --- a/Sources/Parsing/LibYAML.Scanner.Fetch.swift +++ b/Sources/Parsing/LibYAML.Scanner.Fetch.swift @@ -352,7 +352,7 @@ extension PureYAML.Parsing.LibYAML.Scanner { indentEngine.enqueue(token) } - private func isDocumentMarker(_ character: Unicode.Scalar) -> Bool { + private mutating func isDocumentMarker(_ character: Unicode.Scalar) -> Bool { reader.peek(0) == character && reader.peek(1) == character && reader.peek(2) == character diff --git a/Sources/Parsing/LibYAML.Scanner.FlowScalar.swift b/Sources/Parsing/LibYAML.Scanner.FlowScalar.swift index d4f6f04..945457b 100644 --- a/Sources/Parsing/LibYAML.Scanner.FlowScalar.swift +++ b/Sources/Parsing/LibYAML.Scanner.FlowScalar.swift @@ -215,7 +215,7 @@ extension PureYAML.Parsing.LibYAML.Scanner { PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a quoted scalar", problem: problem, mark: mark) } - private func documentMarkerAhead() -> Bool { + private mutating func documentMarkerAhead() -> Bool { let first = reader.peek() guard first == "-" || first == "." else { return false diff --git a/Sources/Parsing/LibYAML.Scanner.Guards.swift b/Sources/Parsing/LibYAML.Scanner.Guards.swift index 6108499..1f949ee 100644 --- a/Sources/Parsing/LibYAML.Scanner.Guards.swift +++ b/Sources/Parsing/LibYAML.Scanner.Guards.swift @@ -29,7 +29,7 @@ extension PureYAML.Parsing.LibYAML.Scanner { /// same line: a mapping value indicator or a node property (anchor/tag). After /// another block entry a compact sequence is allowed, so that is not barred. func lastTokenForbidsInlineSequence() -> Bool { - switch indentEngine.tokens.last?.type { + switch indentEngine.lastToken?.type { case .anchor, .tag: true case .value: diff --git a/Sources/Parsing/LibYAML.Scanner.PlainScalar.swift b/Sources/Parsing/LibYAML.Scanner.PlainScalar.swift index 900e14c..a6cbe15 100644 --- a/Sources/Parsing/LibYAML.Scanner.PlainScalar.swift +++ b/Sources/Parsing/LibYAML.Scanner.PlainScalar.swift @@ -132,7 +132,7 @@ extension PureYAML.Parsing.LibYAML.Scanner { PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a plain scalar", problem: problem, mark: mark) } - private func documentMarkerAhead() -> Bool { + private mutating func documentMarkerAhead() -> Bool { let first = reader.peek() guard first == "-" || first == "." else { return false diff --git a/Sources/Parsing/LibYAML.Scanner.SkipToToken.swift b/Sources/Parsing/LibYAML.Scanner.SkipToToken.swift index a210daa..3e18a91 100644 --- a/Sources/Parsing/LibYAML.Scanner.SkipToToken.swift +++ b/Sources/Parsing/LibYAML.Scanner.SkipToToken.swift @@ -78,7 +78,7 @@ extension PureYAML.Parsing.LibYAML.Scanner { /// Whether the rest of the current line, from the current position, is only /// spaces and tabs up to a line break or end of stream (a blank line). - func blankLineAhead() -> Bool { + mutating func blankLineAhead() -> Bool { var offset = 0 while let character = reader.peek(offset), character == " " || Classify.isTab(character) { offset += 1 diff --git a/Sources/Parsing/LibYAML.Scanner.Streaming.swift b/Sources/Parsing/LibYAML.Scanner.Streaming.swift new file mode 100644 index 0000000..2e71534 --- /dev/null +++ b/Sources/Parsing/LibYAML.Scanner.Streaming.swift @@ -0,0 +1,59 @@ +extension PureYAML.Parsing.LibYAML.Scanner { + /// The result of one `scanStep()` call. + enum ScanOutcome: Equatable { + /// A token is safe to release to the caller now. + case token(PureYAML.Parsing.LibYAML.Token) + /// The buffered input was exhausted mid-fetch; append more input (via + /// `reader.append(_:)`) and call `scanStep()` again. The scanner's state is + /// unchanged from before this call, as if it had not been called. + case needMoreInput + /// STREAM-END has been released and no further tokens remain. + case done + } + + /// Resumable, one-token-at-a-time counterpart to `scanAll()`. Where `scanAll()` + /// assumes the full input is already buffered, `scanStep()` may be driven across + /// a `Reader` still receiving chunks (`isFinal == false`): a fetch that runs off + /// the buffered end is rolled back in full (as if it had not been attempted) + /// rather than mistaken for true end of input. + /// + /// A token already queued is not always safe to hand out immediately: libyaml's + /// simple-key mechanism can still splice a KEY (and a BLOCK-MAPPING-START) in + /// front of it once a later `:` is found (`saveSimpleKey`/`insert(_:atTokenNumber:)` + /// in `LibYAML.Scanner.swift`/`LibYAML.IndentEngine.swift`). Port of libyaml's + /// `yaml_parser_fetch_more_tokens` release condition: the token at the queue + /// front (position `tokensParsed`) is safe to release only when no currently + /// possible simple key (at any flow level) is still recorded at that exact + /// position; such a key could still resolve into a splice targeting that slot. + mutating func scanStep() throws -> ScanOutcome { + while true { + if isFrontSafeToRelease, let token = indentEngine.dequeue() { + return .token(token) + } + if streamEndProduced { + return .done + } + let checkpoint = self + do { + try fetchNextToken() + } catch { + if !reader.isFinal, reader.ranDry { + self = checkpoint + return .needMoreInput + } + throw error + } + if !reader.isFinal, reader.ranDry { + self = checkpoint + return .needMoreInput + } + } + } + + private var isFrontSafeToRelease: Bool { + guard !indentEngine.tokens.isEmpty else { + return false + } + return !simpleKeys.contains { $0.possible && $0.tokenNumber == indentEngine.tokensParsed } + } +} diff --git a/Sources/Parsing/LibYAML.Scanner.swift b/Sources/Parsing/LibYAML.Scanner.swift index b0e2676..a32334e 100644 --- a/Sources/Parsing/LibYAML.Scanner.swift +++ b/Sources/Parsing/LibYAML.Scanner.swift @@ -43,8 +43,8 @@ extension PureYAML.Parsing.LibYAML { /// `parser->stream_end_produced`. var streamEndProduced = false - init(_ input: String = "") { - reader = Reader(input) + init(_ input: String = "", isFinal: Bool = true) { + reader = Reader(input, isFinal: isFinal) } /// `parser->flow_level`, kept on the indent engine. diff --git a/Tests/RealYAMLCorpusPropertyTests.swift b/Tests/RealYAMLCorpusPropertyTests.swift index 0966a9c..e134e99 100644 --- a/Tests/RealYAMLCorpusPropertyTests.swift +++ b/Tests/RealYAMLCorpusPropertyTests.swift @@ -30,15 +30,31 @@ struct RealYAMLCorpusPropertyTests { let source = try seed.load() for mutation in PropertyMutation.allCases { let mutated = mutation.apply(to: source, seedID: seed.id) + let mutatedLineCount = mutated.split(separator: "\n", omittingEmptySubsequences: false).count let report = PureYAML.diagnosticValidationReport(mutated, file: "\(seed.id)-\(mutation.rawValue).yaml") #expect(!report.diagnostics.isEmpty, "\(seed.id) \(mutation.rawValue) produced no diagnostics") #expect(report.diagnostics.allSatisfy { $0.kind == .parse }) let codes = report.diagnostics.map { $0.code ?? "nil" }.joined(separator: ",") + let matching = report.diagnostics.filter { $0.code == mutation.expectedCode } #expect( - report.diagnostics.contains { $0.code == mutation.expectedCode }, + !matching.isEmpty, "\(seed.id) \(mutation.rawValue) expected \(mutation.expectedCode), got \(codes)", ) + for diagnostic in matching { + #expect(!(diagnostic.reason.isEmpty), "\(seed.id) \(mutation.rawValue) has an empty reason") + let line = try #require(diagnostic.line, "\(seed.id) \(mutation.rawValue) is missing a line") + #expect( + (1 ... mutatedLineCount).contains(line), + "\(seed.id) \(mutation.rawValue) line \(line) is out of range 1...\(mutatedLineCount)", + ) + // Some `ParseError` cases (e.g. `.tabIndentation`) are line-only by + // design; a preflight-scanner diagnostic for the same code carries a + // column too. Check it when present rather than requiring it always. + if let column = diagnostic.column { + #expect(column >= 1, "\(seed.id) \(mutation.rawValue) column \(column) is not 1-based") + } + } #expect(!report.description.isEmpty) #expect(!report.jsonDescription(title: "Generated Corpus Mutation").isEmpty) } @@ -92,12 +108,15 @@ private enum PropertyMutation: String, CaseIterable { case missingSequenceSpace case unterminatedQuotedString case undefinedAlias + case invalidControlCharacter + case mixedTabSpaceIndentation + case truncatedFlowCollection var expectedCode: String { switch self { case .missingMappingSpace: "missingMappingSpace" - case .tabIndentation: + case .tabIndentation, .mixedTabSpaceIndentation: "tabIndentation" case .missingSequenceSpace: "missingSequenceSpace" @@ -105,6 +124,32 @@ private enum PropertyMutation: String, CaseIterable { "unterminatedQuotedString" case .undefinedAlias: "undefinedAlias" + case .invalidControlCharacter: + "invalidControlCharacter" + case .truncatedFlowCollection: + "malformed" + } + } + + /// Mutations whose broken content is appended after the embedded, untouched + /// real-seed sample rather than before it: an unterminated construct must be + /// the last thing in the document, since anything appended after it would + /// either close it (defeating the mutation) or itself become unparseable + /// content inside it (making the resulting diagnostic unpredictable). + private var isAppendedAfterSample: Bool { + self == .unterminatedQuotedString || self == .truncatedFlowCollection + } + + /// The broken line for a mutation whose content is appended after the + /// sample. Only meaningful when `isAppendedAfterSample` is true. + private var appendedSuffix: String { + switch self { + case .unterminatedQuotedString: + "mutation: \"unterminated scalar" + case .truncatedFlowCollection: + "mutation: [a, b, c" + default: + "" } } @@ -112,12 +157,12 @@ private enum PropertyMutation: String, CaseIterable { let sample = source.split(separator: "\n", omittingEmptySubsequences: false) .prefix(3) .joined(separator: "\n") - if self == .unterminatedQuotedString { + if isAppendedAfterSample { return """ # mutation-source: \(seedID) sample: | \(sample.replacingOccurrences(of: "\n", with: "\n ")) - mutation: "unterminated scalar + \(appendedSuffix) """ } @@ -128,10 +173,14 @@ private enum PropertyMutation: String, CaseIterable { "\tmutation: tab indentation" case .missingSequenceSpace: "-mutation sequence marker" - case .unterminatedQuotedString: - "mutation: \"unterminated scalar" case .undefinedAlias: "mutation: *undefinedMutationAlias" + case .invalidControlCharacter: + "mutation: bad\u{0}value" + case .mixedTabSpaceIndentation: + " \tmutation: mixed tab and space indentation" + case .unterminatedQuotedString, .truncatedFlowCollection: + "" } return """ # mutation-source: \(seedID) diff --git a/Tests/StreamingScannerDifferentialTests.swift b/Tests/StreamingScannerDifferentialTests.swift new file mode 100644 index 0000000..e4867aa --- /dev/null +++ b/Tests/StreamingScannerDifferentialTests.swift @@ -0,0 +1,121 @@ +import Foundation +@testable import PureYAML +import Testing + +/// Differential proof for the resumable `Scanner.scanStep()` (issue #4's first +/// increment: a checkpoint-and-retry wrapper around the existing batch +/// `fetchNextToken()`, see `LibYAML.Scanner.Streaming.swift`). The scanner is a +/// pure value type, so `scanStep()` reuses every fetch function unmodified; the +/// only new behavior is (1) `Reader` flags `ranDry` instead of asserting end of +/// input when `!isFinal`, and (2) tokens are released one at a time only when no +/// possible simple key still targets the queue front. Neither is exercised by +/// `scanAll()`, so this is genuinely new code path, not a refactor, and needs its +/// own proof: driving the same input through `scanStep()` at several chunk sizes +/// must reproduce `scanAll()`'s token array (or thrown error) exactly. +@Suite("Streaming scanner differential") +struct StreamingScannerDifferentialTests { + /// A handful of representative always-on cases so `swift test` gets real + /// coverage without the opt-in whole-suite flag: a plain scalar split + /// mid-token, a block mapping, a flow collection, an anchor/alias pair, and + /// a deliberately malformed input (an unterminated quoted scalar). + @Test("Representative cases match scanAll() at several chunk sizes", arguments: [ + "title: Example API", + """ + title: Example + servers: + - url: / + """, + "flow: {a: 1, b: [2, 3]}", + """ + base: &anchor + value: 1 + derived: *anchor + """, + "block: |\n line one\n line two\n", + "title: \"unterminated", + ]) + func test_representativeCasesMatchScanAllAtSeveralChunkSizes(yaml: String) throws { + try assertStreamingMatchesBatch(yaml) + } + + @Test("Every yaml-test-suite case matches scanAll() at several chunk sizes") + func test_wholeSuiteMatchesScanAllAtSeveralChunkSizes() throws { + guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else { + return + } + let manifest = try YAMLTestSuiteManifest.load() + var mismatches: [String] = [] + for testCase in manifest.cases { + let yaml = try testCase.loadInput() + do { + try assertStreamingMatchesBatch(yaml) + } catch { + mismatches.append("\(testCase.id): \(error)") + } + } + #expect(mismatches.isEmpty, "streaming/batch mismatches (\(mismatches.count)): \(mismatches.prefix(10).joined(separator: "; "))") + } +} + +/// Runs the batch scanner once as the reference, then the streaming scanner at +/// several chunk sizes (including feeding the whole input as a single "chunk", +/// which should degenerate to one dry run followed by success), and asserts each +/// streaming run reproduces the reference token array or thrown error exactly. +private func assertStreamingMatchesBatch(_ yaml: String) throws { + let reference = batchScan(yaml) + for chunkSize in [1, 2, 3, 7, 64, yaml.unicodeScalars.count + 1] { + let observed = try streamingScan(yaml, chunkSize: chunkSize) + #expect(observed == reference, "chunk size \(chunkSize) diverged from scanAll() for: \(yaml.debugDescription)") + } +} + +private enum ScanResult: Equatable { + case tokens([PureYAML.Parsing.LibYAML.Token]) + case threw(String) +} + +private func batchScan(_ yaml: String) -> ScanResult { + var scanner = PureYAML.Parsing.LibYAML.Scanner(yaml) + do { + return try .tokens(scanner.scanAll()) + } catch { + return .threw("\(error)") + } +} + +/// Drives `scanStep()` over `yaml` split into `chunkSize`-scalar pieces, feeding +/// one chunk whenever the scanner reports `.needMoreInput`, and marking the +/// reader final once every chunk has been appended. +private func streamingScan(_ yaml: String, chunkSize: Int) throws -> ScanResult { + let scalars = Array(yaml.unicodeScalars) + var chunks: [String] = [] + var index = 0 + while index < scalars.count { + let end = min(index + chunkSize, scalars.count) + chunks.append(String(String.UnicodeScalarView(scalars[index ..< end]))) + index = end + } + + var scanner = PureYAML.Parsing.LibYAML.Scanner("", isFinal: false) + var nextChunk = 0 + var tokens: [PureYAML.Parsing.LibYAML.Token] = [] + do { + while true { + switch try scanner.scanStep() { + case let .token(token): + tokens.append(token) + case .done: + return .tokens(tokens) + case .needMoreInput: + if nextChunk < chunks.count { + scanner.reader.append(chunks[nextChunk]) + nextChunk += 1 + } else { + scanner.reader.markFinal() + } + } + } + } catch { + return .threw("\(error)") + } +}