NodetteHQ/PureYAMLStreaming
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: NodetteHQ:main
Branches Tags
NodetteHQ:main
NodetteHQ:pages
...
pull from: NodetteHQ:pages
Branches Tags
NodetteHQ:main
NodetteHQ:pages

No commits in common. "main" and "pages" have entirely different histories.
main ... pages

37 changed files with 15 additions and 1587 deletions
Show Stats Expand all files Collapse all files

12
.gitea/workflows/rules-freshness.yml → .forgejo/workflows/rules-conformance.yml
View File

@ -1,17 +1,17 @@
# Managed by rules-swift (scripts/sync-rules.sh). Verifies only that this repo's
# freshness marker matches the current shared-rules version. It does not claim
# semantic rule conformance; each consumer owns and runs its project-specific gates.
name: rules-freshness
# Managed by rules-swift (scripts/sync-rules.sh). Verifies this repo tracks the
# CURRENT shared-rules baseline. The repo's own .swiftformat/.swiftlint.yml are
# repo-specific (baseline + project rules) and are intentionally NOT byte-checked.
name: rules-conformance
on:
push:
pull_request:
workflow_dispatch:
jobs:
rules:
runs-on: ubuntu-latest
runs-on: macos
steps:
- uses: actions/checkout@v4
- name: Shared-rules freshness marker
- name: Rules baseline freshness (VERSION)
run: |
BASE=https://codeberg.org/Mihaela/rules-swift/raw/branch/main
canon=$(curl -fsSL "$BASE/VERSION" | tr -d '[:space:]')

2
.rules-version
View File

@ -1 +1 @@
2.14.0
1.1.2

21
LICENSE
View File

@ -1,21 +0,0 @@
MIT License
Copyright (c) 2026 Mihaela Mj
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

24
Package.resolved
View File

@ -1,24 +0,0 @@
{
"originHash" : "651d260da7e20e6ca89d05327ce8dc35c2397a5027fbf040bdf598ca59c18570",
"pins" : [
{
"identity" : "pureyaml",
"kind" : "remoteSourceControl",
"location" : "https://git.aleahim.com/NodetteHQ/PureYAML.git",
"state" : {
"branch" : "main",
"revision" : "2fcf54684e4216a1459cb8bd7bb14dac05b856ca"
}
},
{
"identity" : "swiftwasihttpclient",
"kind" : "remoteSourceControl",
"location" : "https://codeberg.org/Mihaela/SwiftWASIHTTPClient.git",
"state" : {
"branch" : "main",
"revision" : "66b1df6a7da8886e251c0851d1decdd32aaca72c"
}
}
],
"version" : 3
}

42
Package.swift
View File

@ -1,42 +0,0 @@
// swift-tools-version: 6.1
import PackageDescription
let package = Package(
name: "PureYAMLStreaming",
platforms: [
.macOS(.v13),
],
products: [
.library(
name: "PureYAMLStreaming",
targets: ["PureYAMLStreaming"]
),
.executable(
name: "pureyaml-streaming-wasm-smoke",
targets: ["PureYAMLStreamingWasmSmoke"]
),
],
dependencies: [
.package(url: "https://git.aleahim.com/NodetteHQ/PureYAML.git", branch: "main"),
.package(url: "https://codeberg.org/Mihaela/SwiftWASIHTTPClient.git", branch: "main"),
],
targets: [
.target(
name: "PureYAMLStreaming",
dependencies: ["PureYAML"]
),
.executableTarget(
name: "PureYAMLStreamingWasmSmoke",
dependencies: [
"PureYAMLStreaming",
"PureYAML",
.product(name: "WASIHTTPClient", package: "SwiftWASIHTTPClient"),
]
),
.testTarget(
name: "PureYAMLStreamingTests",
dependencies: ["PureYAMLStreaming"]
),
]
)

115
README.md
View File

@ -1,115 +0,0 @@
# PureYAMLStreaming
[![macOS](https://git.aleahim.com/NodetteHQ/PureYAMLStreaming/actions/workflows/macos.yml/badge.svg?branch=main)](https://git.aleahim.com/NodetteHQ/PureYAMLStreaming/actions?workflow=macos.yml)
[![Linux](https://git.aleahim.com/NodetteHQ/PureYAMLStreaming/actions/workflows/linux.yml/badge.svg?branch=main)](https://git.aleahim.com/NodetteHQ/PureYAMLStreaming/actions?workflow=linux.yml)
[![Windows](https://git.aleahim.com/NodetteHQ/PureYAMLStreaming/actions/workflows/windows.yml/badge.svg?branch=main)](https://git.aleahim.com/NodetteHQ/PureYAMLStreaming/actions?workflow=windows.yml)
[![WASM](https://git.aleahim.com/NodetteHQ/PureYAMLStreaming/actions/workflows/wasm.yml/badge.svg?branch=main)](https://git.aleahim.com/NodetteHQ/PureYAMLStreaming/actions?workflow=wasm.yml)
[![WASM Site](https://git.aleahim.com/NodetteHQ/PureYAMLStreaming/actions/workflows/wasm-site.yml/badge.svg?branch=main)](https://git.aleahim.com/NodetteHQ/PureYAMLStreaming/actions?workflow=wasm-site.yml)
Experimental full-streaming incubator for PureYAML.
This package is intentionally separate from `PureYAML` while the chunked input,
document streaming, and eventual event streaming APIs settle. It depends on
PureYAML for the stable model and parser semantics, then adds streaming input
adapters around that stable core.
## Status
Prototype.
The current implementation streams YAML input in chunks, detects document
boundaries with PureYAML's resumable token scanner
(`PureYAML.Streaming.Scanner`), and emits one parsed
`PureYAML.Stream.Document` at a time. That gives bounded memory for
multi-document streams where each document fits in memory.
Boundary detection is grammar-driven, not line-based: a `---` counts as a
boundary only where the token scanner emits a `documentStart` token, a
directive starts a new document only after an explicit `...`, and a chunk
boundary may fall anywhere, including mid-token. Splitting is lossless; the
emitted document sources concatenate back to the exact input.
A future milestone will surface token-level events to event-level consumers
directly; the public `StreamingToken` currently carries whole document
sources only.
## Memory Contract
- `parseDocuments` is bounded by the largest current document plus parser state.
- `collectDocuments` is a convenience helper and retains all returned documents.
- Anchors, aliases, merge keys, and `PureYAML.Model.Value` construction can
require retaining semantic state until the current document is complete.
- Future event-level APIs should be lower memory than document-level APIs, but
they still cannot promise constant memory for every YAML feature.
## Example
```swift
import Foundation
import PureYAMLStreaming
let url = URL(fileURLWithPath: "stream.yaml")
let parser = PureYAMLStreaming.Parser()
try parser.parseDocuments(from: url) { document in
print("document", document.index, document.value)
}
```
## WASM Browser Benchmark Site
Live site:
`https://mihaela.codeberg.page/PureYAMLStreaming/`
Deployed from the `pages` branch of the public repository
`https://codeberg.org/Mihaela/PureYAMLStreaming`, which holds only the built site
(this repository's own `pages` branch is a copy): `bash scripts/build-wasm-site.sh`,
copy `WasmSite/` (with the gzipped module, not the raw one) onto that branch, and push
it to the Codeberg repository; a webhook there triggers the git-pages redeploy.
Build the Swift WASI benchmark executable and copy it into the static test site:
```sh
bash scripts/build-wasm-site.sh
```
The build script requires `npm` so it can stage the pinned browser WASI shim as
a local `WasmSite/vendor/` asset. The deployed page does not import the shim from
a runtime CDN.
Serve the site locally:
```sh
bash scripts/serve-wasm-site.sh
```
Then open `http://localhost:8080` and click **Run Buffered Benchmark** or
**Run True Streaming Benchmark**. The page loads
`pureyaml-streaming-wasm-smoke.wasm`, lets you choose from 10+ public
multi-megabyte YAML files, and reports fetch time, Swift parse time,
throughput, document count, stdout JSON, and a timestamped run log.
Inside Swift, `SwiftWASIHTTPClient.HostHTTPClient` models that host-provided
HTTP response. The benchmark then parses the response body through
`PureYAMLStreaming.ChunkedUTF8Reader` and `ResumableDocumentScanner`, reporting
chunk count, chunk size, scanner document-source emissions, fetch time, parse
time, throughput, document count, and stdout JSON.
The buffered benchmark fetches the selected YAML file in JavaScript before
invoking the WASM guest. It keeps `SwiftWASIHTTPClient.HostHTTPClient` in the
loop to model the host-response boundary.
The experimental true-streaming benchmark requires `crossOriginIsolated`,
`SharedArrayBuffer`, `Worker`, and `Atomics.wait`. A same-origin COOP/COEP
service worker enables that mode on localhost and Codeberg Pages. In this path,
the main thread reads `fetch().body` with a `ReadableStream` reader, writes each
network chunk into a `SharedArrayBuffer` ring buffer, and a Worker-owned WASI
runtime serves those bytes through a blocking custom stdin `fd_read`. Swift then
parses stdin with `ChunkedUTF8Reader` without calling `readDataToEndOfFile()`.
## Relationship To PureYAML
`PureYAML` remains the dependency-free core package. This package is the
incubator for file, byte, and async streaming APIs. Stable pieces can move back
into PureYAML later once the API and memory behavior are proven.

157
Sources/PureYAMLStreaming/ChunkedUTF8Reader.swift
View File

@ -1,157 +0,0 @@
import Foundation
public extension PureYAMLStreaming {
/// Bounded UTF-8 chunk reader with byte-accurate mark tracking.
struct ChunkedUTF8Reader: Sendable {
private var source: Source
private let chunkSize: Int
private var pendingBytes: [UInt8] = []
private var markStorage = StreamingMark.start
private var previousWasCarriageReturn = false
public init(fileURL: URL, chunkSize: Int = 64 * 1024) throws {
guard chunkSize > 0 else {
throw StreamingError.invalidChunkSize(chunkSize)
}
source = try .file(FileHandle(forReadingFrom: fileURL))
self.chunkSize = chunkSize
}
public init(fileHandle: FileHandle, chunkSize: Int = 64 * 1024) throws {
guard chunkSize > 0 else {
throw StreamingError.invalidChunkSize(chunkSize)
}
source = .file(fileHandle)
self.chunkSize = chunkSize
}
public init(data: Data, chunkSize: Int = 64 * 1024) throws {
guard chunkSize > 0 else {
throw StreamingError.invalidChunkSize(chunkSize)
}
source = .data(data, offset: 0)
self.chunkSize = chunkSize
}
public var mark: StreamingMark {
markStorage
}
/// Reads the next valid UTF-8 chunk. A multibyte scalar split at a chunk
/// boundary is carried to the next read.
public mutating func readChunk() throws -> String? {
while true {
let bytes = try readRawBytes()
if bytes.isEmpty {
guard !pendingBytes.isEmpty else {
return nil
}
return try decodePendingAtEnd()
}
let combined = pendingBytes + bytes
let splitIndex = validPrefixLength(in: combined)
let prefix = Array(combined.prefix(splitIndex))
pendingBytes = Array(combined.dropFirst(splitIndex))
guard pendingBytes.count < 4 else {
throw StreamingError.invalidUTF8(
line: markStorage.line,
column: markStorage.column,
index: markStorage.index
)
}
guard !prefix.isEmpty else {
continue
}
guard let chunk = String(bytes: prefix, encoding: .utf8) else {
throw StreamingError.invalidUTF8(
line: markStorage.line,
column: markStorage.column,
index: markStorage.index
)
}
advanceMark(over: chunk)
return chunk
}
}
private mutating func readRawBytes() throws -> [UInt8] {
switch source {
case let .file(handle):
let data = try handle.read(upToCount: chunkSize) ?? Data()
return Array(data)
case let .data(data, offset):
guard offset < data.count else {
return []
}
let end = min(offset + chunkSize, data.count)
source = .data(data, offset: end)
return Array(data[offset ..< end])
}
}
private mutating func decodePendingAtEnd() throws -> String? {
guard let chunk = String(bytes: pendingBytes, encoding: .utf8) else {
throw StreamingError.invalidUTF8(
line: markStorage.line,
column: markStorage.column,
index: markStorage.index
)
}
pendingBytes.removeAll(keepingCapacity: true)
advanceMark(over: chunk)
return chunk.isEmpty ? nil : chunk
}
private func validPrefixLength(in bytes: [UInt8]) -> Int {
guard !bytes.isEmpty else {
return 0
}
var index = bytes.count
let lowerBound = max(0, bytes.count - 4)
while index >= lowerBound {
if String(bytes: bytes.prefix(index), encoding: .utf8) != nil {
return index
}
if index == 0 {
break
}
index -= 1
}
return 0
}
private mutating func advanceMark(over chunk: String) {
for scalar in chunk.unicodeScalars {
markStorage.index += scalar.utf8.count
switch scalar.value {
case 13:
markStorage.line += 1
markStorage.column = 1
previousWasCarriageReturn = true
case 10:
if previousWasCarriageReturn {
previousWasCarriageReturn = false
} else {
markStorage.line += 1
markStorage.column = 1
}
default:
previousWasCarriageReturn = false
markStorage.column += 1
}
}
}
}
}
extension PureYAMLStreaming.ChunkedUTF8Reader {
private enum Source: @unchecked Sendable {
case file(FileHandle)
case data(Data, offset: Int)
}
}

91
Sources/PureYAMLStreaming/Parser.swift
View File

@ -1,91 +0,0 @@
import Foundation
import PureYAML
public extension PureYAMLStreaming {
/// Chunked YAML stream parser that emits complete documents as they finish.
struct Parser: Sendable {
public var chunkSize: Int
public var pureParser: PureYAML.Parsing.Parser
public init(
chunkSize: Int = 64 * 1024,
pureParser: PureYAML.Parsing.Parser = .init()
) {
self.chunkSize = chunkSize
self.pureParser = pureParser
}
/// Parses documents from a file and calls `body` once per completed document.
///
/// This API retains only the current source document, parser state, and any
/// values PureYAML must build for that document. It does not retain prior
/// documents unless the caller does.
public func parseDocuments(
from url: URL,
_ body: (PureYAML.Stream.Document) throws -> Void
) throws {
var reader = try ChunkedUTF8Reader(fileURL: url, chunkSize: chunkSize)
try parseDocuments(from: &reader, body)
}
/// Parses documents from UTF-8 data and calls `body` once per completed document.
public func parseDocuments(
data: Data,
_ body: (PureYAML.Stream.Document) throws -> Void
) throws {
var reader = try ChunkedUTF8Reader(data: data, chunkSize: chunkSize)
try parseDocuments(from: &reader, body)
}
/// Convenience API that collects every streamed document.
///
/// This forfeits the streaming memory benefit by retaining all documents.
public func collectDocuments(from url: URL) throws -> [PureYAML.Stream.Document] {
var documents: [PureYAML.Stream.Document] = []
try parseDocuments(from: url) { document in
documents.append(document)
}
return documents
}
/// Convenience API that collects every streamed document from data.
public func collectDocuments(data: Data) throws -> [PureYAML.Stream.Document] {
var documents: [PureYAML.Stream.Document] = []
try parseDocuments(data: data) { document in
documents.append(document)
}
return documents
}
private func parseDocuments(
from reader: inout ChunkedUTF8Reader,
_ body: (PureYAML.Stream.Document) throws -> Void
) throws {
var scanner = ResumableDocumentScanner()
var nextIndex = 0
while let chunk = try reader.readChunk() {
let completed = try scanner.consume(chunk)
for source in completed {
try emit(source, index: &nextIndex, body)
}
}
for source in try scanner.finish() where !source.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
try emit(source, index: &nextIndex, body)
}
}
private func emit(
_ source: String,
index: inout Int,
_ body: (PureYAML.Stream.Document) throws -> Void
) throws {
let documents = try pureParser.parseStream(source)
for document in documents {
try body(.init(index: index, value: document.value))
index += 1
}
}
}
}

2
Sources/PureYAMLStreaming/PureYAMLStreaming.swift
View File

@ -1,2 +0,0 @@
/// Root namespace for PureYAML streaming extensions.
public enum PureYAMLStreaming {}

182
Sources/PureYAMLStreaming/ResumableDocumentScanner.swift
View File

@ -1,182 +0,0 @@
import PureYAML
public extension PureYAMLStreaming {
/// Resumable document-boundary scanner driven by `PureYAML.Streaming.Scanner`.
///
/// Chunks are tokenized by PureYAML's resumable token scanner (proven
/// byte-for-byte identical to batch scanning across the yaml-test-suite
/// corpus), and the buffered source text is cut immediately before each
/// `documentStart` token, or before the first directive token that follows
/// an explicit `...` document end. Unlike the previous line-based
/// heuristic, a `---` that the grammar does not treat as a document start
/// (for example inside a quoted scalar) can never misfire as a boundary,
/// directives stay attached to the document they introduce, and a chunk
/// boundary may fall anywhere, including mid-token.
///
/// Guarantees:
/// - Lossless: concatenating every emitted segment (from `consume(_:)` and
/// `finish()`, in order) reproduces the fed input exactly.
/// - Boundary-sound: no YAML document ever spans two segments, so parsing
/// the segments one at a time yields the same documents as batch parsing
/// the whole stream.
/// - Bounded: after each cut the token scanner is restarted on the
/// remaining buffer (a document boundary fully resets scanner state), so
/// retained state is bounded by the current document, not the stream.
struct ResumableDocumentScanner: Sendable {
private var scanner = PureYAML.Streaming.Scanner()
private var buffer: [Unicode.Scalar] = []
private var scannerFedCount = 0
private var segmentHasContent = false
private var inDirectivePreamble = false
private var afterDocumentEnd = false
private var isFinal = false
public init() {}
/// Appends a chunk of YAML source and returns every document source
/// completed by it. A chunk may end anywhere, including mid-token.
public mutating func consume(_ chunk: String) throws -> [String] {
buffer.append(contentsOf: chunk.unicodeScalars)
feedScanner()
return try drainTokens()
}
/// Like `consume(_:)`, but wraps each completed document source in a
/// ``StreamingToken``.
public mutating func consumeToken(_ chunk: String) throws -> [StreamingToken] {
try consume(chunk).map(StreamingToken.documentSource)
}
/// Marks the end of input and returns the remaining document sources:
/// any boundary that only end-of-input could confirm (for example a
/// trailing `---` line), followed by the final buffered document.
public mutating func finish() throws -> [String] {
isFinal = true
feedScanner()
scanner.finish()
var documents = try drainTokens()
if !buffer.isEmpty {
documents.append(String(String.UnicodeScalarView(buffer)))
}
self = ResumableDocumentScanner()
return documents
}
/// Feeds the token scanner every buffered scalar it has not seen yet,
/// except a trailing CR while more input may still arrive. Whether
/// that CR is a lone break or the first half of a CR LF pair is
/// undecidable until the next scalar (or end of input) is known, and
/// PureYAML's persisted plain-scalar suspension counts a pair split
/// exactly there as two line breaks, shifting every later token mark
/// down a line (observed at PureYAML commit 2fcf546). Withholding the
/// CR keeps every break the scanner sees whole, so the token line
/// numbers this type cuts by always match batch scanning.
private mutating func feedScanner() {
var target = buffer.count
if !isFinal, buffer.last == "\r" {
target -= 1
}
guard target > scannerFedCount else {
return
}
scanner.feed(String(String.UnicodeScalarView(buffer[scannerFedCount ..< target])))
scannerFedCount = target
}
private mutating func drainTokens() throws -> [String] {
var documents: [String] = []
drain: while true {
switch try scanner.next() {
case let .token(token):
handle(token, documents: &documents)
case .needMoreInput, .done:
break drain
}
}
return documents
}
private mutating func handle(_ token: PureYAML.Streaming.Token, documents: inout [String]) {
switch token.kind {
case .streamStart, .streamEnd:
return
case .documentStart:
if segmentHasContent, !inDirectivePreamble {
// The restarted scanner re-emits this token from the new
// buffer start; that re-emission sets the segment flags.
documents.append(cutSegment(beforeLine: token.range.start.line))
return
}
inDirectivePreamble = false
afterDocumentEnd = false
segmentHasContent = true
case .versionDirective, .tagDirective, .reservedDirective:
// A directive is a legal document boundary only after an
// explicit `...`. Without one, batch parsing rejects the
// stream, so the directive must stay in the current segment
// for the per-segment parse to reject it the same way.
if segmentHasContent, !inDirectivePreamble, afterDocumentEnd {
documents.append(cutSegment(beforeLine: token.range.start.line))
return
}
inDirectivePreamble = true
afterDocumentEnd = false
segmentHasContent = true
case .documentEnd:
inDirectivePreamble = false
afterDocumentEnd = true
segmentHasContent = true
default:
inDirectivePreamble = false
afterDocumentEnd = false
segmentHasContent = true
}
}
/// Cuts the buffer at the start of `line` (0-based, relative to the
/// current scanner instance, whose input always begins at `buffer[0]`),
/// emits the prefix, and restarts the token scanner on the remainder so
/// that no state older than the current document is retained.
///
/// Boundary tokens (`---` and directives) always start at column 0, so
/// the cut offset is that line's first scalar. Line counting mirrors
/// the scanner's reader exactly: CR LF is one break, and CR, LF, NEL,
/// LINE SEPARATOR, and PARAGRAPH SEPARATOR each end a line.
private mutating func cutSegment(beforeLine line: Int) -> String {
var offset = 0
var linesSeen = 0
while linesSeen < line, offset < buffer.count {
if buffer[offset] == "\r", offset + 1 < buffer.count, buffer[offset + 1] == "\n" {
offset += 2
linesSeen += 1
} else if Self.isBreak(buffer[offset]) {
offset += 1
linesSeen += 1
} else {
offset += 1
}
}
let segment = String(String.UnicodeScalarView(buffer[..<offset]))
buffer.removeFirst(offset)
segmentHasContent = false
inDirectivePreamble = false
afterDocumentEnd = false
scanner = PureYAML.Streaming.Scanner()
scannerFedCount = 0
feedScanner()
if isFinal {
scanner.finish()
}
return segment
}
/// The scanner reader's break set (libyaml's `IS_BREAK`).
private static func isBreak(_ scalar: Unicode.Scalar) -> Bool {
switch scalar {
case "\r", "\n", "\u{85}", "\u{2028}", "\u{2029}": true
default: false
}
}
}
}

21
Sources/PureYAMLStreaming/StreamingError.swift
View File

@ -1,21 +0,0 @@
import PureYAML
public extension PureYAMLStreaming {
/// Errors from the chunked streaming layer before or around PureYAML parsing.
enum StreamingError: Error, Equatable, Sendable, CustomStringConvertible {
case invalidUTF8(line: Int, column: Int, index: Int)
case invalidChunkSize(Int)
case nestedDocumentStart(line: Int, column: Int)
public var description: String {
switch self {
case let .invalidUTF8(line, column, index):
"invalid UTF-8 at line \(line), column \(column), byte \(index)"
case let .invalidChunkSize(size):
"chunk size must be greater than zero, got \(size)"
case let .nestedDocumentStart(line, column):
"document start marker found inside an open collection at line \(line), column \(column)"
}
}
}
}

16
Sources/PureYAMLStreaming/StreamingMark.swift
View File

@ -1,16 +0,0 @@
public extension PureYAMLStreaming {
/// Source position at a streaming boundary.
struct StreamingMark: Equatable, Sendable {
public var line: Int
public var column: Int
public var index: Int
public init(line: Int, column: Int, index: Int) {
self.line = line
self.column = column
self.index = index
}
public static let start = StreamingMark(line: 1, column: 1, index: 0)
}
}

6
Sources/PureYAMLStreaming/StreamingToken.swift
View File

@ -1,6 +0,0 @@
public extension PureYAMLStreaming {
/// Tokens emitted by the current streaming scanner milestone.
enum StreamingToken: Equatable, Sendable {
case documentSource(String)
}
}

201
Sources/PureYAMLStreamingWasmSmoke/main.swift
View File

@ -1,201 +0,0 @@
import Foundation
import PureYAML
import PureYAMLStreaming
import WASIHTTPClient
struct SmokeResult: Encodable {
var ok: Bool
var transport: String
var httpStatusCode: Int?
var inputByteCount: Int
var chunkSize: Int
var chunkCount: Int
var maxChunkByteCount: Int
var stdinReadCount: Int
var maxStdinReadByteCount: Int
var scannerDocumentSourceCount: Int
var documentCount: Int
var firstName: String?
var lastName: String?
var parseMilliseconds: Double
var message: String
}
struct ParseStats {
var inputByteCount = 0
var chunkCount = 0
var maxChunkByteCount = 0
var stdinReadCount = 0
var maxStdinReadByteCount = 0
var scannerDocumentSourceCount = 0
var documentCount = 0
var firstName: String?
var lastName: String?
}
let fallbackYAML = Data(
"""
---
name: browser-0
enabled: true
nested:
index: 0
---
name: browser-1
enabled: false
nested:
index: 1
---
name: browser-2
enabled: true
nested:
index: 2
""".utf8
)
do {
let arguments = Array(CommandLine.arguments.dropFirst())
let isStreamingStdin = arguments.first == "--stream-stdin"
let transport: String
let statusCode: Int?
let stats: ParseStats
let chunkSize = 31
let parseStart = Date()
if isStreamingStdin {
statusCode = arguments.dropFirst().dropFirst().first.flatMap(Int.init) ?? 200
transport = "SharedArrayBuffer fd_read stdin"
stats = try parseInstrumentedDocuments(fileHandle: FileHandle.standardInput, chunkSize: chunkSize)
} else if let hostRequestURL = arguments.first.flatMap(URL.init(string:)) {
let hostStatusCode = arguments.dropFirst().first.flatMap(Int.init)
let hostBody = FileHandle.standardInput.readDataToEndOfFile()
let responseStatusCode = hostStatusCode ?? 200
let client = HostHTTPClient { request in
guard request.method == .get else {
throw HTTPClientError.connectionFailed("benchmark only supports GET")
}
guard request.url == hostRequestURL else {
throw HTTPClientError.connectionFailed("host response did not match requested URL")
}
return HTTPResponse(statusCode: responseStatusCode, body: hostBody)
}
let response = try await client.get(hostRequestURL)
guard 200 ..< 300 ~= response.statusCode else {
throw HTTPClientError.connectionFailed("host returned HTTP \(response.statusCode)")
}
transport = "SwiftWASIHTTPClient.HostHTTPClient"
statusCode = response.statusCode
stats = try parseInstrumentedDocuments(data: response.body, chunkSize: chunkSize)
} else {
transport = "embedded fallback"
statusCode = nil
stats = try parseInstrumentedDocuments(data: fallbackYAML, chunkSize: chunkSize)
}
let parseMilliseconds = Date().timeIntervalSince(parseStart) * 1000
let ok = stats.documentCount > 0
let result = SmokeResult(
ok: ok,
transport: transport,
httpStatusCode: statusCode,
inputByteCount: stats.inputByteCount,
chunkSize: chunkSize,
chunkCount: stats.chunkCount,
maxChunkByteCount: stats.maxChunkByteCount,
stdinReadCount: stats.stdinReadCount,
maxStdinReadByteCount: stats.maxStdinReadByteCount,
scannerDocumentSourceCount: stats.scannerDocumentSourceCount,
documentCount: stats.documentCount,
firstName: stats.firstName,
lastName: stats.lastName,
parseMilliseconds: parseMilliseconds,
message: ok ? "PureYAMLStreaming WASM parse passed" : "PureYAMLStreaming WASM parse found no documents"
)
try print(renderJSON(result))
} catch {
let result = SmokeResult(
ok: false,
transport: "failed",
httpStatusCode: nil,
inputByteCount: 0,
chunkSize: 0,
chunkCount: 0,
maxChunkByteCount: 0,
stdinReadCount: 0,
maxStdinReadByteCount: 0,
scannerDocumentSourceCount: 0,
documentCount: 0,
firstName: nil,
lastName: nil,
parseMilliseconds: 0,
message: String(describing: error)
)
print((try? renderJSON(result)) ?? #"{"ok":false,"message":"failed to render error"}"#)
}
func parseInstrumentedDocuments(data: Data, chunkSize: Int) throws -> ParseStats {
var reader = try PureYAMLStreaming.ChunkedUTF8Reader(data: data, chunkSize: chunkSize)
return try parseInstrumentedDocuments(reader: &reader)
}
func parseInstrumentedDocuments(fileHandle: FileHandle, chunkSize: Int) throws -> ParseStats {
var reader = try PureYAMLStreaming.ChunkedUTF8Reader(fileHandle: fileHandle, chunkSize: chunkSize)
return try parseInstrumentedDocuments(reader: &reader)
}
func parseInstrumentedDocuments(reader: inout PureYAMLStreaming.ChunkedUTF8Reader) throws -> ParseStats {
var scanner = PureYAMLStreaming.ResumableDocumentScanner()
let pureParser = PureYAML.Parsing.Parser()
var stats = ParseStats()
while let chunk = try reader.readChunk() {
let byteCount = Data(chunk.utf8).count
stats.inputByteCount += byteCount
stats.chunkCount += 1
stats.maxChunkByteCount = max(stats.maxChunkByteCount, byteCount)
stats.stdinReadCount += 1
stats.maxStdinReadByteCount = max(stats.maxStdinReadByteCount, byteCount)
let completedSources = try scanner.consume(chunk)
stats.scannerDocumentSourceCount += completedSources.count
for source in completedSources {
try emitDocuments(from: source, parser: pureParser, stats: &stats)
}
}
for source in try scanner.finish() where !source.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
stats.scannerDocumentSourceCount += 1
try emitDocuments(from: source, parser: pureParser, stats: &stats)
}
return stats
}
func emitDocuments(
from source: String,
parser: PureYAML.Parsing.Parser,
stats: inout ParseStats
) throws {
let documents = try parser.parseStream(source)
for document in documents {
let name = stringValue("name", in: document.value)
if stats.documentCount == 0 {
stats.firstName = name
}
stats.lastName = name
stats.documentCount += 1
}
}
func stringValue(_ key: String, in value: PureYAML.Model.Value) -> String? {
guard case let .mapping(mapping) = value,
case let .string(string)? = mapping[key]
else {
return nil
}
return string
}
func renderJSON(_ value: some Encodable) throws -> String {
let encoder = JSONEncoder()
encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
return try String(decoding: encoder.encode(value), as: UTF8.self)
}

99
Tests/PureYAMLStreamingTests/DocumentScannerDifferentialTests.swift
View File

@ -1,99 +0,0 @@
import Foundation
import Testing
@testable import PureYAMLStreaming
/// Differential proof that the token-driven `ResumableDocumentScanner` emits
/// byte-identical document splits to the retired line-based heuristic on every
/// input where that heuristic was correct: the existing fixture corpus,
/// generated multi-document streams, and handwritten benign shapes. Inputs
/// where the heuristic was wrong (directives, grammar-internal `---` lines,
/// input ending on a `---` line without a newline) are covered by
/// `DocumentScannerTests` instead, which pins the corrected behavior.
@Suite("Document scanner differential")
struct DocumentScannerDifferentialTests {
/// Streams the line-based heuristic split correctly: newline-terminated,
/// directive-free, and with every column-zero `---` a real boundary.
static let benignInputs: [String] = [
"---\na: 1\n---\nb: 2\n",
"name: only\n",
"# note\n---\na: 1\n",
"---\na: 1\n# trailing\n---\nb: 2\n",
"a: 1\n--- # comment\nb: 2\n",
"a: 1\n...\nb: 2\n",
"---\na: 1\n...\n---\nb: 2\n",
"---\n---\n",
"a: 1\r\n---\r\nb: 2\r\n",
"# only comments\n\n",
"",
]
@Test("matches the line-based reference on benign streams", arguments: benignInputs)
func matchesReferenceOnBenignStreams(input: String) throws {
for chunkSize in [1, 2, 3, 7, 64, max(input.unicodeScalars.count, 1)] {
let rewired = try rewiredSegments(input, chunkSize: chunkSize)
let reference = try referenceSegments(input, chunkSize: chunkSize)
#expect(rewired == reference, "chunk size \(chunkSize) diverged for: \(input.debugDescription)")
#expect(rewired.joined() == input)
}
}
@Test("matches the line-based reference on a generated multi-document stream")
func matchesReferenceOnGeneratedStream() throws {
var input = ""
for index in 0 ..< 300 {
input += """
---
name: doc-\(index)
enabled: \(index.isMultiple(of: 2) ? "true" : "false")
nested:
index: \(index)
tags:
- real
- streaming
- chunk-\(index % 17)
"""
}
for chunkSize in [7, 31, 4096] {
let rewired = try rewiredSegments(input, chunkSize: chunkSize)
let reference = try referenceSegments(input, chunkSize: chunkSize)
#expect(rewired.count == 300)
#expect(rewired == reference, "chunk size \(chunkSize) diverged on the generated stream")
#expect(rewired.joined() == input)
}
}
@Test("matches the line-based reference across the Geekbench corpus")
func matchesReferenceAcrossCorpus() throws {
let fixtureRoot = URL(fileURLWithPath: #filePath)
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent()
.appendingPathComponent("PureYAMLGeekbench")
.appendingPathComponent("Fixtures")
.appendingPathComponent("real-yaml")
guard FileManager.default.fileExists(atPath: fixtureRoot.path) else {
return
}
let files = try FileManager.default.contentsOfDirectory(
at: fixtureRoot,
includingPropertiesForKeys: nil
).filter { url in
url.pathExtension == "yaml" || url.pathExtension == "yml"
}
#expect(files.count >= 100)
for file in files {
let input = try String(decoding: Data(contentsOf: file), as: UTF8.self)
for chunkSize in [31, 4097] {
let rewired = try rewiredSegments(input, chunkSize: chunkSize)
let reference = try referenceSegments(input, chunkSize: chunkSize)
#expect(rewired == reference, "chunk size \(chunkSize) diverged for \(file.lastPathComponent)")
#expect(rewired.joined() == input, "segments lost bytes for \(file.lastPathComponent)")
}
}
}
}

54
Tests/PureYAMLStreamingTests/DocumentScannerTestSupport.swift
View File

@ -1,54 +0,0 @@
import Foundation
import PureYAML
@testable import PureYAMLStreaming
/// The integer stored under `key` when `value` is a mapping, or nil.
func intValue(_ key: String, in value: PureYAML.Model.Value) -> Int? {
guard case let .mapping(mapping) = value,
case let .int(number)? = mapping[key]
else {
return nil
}
return number
}
/// Splits `input` into chunks of `size` Unicode scalars, so a chunk boundary
/// can land anywhere: mid-line, mid-token, and mid-grapheme.
func scalarChunks(_ input: String, size: Int) -> [String] {
precondition(size > 0)
let scalars = Array(input.unicodeScalars)
var chunks: [String] = []
var index = 0
while index < scalars.count {
let end = min(index + size, scalars.count)
chunks.append(String(String.UnicodeScalarView(scalars[index ..< end])))
index = end
}
return chunks
}
/// All document sources the rewired scanner emits for `input` fed in
/// `chunkSize`-scalar chunks, in emission order.
func rewiredSegments(_ input: String, chunkSize: Int) throws -> [String] {
var scanner = PureYAMLStreaming.ResumableDocumentScanner()
var segments: [String] = []
for chunk in scalarChunks(input, size: chunkSize) {
segments += try scanner.consume(chunk)
}
segments += try scanner.finish()
return segments
}
/// All document sources the frozen line-based reference emits for `input` fed
/// in `chunkSize`-scalar chunks, in emission order.
func referenceSegments(_ input: String, chunkSize: Int) throws -> [String] {
var scanner = LineBasedReferenceScanner()
var segments: [String] = []
for chunk in scalarChunks(input, size: chunkSize) {
segments += try scanner.consume(chunk)
}
if let final = try scanner.finish() {
segments.append(final)
}
return segments
}

162
Tests/PureYAMLStreamingTests/DocumentScannerTests.swift
View File

@ -1,162 +0,0 @@
import Foundation
import PureYAML
import Testing
@testable import PureYAMLStreaming
@Suite("Resumable document scanner")
struct DocumentScannerTests {
/// Grammar-valid streams the scanner must split without loss for any
/// chunking. Each exercises a distinct boundary shape: explicit markers,
/// no markers, comments around markers, marker with trailing comment,
/// block scalar bodies, explicit document ends, empty documents, CRLF,
/// a non-newline break character, comment-only input, and empty input.
static let chunkingInputs: [String] = [
"---\na: 1\n---\nb: 2\n",
"name: only\n",
"# note\n---\na: 1\n",
"---\na: 1\n# trailing\n---\nb: 2\n",
"a: 1\n--- # comment\nb: 2\n",
"a: |\n ---\n not a boundary\n---\nb: 2\n",
"a: 1\n...\nb: 2\n",
"---\na: 1\n...\n---\nb: 2\n",
"---\n---\n",
"a: 1\r\n---\r\nb: 2\r\n",
"a: \"x\u{2028} y\"\n---\nb: 2\n",
"# only comments\n\n",
"",
]
@Test("splits explicit documents at --- markers")
func splitsExplicitDocuments() throws {
let segments = try rewiredSegments("---\na: 1\n---\nb: 2\n", chunkSize: 1024)
#expect(segments == ["---\na: 1\n", "---\nb: 2\n"])
}
@Test("emits identical lossless segments for any chunking", arguments: chunkingInputs)
func chunkingDoesNotChangeSegments(input: String) throws {
let wholeFeed = try rewiredSegments(input, chunkSize: max(input.unicodeScalars.count, 1))
#expect(wholeFeed.joined() == input)
for chunkSize in [1, 2, 3, 7, 64] {
let segments = try rewiredSegments(input, chunkSize: chunkSize)
#expect(segments == wholeFeed, "chunk size \(chunkSize) changed the segments for: \(input.debugDescription)")
}
}
@Test("attaches leading directives to their document")
func attachesLeadingDirectives() throws {
let input = "%YAML 1.2\n---\na: 1\n"
#expect(try rewiredSegments(input, chunkSize: 3) == [input])
let documents = try PureYAMLStreaming.Parser(chunkSize: 3).collectDocuments(data: Data(input.utf8))
#expect(documents.count == 1)
#expect(intValue("a", in: documents[0].value) == 1)
}
@Test("splits before directives that follow an explicit document end")
func splitsBeforeDirectivesAfterDocumentEnd() throws {
let input = "---\na: 1\n...\n%YAML 1.2\n---\nb: 2\n"
let segments = try rewiredSegments(input, chunkSize: 5)
#expect(segments == ["---\na: 1\n...\n", "%YAML 1.2\n---\nb: 2\n"])
let documents = try PureYAMLStreaming.Parser(chunkSize: 5).collectDocuments(data: Data(input.utf8))
#expect(documents.count == 2)
}
@Test("keeps a directive without a preceding document end in the erroneous segment")
func keepsInvalidDirectiveInSegment() throws {
// Batch parsing rejects a directive that follows content without an
// explicit `...`, so the splitter must not carve out a boundary that
// would make both halves individually valid.
let input = "a: 1\n%YAML 1.2\n---\nb: 2\n"
#expect(throws: (any Error).self) {
try PureYAML.Parsing.Parser().parseStream(input)
}
let segments = try rewiredSegments(input, chunkSize: 4)
#expect(segments == [input])
#expect(throws: (any Error).self) {
try PureYAML.Parsing.Parser().parseStream(segments[0])
}
}
@Test("rejects a grammar-internal --- like batch parsing instead of splitting")
func rejectsDocumentMarkerInsideQuotedScalar() throws {
// The old line-based heuristic cut this quoted scalar in half. The
// grammar rejects a document indicator inside a quoted scalar, so the
// token-driven scanner must surface the same rejection batch parsing
// does, not fabricate a boundary.
let input = "a: \"x\n--- y\"\nb: 2\n"
#expect(throws: (any Error).self) {
try PureYAML.Parsing.Parser().parseStream(input)
}
#expect(throws: (any Error).self) {
_ = try rewiredSegments(input, chunkSize: 4)
}
}
@Test("emits a boundary confirmable only at end of input")
func emitsBoundaryAtEndOfInput() throws {
// A trailing `---` line without a newline is a document start only if
// no fourth `-` follows, which end of input alone can confirm. The old
// scanner dropped the first document here.
var scanner = PureYAMLStreaming.ResumableDocumentScanner()
let consumed = try scanner.consume("a: 1\n---")
let finished = try scanner.finish()
#expect(consumed + finished == ["a: 1\n", "---"])
let documents = try PureYAMLStreaming.Parser(chunkSize: 3).collectDocuments(data: Data("a: 1\n---".utf8))
#expect(documents.count == 2)
#expect(intValue("a", in: documents[0].value) == 1)
#expect(documents[1].value == .null)
}
@Test("preserves CRLF line endings across boundaries")
func preservesCRLF() throws {
let segments = try rewiredSegments("a: 1\r\n---\r\nb: 2\r\n", chunkSize: 2)
#expect(segments == ["a: 1\r\n", "---\r\nb: 2\r\n"])
}
@Test("counts non-newline break characters like the token scanner")
func countsUnicodeBreaks() throws {
// U+2028 is a line break to the scanner's reader, so the boundary
// line's index must account for it when mapping back to text. The
// continuation after the break is indented, as the grammar requires
// for a multi-line quoted scalar.
let segments = try rewiredSegments("a: \"x\u{2028} y\"\n---\nb: 2\n", chunkSize: 3)
#expect(segments == ["a: \"x\u{2028} y\"\n", "---\nb: 2\n"])
}
@Test("lumps a bare document after an explicit end into one segment")
func lumpsBareDocumentAfterEnd() throws {
let input = "a: 1\n...\nb: 2\n"
#expect(try rewiredSegments(input, chunkSize: 4) == [input])
let documents = try PureYAMLStreaming.Parser(chunkSize: 4).collectDocuments(data: Data(input.utf8))
#expect(documents.count == 2)
}
@Test("returns nothing for empty input")
func returnsNothingForEmptyInput() throws {
var scanner = PureYAMLStreaming.ResumableDocumentScanner()
#expect(try scanner.consume("") == [])
#expect(try scanner.finish() == [])
}
@Test("returns the raw remainder for comment-only input")
func returnsRemainderForCommentOnlyInput() throws {
let input = "# only\n\n"
#expect(try rewiredSegments(input, chunkSize: 2) == [input])
let documents = try PureYAMLStreaming.Parser(chunkSize: 2).collectDocuments(data: Data(input.utf8))
#expect(documents.isEmpty)
}
@Test("wraps document sources in streaming tokens")
func wrapsDocumentSourcesInStreamingTokens() throws {
var scanner = PureYAMLStreaming.ResumableDocumentScanner()
var tokens: [PureYAMLStreaming.StreamingToken] = []
tokens += try scanner.consumeToken("---\na: 1\n---\n")
tokens += try scanner.finish().map(PureYAMLStreaming.StreamingToken.documentSource)
#expect(tokens == [.documentSource("---\na: 1\n"), .documentSource("---\n")])
}
}

83
Tests/PureYAMLStreamingTests/LineBasedReferenceScanner.swift
View File

@ -1,83 +0,0 @@
import Foundation
/// The line-based document-boundary scanner this package shipped before the
/// token-driven rewiring, kept verbatim (minus the namespace) as a
/// differential reference. `DocumentScannerDifferentialTests` proves the
/// rewired scanner emits byte-identical segment lists on every input where
/// this heuristic was correct, so the reference must stay frozen; do not fix
/// its known defects (misfiring on grammar-internal `---` lines, splitting
/// directives away from their document, and dropping the previous document
/// when the input ends on a `---` line without a trailing newline).
struct LineBasedReferenceScanner {
private var pendingLine = ""
private var currentDocument = ""
private var hasCurrentDocumentContent = false
private var line = 1
init() {}
mutating func consume(_ chunk: String) throws -> [String] {
var documents: [String] = []
pendingLine.append(chunk)
while let newlineRange = pendingLine.rangeOfCharacter(from: .newlines) {
let lineText = String(pendingLine[..<newlineRange.lowerBound])
let newline = String(pendingLine[newlineRange])
pendingLine.removeSubrange(..<newlineRange.upperBound)
try consumeLine(lineText + newline, documents: &documents)
line += 1
}
return documents
}
mutating func finish() throws -> String? {
if !pendingLine.isEmpty {
var ignored: [String] = []
try consumeLine(pendingLine, documents: &ignored)
pendingLine.removeAll(keepingCapacity: true)
}
defer {
currentDocument.removeAll(keepingCapacity: true)
hasCurrentDocumentContent = false
}
return currentDocument.isEmpty ? nil : currentDocument
}
private mutating func consumeLine(
_ sourceLine: String,
documents: inout [String]
) throws {
if isDocumentStart(sourceLine), hasCurrentDocumentContent {
documents.append(currentDocument)
currentDocument = sourceLine
hasCurrentDocumentContent = true
return
}
currentDocument.append(sourceLine)
if hasSemanticContent(sourceLine) {
hasCurrentDocumentContent = true
}
}
private func isDocumentStart(_ sourceLine: String) -> Bool {
let trimmedNewline = sourceLine.trimmingCharacters(in: .newlines)
guard trimmedNewline.hasPrefix("---") else {
return false
}
let markerEnd = trimmedNewline.index(trimmedNewline.startIndex, offsetBy: 3)
if markerEnd != trimmedNewline.endIndex {
let next = trimmedNewline[markerEnd]
guard next == " " || next == "\t" || next == "#" else {
return false
}
}
return trimmedNewline.prefix(3) == "---"
}
private func hasSemanticContent(_ sourceLine: String) -> Bool {
let trimmed = sourceLine.trimmingCharacters(in: .whitespacesAndNewlines)
return !trimmed.isEmpty && !trimmed.hasPrefix("#")
}
}

157
Tests/PureYAMLStreamingTests/StreamingTests.swift
View File

@ -1,157 +0,0 @@
import Foundation
import PureYAML
import Testing
@testable import PureYAMLStreaming
@Suite("PureYAML Streaming")
struct StreamingTests {
@Test("chunked reader preserves UTF-8 split across chunks")
func chunkedReaderPreservesSplitUTF8() throws {
var reader = try PureYAMLStreaming.ChunkedUTF8Reader(
data: Data("aé\nb".utf8),
chunkSize: 2
)
var chunks: [String] = []
while let chunk = try reader.readChunk() {
chunks.append(chunk)
}
#expect(chunks.joined() == "aé\nb")
#expect(reader.mark == .init(line: 2, column: 2, index: 5))
}
@Test("document scanner emits documents across chunk boundaries")
func scannerEmitsDocumentsAcrossChunks() throws {
var scanner = PureYAMLStreaming.ResumableDocumentScanner()
var documents: [String] = []
documents += try scanner.consume("---\na: 1\n--")
documents += try scanner.consume("-\nb: 2\n")
documents += try scanner.finish()
#expect(documents == ["---\na: 1\n", "---\nb: 2\n"])
}
@Test("parser streams documents in order")
func parserStreamsDocumentsInOrder() throws {
let yaml = Data(
"""
---
name: one
---
name: two
""".utf8
)
let parser = PureYAMLStreaming.Parser(chunkSize: 5)
let documents = try parser.collectDocuments(data: yaml)
#expect(documents.count == 2)
#expect(documents[0].index == 0)
#expect(documents[1].index == 1)
#expect(stringValue("name", in: documents[0].value) == "one")
#expect(stringValue("name", in: documents[1].value) == "two")
}
@Test("streaming matches PureYAML on Geekbench fixtures")
func streamingMatchesGeekbenchFixtures() throws {
let fixtureRoot = geekbenchFixtureRoot()
guard FileManager.default.fileExists(atPath: fixtureRoot.path) else {
return
}
let files = try FileManager.default.contentsOfDirectory(
at: fixtureRoot,
includingPropertiesForKeys: nil
).filter { url in
url.pathExtension == "yaml" || url.pathExtension == "yml"
}
#expect(files.count >= 100)
let streamingParser = PureYAMLStreaming.Parser(chunkSize: 31)
let pureParser = PureYAML.Parsing.Parser()
for file in files {
let data = try Data(contentsOf: file)
var streamed: [PureYAML.Model.Value] = []
try streamingParser.parseDocuments(from: file) { document in
streamed.append(document.value)
}
let direct = try pureParser.parseStream(String(decoding: data, as: UTF8.self)).map(\.value)
#expect(streamed == direct)
}
}
@Test("parser streams generated large file without collecting documents")
func parserStreamsGeneratedLargeFileWithoutCollectingDocuments() throws {
let fileURL = FileManager.default.temporaryDirectory
.appendingPathComponent("pureyaml-stream-\(UUID().uuidString).yaml")
try writeGeneratedStream(to: fileURL, documentCount: 5000)
defer {
try? FileManager.default.removeItem(at: fileURL)
}
let parser = PureYAMLStreaming.Parser(chunkSize: 37)
var count = 0
var firstName: String?
var lastName: String?
try parser.parseDocuments(from: fileURL) { document in
let name = stringValue("name", in: document.value)
if count == 0 {
firstName = name
}
lastName = name
#expect(document.index == count)
count += 1
}
#expect(count == 5000)
#expect(firstName == "doc-0")
#expect(lastName == "doc-4999")
}
private func stringValue(_ key: String, in value: PureYAML.Model.Value) -> String? {
guard case let .mapping(mapping) = value,
case let .string(string)? = mapping[key]
else {
return nil
}
return string
}
private func geekbenchFixtureRoot() -> URL {
URL(fileURLWithPath: #filePath)
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent()
.appendingPathComponent("PureYAMLGeekbench")
.appendingPathComponent("Fixtures")
.appendingPathComponent("real-yaml")
}
private func writeGeneratedStream(to fileURL: URL, documentCount: Int) throws {
FileManager.default.createFile(atPath: fileURL.path, contents: nil)
let handle = try FileHandle(forWritingTo: fileURL)
defer {
try? handle.close()
}
for index in 0 ..< documentCount {
let document = """
---
name: doc-\(index)
enabled: \(index.isMultiple(of: 2) ? "true" : "false")
nested:
index: \(index)
tags:
- real
- streaming
- chunk-\(index % 17)
"""
try handle.write(contentsOf: Data(document.utf8))
}
}
}

0
WasmSite/coi-serviceworker.js → coi-serviceworker.js
View File

64
docs/ARCHITECTURE.md
View File

@ -1,64 +0,0 @@
# Streaming Architecture
PureYAMLStreaming separates three concerns:
1. `ChunkedUTF8Reader` pulls bounded byte chunks from a file or data source and
tracks byte offsets plus line and column marks.
2. `ResumableDocumentScanner` consumes chunks, tokenizes them with
`PureYAML.Streaming.Scanner`, and emits one document source at a time, cut
immediately before each `documentStart` token or before the first directive
token that follows an explicit `...` document end.
3. `Parser` parses each emitted document through PureYAML and hands completed
documents to the caller.
```mermaid
flowchart TD
fileSource["File or Data"] --> chunkedReader["Chunked UTF8 Reader"]
chunkedReader --> documentScanner["Resumable Document Scanner"]
documentScanner --> pureParser["PureYAML Parser"]
pureParser --> documentSink["Document Sink"]
```
## Token-Driven Boundary Detection
PureYAML ships a resumable token scanner as public API
(`PureYAML.Streaming.Scanner`), proven byte-for-byte identical to batch
scanning across the yaml-test-suite corpus at multiple chunk sizes.
`ResumableDocumentScanner` drives it token by token and cuts the buffered
source text at grammar-confirmed boundaries, so a `---` inside a quoted
scalar can never misfire as a document split and directives stay attached to
the document they introduce. Chunk boundaries may fall anywhere, including
mid-token; the token scanner suspends and resumes across them.
Two implementation details keep the contract tight:
- After each cut the token scanner is restarted on the remaining buffer. A
document boundary fully resets scanner state, and the restart bounds
retained state by the current document rather than the whole stream (the
scanner's reader keeps everything it was ever fed).
- A trailing CR is withheld from the token scanner while more input may still
arrive, so it never sees a CR LF pair split at a suspension point. Split
exactly there, the persisted plain-scalar suspension counts the pair as two
line breaks and every later token mark drifts down a line (observed at
PureYAML commit `2fcf546`). Withholding the CR keeps the token line numbers
the cuts rely on identical to batch scanning.
Parsing stays document-at-a-time: each emitted segment goes through the
stable batch `PureYAML` entry point, which keeps the parser semantics
identical to non-streaming use.
## Future Event Streaming
The next milestone can feed token-level events to a public event sink
directly instead of emitting whole document sources.
The intended final shape is:
```mermaid
flowchart TD
bytes["Async Bytes"] --> utf8Reader["Chunked UTF8 Reader"]
utf8Reader --> tokenScanner["Resumable Token Scanner"]
tokenScanner --> eventParser["Token Event Parser"]
eventParser --> eventSink["Event Sink"]
eventParser --> documentComposer["Optional Document Composer"]
```

0
WasmSite/index.html → index.html
View File

0
WasmSite/main.js → main.js
View File

BIN
pureyaml-streaming-wasm-smoke.wasm.gz Normal file
View File

Binary file not shown.

68
scripts/build-wasm-site.sh
View File

@ -1,68 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SDK_ID="${PUREYAML_STREAMING_WASM_SDK:-swift-6.3.2-RELEASE_wasm}"
SWIFT_SELECTOR="${PUREYAML_STREAMING_WASM_SWIFT_SELECTOR:-+6.3.2}"
PRODUCT="pureyaml-streaming-wasm-smoke"
SITE_DIR="$ROOT_DIR/WasmSite"
VENDOR_DIR="$SITE_DIR/vendor/browser_wasi_shim"
swift_command() {
if command -v swiftly >/dev/null 2>&1; then
swiftly run swift "$@" "$SWIFT_SELECTOR"
else
swift "$@"
fi
}
if ! swift_command sdk list | grep -qx "$SDK_ID"; then
echo "wasm: missing Swift SDK '$SDK_ID'" >&2
echo "wasm: install the SDK or set PUREYAML_STREAMING_WASM_SDK" >&2
exit 2
fi
swift_command build \
--package-path "$ROOT_DIR" \
--swift-sdk "$SDK_ID" \
-c release \
--product "$PRODUCT"
BIN_PATH="$(swift_command build \
--package-path "$ROOT_DIR" \
--swift-sdk "$SDK_ID" \
-c release \
--show-bin-path)"
WASM_PATH="$BIN_PATH/$PRODUCT.wasm"
STAGED_WASM="$SITE_DIR/$PRODUCT.wasm"
if command -v wasm-opt >/dev/null 2>&1; then
wasm-opt -Oz --strip-debug --strip-producers "$WASM_PATH" -o "$STAGED_WASM"
else
cp "$WASM_PATH" "$STAGED_WASM"
fi
gzip -9 -c "$STAGED_WASM" > "$STAGED_WASM.gz"
if command -v npm >/dev/null 2>&1; then
TMP_NPM_DIR="$(mktemp -d)"
trap 'rm -rf "$TMP_NPM_DIR"' EXIT
(
cd "$TMP_NPM_DIR"
npm init -y >/dev/null
npm install @bjorn3/browser_wasi_shim@0.3.0 >/dev/null
)
rm -rf "$VENDOR_DIR"
mkdir -p "$VENDOR_DIR"
cp "$TMP_NPM_DIR/node_modules/@bjorn3/browser_wasi_shim/dist/"*.js "$VENDOR_DIR/"
else
echo "wasm: npm is required to vendor browser_wasi_shim" >&2
exit 2
fi
echo "WASM test site ready:"
echo " $SITE_DIR/index.html"
echo " $STAGED_WASM.gz"
echo
echo "Serve it with:"
echo " bash scripts/serve-wasm-site.sh"

7
scripts/check.sh
View File

@ -1,7 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
swift build --package-path "$ROOT_DIR"
swift test --package-path "$ROOT_DIR"

8
scripts/serve-wasm-site.sh
View File

@ -1,8 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PORT="${1:-8080}"
cd "$ROOT_DIR/WasmSite"
python3 -m http.server "$PORT"

0
WasmSite/stream-worker.js → stream-worker.js
View File

0
WasmSite/style.css → style.css
View File

1
vendor/browser_wasi_shim/debug.js vendored Normal file
View File

@ -0,0 +1 @@
let Debug=class Debug{enable(enabled){this.log=createLogger(enabled===undefined?true:enabled,this.prefix)}get enabled(){return this.isEnabled}constructor(isEnabled){this.isEnabled=isEnabled;this.prefix="wasi:";this.enable(isEnabled)}};function createLogger(enabled,prefix){if(enabled){const a=console.log.bind(console,"%c%s","color: #265BA0",prefix);return a}else{return()=>{}}}export const debug=new Debug(false);

1
vendor/browser_wasi_shim/fd.js vendored Normal file
View File

@ -0,0 +1 @@
import*as wasi from"./wasi_defs.js";export class Fd{fd_allocate(offset,len){return wasi.ERRNO_NOTSUP}fd_close(){return 0}fd_fdstat_get(){return{ret:wasi.ERRNO_NOTSUP,fdstat:null}}fd_fdstat_set_flags(flags){return wasi.ERRNO_NOTSUP}fd_fdstat_set_rights(fs_rights_base,fs_rights_inheriting){return wasi.ERRNO_NOTSUP}fd_filestat_get(){return{ret:wasi.ERRNO_NOTSUP,filestat:null}}fd_filestat_set_size(size){return wasi.ERRNO_NOTSUP}fd_filestat_set_times(atim,mtim,fst_flags){return wasi.ERRNO_NOTSUP}fd_pread(size,offset){return{ret:wasi.ERRNO_NOTSUP,data:new Uint8Array}}fd_prestat_get(){return{ret:wasi.ERRNO_NOTSUP,prestat:null}}fd_pwrite(data,offset){return{ret:wasi.ERRNO_NOTSUP,nwritten:0}}fd_read(size){return{ret:wasi.ERRNO_NOTSUP,data:new Uint8Array}}fd_readdir_single(cookie){return{ret:wasi.ERRNO_NOTSUP,dirent:null}}fd_seek(offset,whence){return{ret:wasi.ERRNO_NOTSUP,offset:0n}}fd_sync(){return 0}fd_tell(){return{ret:wasi.ERRNO_NOTSUP,offset:0n}}fd_write(data){return{ret:wasi.ERRNO_NOTSUP,nwritten:0}}path_create_directory(path){return wasi.ERRNO_NOTSUP}path_filestat_get(flags,path){return{ret:wasi.ERRNO_NOTSUP,filestat:null}}path_filestat_set_times(flags,path,atim,mtim,fst_flags){return wasi.ERRNO_NOTSUP}path_link(path,inode,allow_dir){return wasi.ERRNO_NOTSUP}path_unlink(path){return{ret:wasi.ERRNO_NOTSUP,inode_obj:null}}path_lookup(path,dirflags){return{ret:wasi.ERRNO_NOTSUP,inode_obj:null}}path_open(dirflags,path,oflags,fs_rights_base,fs_rights_inheriting,fd_flags){return{ret:wasi.ERRNO_NOTDIR,fd_obj:null}}path_readlink(path){return{ret:wasi.ERRNO_NOTSUP,data:null}}path_remove_directory(path){return wasi.ERRNO_NOTSUP}path_rename(old_path,new_fd,new_path){return wasi.ERRNO_NOTSUP}path_unlink_file(path){return wasi.ERRNO_NOTSUP}}export class Inode{}

1
vendor/browser_wasi_shim/fs_mem.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
vendor/browser_wasi_shim/fs_opfs.js vendored Normal file
View File

@ -0,0 +1 @@
import*as wasi from"./wasi_defs.js";import{Fd,Inode}from"./fd.js";export class SyncOPFSFile extends Inode{path_open(oflags,fs_rights_base,fd_flags){if(this.readonly&&(fs_rights_base&BigInt(wasi.RIGHTS_FD_WRITE))==BigInt(wasi.RIGHTS_FD_WRITE)){return{ret:wasi.ERRNO_PERM,fd_obj:null}}if((oflags&wasi.OFLAGS_TRUNC)==wasi.OFLAGS_TRUNC){if(this.readonly)return{ret:wasi.ERRNO_PERM,fd_obj:null};this.handle.truncate(0)}const file=new OpenSyncOPFSFile(this);if(fd_flags&wasi.FDFLAGS_APPEND)file.fd_seek(0n,wasi.WHENCE_END);return{ret:wasi.ERRNO_SUCCESS,fd_obj:file}}get size(){return BigInt(this.handle.getSize())}stat(){return new wasi.Filestat(wasi.FILETYPE_REGULAR_FILE,this.size)}constructor(handle,options){super();this.handle=handle;this.readonly=!!options?.readonly}}export class OpenSyncOPFSFile extends Fd{fd_allocate(offset,len){if(BigInt(this.file.handle.getSize())>offset+len){}else{this.file.handle.truncate(Number(offset+len))}return wasi.ERRNO_SUCCESS}fd_fdstat_get(){return{ret:0,fdstat:new wasi.Fdstat(wasi.FILETYPE_REGULAR_FILE,0)}}fd_filestat_get(){return{ret:0,filestat:new wasi.Filestat(wasi.FILETYPE_REGULAR_FILE,BigInt(this.file.handle.getSize()))}}fd_filestat_set_size(size){this.file.handle.truncate(Number(size));return wasi.ERRNO_SUCCESS}fd_read(size){const buf=new Uint8Array(size);const n=this.file.handle.read(buf,{at:Number(this.position)});this.position+=BigInt(n);return{ret:0,data:buf.slice(0,n)}}fd_seek(offset,whence){let calculated_offset;switch(whence){case wasi.WHENCE_SET:calculated_offset=BigInt(offset);break;case wasi.WHENCE_CUR:calculated_offset=this.position+BigInt(offset);break;case wasi.WHENCE_END:calculated_offset=BigInt(this.file.handle.getSize())+BigInt(offset);break;default:return{ret:wasi.ERRNO_INVAL,offset:0n}}if(calculated_offset<0){return{ret:wasi.ERRNO_INVAL,offset:0n}}this.position=calculated_offset;return{ret:wasi.ERRNO_SUCCESS,offset:this.position}}fd_write(data){if(this.file.readonly)return{ret:wasi.ERRNO_BADF,nwritten:0};const n=this.file.handle.write(data,{at:Number(this.position)});this.position+=BigInt(n);return{ret:wasi.ERRNO_SUCCESS,nwritten:n}}fd_sync(){this.file.handle.flush();return wasi.ERRNO_SUCCESS}constructor(file){super();this.position=0n;this.file=file}}

1
vendor/browser_wasi_shim/index.js vendored Normal file
View File

@ -0,0 +1 @@
import WASI,{WASIProcExit}from"./wasi.js";export{WASI,WASIProcExit};export{Fd,Inode}from"./fd.js";export{File,Directory,OpenFile,OpenDirectory,PreopenDirectory,ConsoleStdout}from"./fs_mem.js";export{SyncOPFSFile,OpenSyncOPFSFile}from"./fs_opfs.js";export{strace}from"./strace.js";export*as wasi from"./wasi_defs.js";

1
vendor/browser_wasi_shim/strace.js vendored Normal file
View File

@ -0,0 +1 @@
export function strace(imports,no_trace){return new Proxy(imports,{get(target,prop,receiver){const f=Reflect.get(target,prop,receiver);if(no_trace.includes(prop)){return f}return function(...args){console.log(prop,"(",...args,")");const result=Reflect.apply(f,receiver,args);console.log(" =",result);return result}}})}

1
vendor/browser_wasi_shim/wasi.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
vendor/browser_wasi_shim/wasi_defs.js vendored Normal file
View File

File diff suppressed because one or more lines are too long