Mihaela Mihaljevic
b4253fd289
Replace the line-based --- heuristic with a scan over PureYAML's resumable token scanner, so a --- inside a quoted scalar can no longer misfire as a document boundary and directives stay attached to the document they introduce. Add a differential suite that checks the new scanner against the old line-based reference across generated streams and the full Geekbench corpus.
100 lines
4.1 KiB
Swift
100 lines
4.1 KiB
Swift
import Foundation
|
|
import Testing
|
|
@testable import PureYAMLStreaming
|
|
|
|
/// Differential proof that the token-driven `ResumableDocumentScanner` emits
|
|
/// byte-identical document splits to the retired line-based heuristic on every
|
|
/// input where that heuristic was correct: the existing fixture corpus,
|
|
/// generated multi-document streams, and handwritten benign shapes. Inputs
|
|
/// where the heuristic was wrong (directives, grammar-internal `---` lines,
|
|
/// input ending on a `---` line without a newline) are covered by
|
|
/// `DocumentScannerTests` instead, which pins the corrected behavior.
|
|
@Suite("Document scanner differential")
|
|
struct DocumentScannerDifferentialTests {
|
|
/// Streams the line-based heuristic split correctly: newline-terminated,
|
|
/// directive-free, and with every column-zero `---` a real boundary.
|
|
static let benignInputs: [String] = [
|
|
"---\na: 1\n---\nb: 2\n",
|
|
"name: only\n",
|
|
"# note\n---\na: 1\n",
|
|
"---\na: 1\n# trailing\n---\nb: 2\n",
|
|
"a: 1\n--- # comment\nb: 2\n",
|
|
"a: 1\n...\nb: 2\n",
|
|
"---\na: 1\n...\n---\nb: 2\n",
|
|
"---\n---\n",
|
|
"a: 1\r\n---\r\nb: 2\r\n",
|
|
"# only comments\n\n",
|
|
"",
|
|
]
|
|
|
|
@Test("matches the line-based reference on benign streams", arguments: benignInputs)
|
|
func matchesReferenceOnBenignStreams(input: String) throws {
|
|
for chunkSize in [1, 2, 3, 7, 64, max(input.unicodeScalars.count, 1)] {
|
|
let rewired = try rewiredSegments(input, chunkSize: chunkSize)
|
|
let reference = try referenceSegments(input, chunkSize: chunkSize)
|
|
#expect(rewired == reference, "chunk size \(chunkSize) diverged for: \(input.debugDescription)")
|
|
#expect(rewired.joined() == input)
|
|
}
|
|
}
|
|
|
|
@Test("matches the line-based reference on a generated multi-document stream")
|
|
func matchesReferenceOnGeneratedStream() throws {
|
|
var input = ""
|
|
for index in 0 ..< 300 {
|
|
input += """
|
|
---
|
|
name: doc-\(index)
|
|
enabled: \(index.isMultiple(of: 2) ? "true" : "false")
|
|
nested:
|
|
index: \(index)
|
|
tags:
|
|
- real
|
|
- streaming
|
|
- chunk-\(index % 17)
|
|
|
|
"""
|
|
}
|
|
|
|
for chunkSize in [7, 31, 4096] {
|
|
let rewired = try rewiredSegments(input, chunkSize: chunkSize)
|
|
let reference = try referenceSegments(input, chunkSize: chunkSize)
|
|
#expect(rewired.count == 300)
|
|
#expect(rewired == reference, "chunk size \(chunkSize) diverged on the generated stream")
|
|
#expect(rewired.joined() == input)
|
|
}
|
|
}
|
|
|
|
@Test("matches the line-based reference across the Geekbench corpus")
|
|
func matchesReferenceAcrossCorpus() throws {
|
|
let fixtureRoot = URL(fileURLWithPath: #filePath)
|
|
.deletingLastPathComponent()
|
|
.deletingLastPathComponent()
|
|
.deletingLastPathComponent()
|
|
.deletingLastPathComponent()
|
|
.appendingPathComponent("PureYAMLGeekbench")
|
|
.appendingPathComponent("Fixtures")
|
|
.appendingPathComponent("real-yaml")
|
|
guard FileManager.default.fileExists(atPath: fixtureRoot.path) else {
|
|
return
|
|
}
|
|
|
|
let files = try FileManager.default.contentsOfDirectory(
|
|
at: fixtureRoot,
|
|
includingPropertiesForKeys: nil
|
|
).filter { url in
|
|
url.pathExtension == "yaml" || url.pathExtension == "yml"
|
|
}
|
|
#expect(files.count >= 100)
|
|
|
|
for file in files {
|
|
let input = try String(decoding: Data(contentsOf: file), as: UTF8.self)
|
|
for chunkSize in [31, 4097] {
|
|
let rewired = try rewiredSegments(input, chunkSize: chunkSize)
|
|
let reference = try referenceSegments(input, chunkSize: chunkSize)
|
|
#expect(rewired == reference, "chunk size \(chunkSize) diverged for \(file.lastPathComponent)")
|
|
#expect(rewired.joined() == input, "segments lost bytes for \(file.lastPathComponent)")
|
|
}
|
|
}
|
|
}
|
|
}
|