Mihaela Mihaljevic
b4253fd289
Replace the line-based --- heuristic with a scan over PureYAML's resumable token scanner, so a --- inside a quoted scalar can no longer misfire as a document boundary and directives stay attached to the document they introduce. Add a differential suite that checks the new scanner against the old line-based reference across generated streams and the full Geekbench corpus.
84 lines
3.0 KiB
Swift
84 lines
3.0 KiB
Swift
import Foundation
|
|
|
|
/// The line-based document-boundary scanner this package shipped before the
|
|
/// token-driven rewiring, kept verbatim (minus the namespace) as a
|
|
/// differential reference. `DocumentScannerDifferentialTests` proves the
|
|
/// rewired scanner emits byte-identical segment lists on every input where
|
|
/// this heuristic was correct, so the reference must stay frozen; do not fix
|
|
/// its known defects (misfiring on grammar-internal `---` lines, splitting
|
|
/// directives away from their document, and dropping the previous document
|
|
/// when the input ends on a `---` line without a trailing newline).
|
|
struct LineBasedReferenceScanner {
|
|
private var pendingLine = ""
|
|
private var currentDocument = ""
|
|
private var hasCurrentDocumentContent = false
|
|
private var line = 1
|
|
|
|
init() {}
|
|
|
|
mutating func consume(_ chunk: String) throws -> [String] {
|
|
var documents: [String] = []
|
|
pendingLine.append(chunk)
|
|
|
|
while let newlineRange = pendingLine.rangeOfCharacter(from: .newlines) {
|
|
let lineText = String(pendingLine[..<newlineRange.lowerBound])
|
|
let newline = String(pendingLine[newlineRange])
|
|
pendingLine.removeSubrange(..<newlineRange.upperBound)
|
|
try consumeLine(lineText + newline, documents: &documents)
|
|
line += 1
|
|
}
|
|
|
|
return documents
|
|
}
|
|
|
|
mutating func finish() throws -> String? {
|
|
if !pendingLine.isEmpty {
|
|
var ignored: [String] = []
|
|
try consumeLine(pendingLine, documents: &ignored)
|
|
pendingLine.removeAll(keepingCapacity: true)
|
|
}
|
|
defer {
|
|
currentDocument.removeAll(keepingCapacity: true)
|
|
hasCurrentDocumentContent = false
|
|
}
|
|
return currentDocument.isEmpty ? nil : currentDocument
|
|
}
|
|
|
|
private mutating func consumeLine(
|
|
_ sourceLine: String,
|
|
documents: inout [String]
|
|
) throws {
|
|
if isDocumentStart(sourceLine), hasCurrentDocumentContent {
|
|
documents.append(currentDocument)
|
|
currentDocument = sourceLine
|
|
hasCurrentDocumentContent = true
|
|
return
|
|
}
|
|
|
|
currentDocument.append(sourceLine)
|
|
if hasSemanticContent(sourceLine) {
|
|
hasCurrentDocumentContent = true
|
|
}
|
|
}
|
|
|
|
private func isDocumentStart(_ sourceLine: String) -> Bool {
|
|
let trimmedNewline = sourceLine.trimmingCharacters(in: .newlines)
|
|
guard trimmedNewline.hasPrefix("---") else {
|
|
return false
|
|
}
|
|
let markerEnd = trimmedNewline.index(trimmedNewline.startIndex, offsetBy: 3)
|
|
if markerEnd != trimmedNewline.endIndex {
|
|
let next = trimmedNewline[markerEnd]
|
|
guard next == " " || next == "\t" || next == "#" else {
|
|
return false
|
|
}
|
|
}
|
|
return trimmedNewline.prefix(3) == "---"
|
|
}
|
|
|
|
private func hasSemanticContent(_ sourceLine: String) -> Bool {
|
|
let trimmed = sourceLine.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
return !trimmed.isEmpty && !trimmed.hasPrefix("#")
|
|
}
|
|
}
|