PureYAMLStreaming/Tests/PureYAMLStreamingTests/DocumentScannerTests.swift

import Foundation
import PureYAML
import Testing
@testable import PureYAMLStreaming

@Suite("Resumable document scanner")
struct DocumentScannerTests {
    /// Grammar-valid streams the scanner must split without loss for any
    /// chunking. Each exercises a distinct boundary shape: explicit markers,
    /// no markers, comments around markers, marker with trailing comment,
    /// block scalar bodies, explicit document ends, empty documents, CRLF,
    /// a non-newline break character, comment-only input, and empty input.
    static let chunkingInputs: [String] = [
        "---\na: 1\n---\nb: 2\n",
        "name: only\n",
        "# note\n---\na: 1\n",
        "---\na: 1\n# trailing\n---\nb: 2\n",
        "a: 1\n--- # comment\nb: 2\n",
        "a: |\n  ---\n  not a boundary\n---\nb: 2\n",
        "a: 1\n...\nb: 2\n",
        "---\na: 1\n...\n---\nb: 2\n",
        "---\n---\n",
        "a: 1\r\n---\r\nb: 2\r\n",
        "a: \"x\u{2028} y\"\n---\nb: 2\n",
        "# only comments\n\n",
        "",
    ]

    @Test("splits explicit documents at --- markers")
    func splitsExplicitDocuments() throws {
        let segments = try rewiredSegments("---\na: 1\n---\nb: 2\n", chunkSize: 1024)
        #expect(segments == ["---\na: 1\n", "---\nb: 2\n"])
    }

    @Test("emits identical lossless segments for any chunking", arguments: chunkingInputs)
    func chunkingDoesNotChangeSegments(input: String) throws {
        let wholeFeed = try rewiredSegments(input, chunkSize: max(input.unicodeScalars.count, 1))
        #expect(wholeFeed.joined() == input)
        for chunkSize in [1, 2, 3, 7, 64] {
            let segments = try rewiredSegments(input, chunkSize: chunkSize)
            #expect(segments == wholeFeed, "chunk size \(chunkSize) changed the segments for: \(input.debugDescription)")
        }
    }

    @Test("attaches leading directives to their document")
    func attachesLeadingDirectives() throws {
        let input = "%YAML 1.2\n---\na: 1\n"
        #expect(try rewiredSegments(input, chunkSize: 3) == [input])

        let documents = try PureYAMLStreaming.Parser(chunkSize: 3).collectDocuments(data: Data(input.utf8))
        #expect(documents.count == 1)
        #expect(intValue("a", in: documents[0].value) == 1)
    }

    @Test("splits before directives that follow an explicit document end")
    func splitsBeforeDirectivesAfterDocumentEnd() throws {
        let input = "---\na: 1\n...\n%YAML 1.2\n---\nb: 2\n"
        let segments = try rewiredSegments(input, chunkSize: 5)
        #expect(segments == ["---\na: 1\n...\n", "%YAML 1.2\n---\nb: 2\n"])

        let documents = try PureYAMLStreaming.Parser(chunkSize: 5).collectDocuments(data: Data(input.utf8))
        #expect(documents.count == 2)
    }

    @Test("keeps a directive without a preceding document end in the erroneous segment")
    func keepsInvalidDirectiveInSegment() throws {
        // Batch parsing rejects a directive that follows content without an
        // explicit `...`, so the splitter must not carve out a boundary that
        // would make both halves individually valid.
        let input = "a: 1\n%YAML 1.2\n---\nb: 2\n"
        #expect(throws: (any Error).self) {
            try PureYAML.Parsing.Parser().parseStream(input)
        }

        let segments = try rewiredSegments(input, chunkSize: 4)
        #expect(segments == [input])
        #expect(throws: (any Error).self) {
            try PureYAML.Parsing.Parser().parseStream(segments[0])
        }
    }

    @Test("rejects a grammar-internal --- like batch parsing instead of splitting")
    func rejectsDocumentMarkerInsideQuotedScalar() throws {
        // The old line-based heuristic cut this quoted scalar in half. The
        // grammar rejects a document indicator inside a quoted scalar, so the
        // token-driven scanner must surface the same rejection batch parsing
        // does, not fabricate a boundary.
        let input = "a: \"x\n--- y\"\nb: 2\n"
        #expect(throws: (any Error).self) {
            try PureYAML.Parsing.Parser().parseStream(input)
        }
        #expect(throws: (any Error).self) {
            _ = try rewiredSegments(input, chunkSize: 4)
        }
    }

    @Test("emits a boundary confirmable only at end of input")
    func emitsBoundaryAtEndOfInput() throws {
        // A trailing `---` line without a newline is a document start only if
        // no fourth `-` follows, which end of input alone can confirm. The old
        // scanner dropped the first document here.
        var scanner = PureYAMLStreaming.ResumableDocumentScanner()
        let consumed = try scanner.consume("a: 1\n---")
        let finished = try scanner.finish()
        #expect(consumed + finished == ["a: 1\n", "---"])

        let documents = try PureYAMLStreaming.Parser(chunkSize: 3).collectDocuments(data: Data("a: 1\n---".utf8))
        #expect(documents.count == 2)
        #expect(intValue("a", in: documents[0].value) == 1)
        #expect(documents[1].value == .null)
    }

    @Test("preserves CRLF line endings across boundaries")
    func preservesCRLF() throws {
        let segments = try rewiredSegments("a: 1\r\n---\r\nb: 2\r\n", chunkSize: 2)
        #expect(segments == ["a: 1\r\n", "---\r\nb: 2\r\n"])
    }

    @Test("counts non-newline break characters like the token scanner")
    func countsUnicodeBreaks() throws {
        // U+2028 is a line break to the scanner's reader, so the boundary
        // line's index must account for it when mapping back to text. The
        // continuation after the break is indented, as the grammar requires
        // for a multi-line quoted scalar.
        let segments = try rewiredSegments("a: \"x\u{2028} y\"\n---\nb: 2\n", chunkSize: 3)
        #expect(segments == ["a: \"x\u{2028} y\"\n", "---\nb: 2\n"])
    }

    @Test("lumps a bare document after an explicit end into one segment")
    func lumpsBareDocumentAfterEnd() throws {
        let input = "a: 1\n...\nb: 2\n"
        #expect(try rewiredSegments(input, chunkSize: 4) == [input])

        let documents = try PureYAMLStreaming.Parser(chunkSize: 4).collectDocuments(data: Data(input.utf8))
        #expect(documents.count == 2)
    }

    @Test("returns nothing for empty input")
    func returnsNothingForEmptyInput() throws {
        var scanner = PureYAMLStreaming.ResumableDocumentScanner()
        #expect(try scanner.consume("") == [])
        #expect(try scanner.finish() == [])
    }

    @Test("returns the raw remainder for comment-only input")
    func returnsRemainderForCommentOnlyInput() throws {
        let input = "# only\n\n"
        #expect(try rewiredSegments(input, chunkSize: 2) == [input])

        let documents = try PureYAMLStreaming.Parser(chunkSize: 2).collectDocuments(data: Data(input.utf8))
        #expect(documents.isEmpty)
    }

    @Test("wraps document sources in streaming tokens")
    func wrapsDocumentSourcesInStreamingTokens() throws {
        var scanner = PureYAMLStreaming.ResumableDocumentScanner()
        var tokens: [PureYAMLStreaming.StreamingToken] = []
        tokens += try scanner.consumeToken("---\na: 1\n---\n")
        tokens += try scanner.finish().map(PureYAMLStreaming.StreamingToken.documentSource)
        #expect(tokens == [.documentSource("---\na: 1\n"), .documentSource("---\n")])
    }
}