From 9298536e52d3538788b3bc915a1755610b1fbe36 Mon Sep 17 00:00:00 2001
From: Mihaela M <mihaelamj@me.com>
Date: Fri, 26 Jun 2026 17:43:19 +0200
Subject: [PATCH] test: measure conformance against the official YAML
 test-suite

Adds an opt-in harness that runs PureYAML over the canonical YAML 1.2
corpus (github.com/yaml/yaml-test-suite) and reports a parse-level pass
rate: a should-parse case must parse, a should-error case must be
rejected. The data is fetched into a gitignored location by
scripts/fetch-yaml-test-suite.sh, never vendored, and the run is gated
behind PUREYAML_RUN_YAML_TEST_SUITE=1 so the default suite is unaffected.

This turns 'best YAML library' from an assertion into a tracked number.
Baseline: 211/333 (63.4%) parse-level: 177/255 valid documents parse,
34/78 invalid documents are rejected. So 78 valid docs are not yet parsed
and 44 invalid docs are wrongly accepted. The stricter value-level check
against each case's in.json is the next step.
---
 CHANGELOG.md                              | 12 ++++
 Tests/YAMLTestSuiteConformanceTests.swift | 85 +++++++++++++++++++++++
 scripts/fetch-yaml-test-suite.sh          | 23 ++++++
 3 files changed, 120 insertions(+)
 create mode 100644 Tests/YAMLTestSuiteConformanceTests.swift
 create mode 100755 scripts/fetch-yaml-test-suite.sh

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1bc2169..aae63f9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- An opt-in conformance harness against the official YAML test-suite, the canonical
+  YAML 1.2 corpus (`Tests/YAMLTestSuiteConformanceTests.swift`,
+  `scripts/fetch-yaml-test-suite.sh`). Gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1`
+  and a separate fetch, so the data is never vendored. It reports parse-level
+  conformance (a should-parse case must parse; a should-error case must be
+  rejected). Current baseline: 211/333 (63.4%), with 78 valid documents not yet
+  parsed and 44 invalid documents not yet rejected. This turns "best YAML library"
+  from an assertion into a tracked number; the stricter value-level check against
+  each case's `in.json` is the next step.
+
 ### Changed
 
 - Split multi-type source files so each file declares exactly one top-level type
diff --git a/Tests/YAMLTestSuiteConformanceTests.swift b/Tests/YAMLTestSuiteConformanceTests.swift
new file mode 100644
index 0000000..6fc299a
--- /dev/null
+++ b/Tests/YAMLTestSuiteConformanceTests.swift
@@ -0,0 +1,85 @@
+import Foundation
+@testable import PureYAML
+import Testing
+
+/// Measures PureYAML against the official YAML test-suite (https://github.com/yaml/yaml-test-suite),
+/// the canonical YAML 1.2 conformance corpus. Opt-in, like the real-world corpus suite: the data is
+/// large and fetched separately, so the run is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1` and
+/// `bash scripts/fetch-yaml-test-suite.sh`. It reports parse-level conformance: a should-parse case
+/// must parse and a should-error case (one shipping an `error` marker) must be rejected. The
+/// stricter value-level check, comparing the parsed tree to each case's `in.json`, is the next step.
+/// The suite prints the pass rate to stderr and asserts only that the corpus was found, so it is a
+/// measurement harness, not yet a threshold gate (the threshold is set once the number is known).
+@Suite("YAML test-suite conformance")
+struct YAMLTestSuiteConformanceTests {
+    private struct Case {
+        let id: String
+        let input: String
+        let expectsError: Bool
+    }
+
+    private func loadCases() -> [Case]? {
+        let manager = FileManager.default
+        let root = ProcessInfo.processInfo.environment["PUREYAML_YAML_TEST_SUITE_DIR"]
+            ?? manager.currentDirectoryPath + "/.build/yaml-test-suite"
+        guard let names = try? manager.contentsOfDirectory(atPath: root) else {
+            return nil
+        }
+        var cases: [Case] = []
+        for name in names.sorted() {
+            let dir = root + "/" + name
+            guard let input = try? String(contentsOfFile: dir + "/in.yaml", encoding: .utf8) else {
+                continue
+            }
+            cases.append(Case(
+                id: name,
+                input: input,
+                expectsError: manager.fileExists(atPath: dir + "/error"),
+            ))
+        }
+        return cases
+    }
+
+    @Test("Parse-level conformance against the official yaml-test-suite")
+    func parseLevelConformance() throws {
+        guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
+            return
+        }
+        let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh")
+        var okExpected = 0, okPassed = 0, errExpected = 0, errPassed = 0
+        var shouldParseFailures: [String] = []
+        var shouldRejectFailures: [String] = []
+        for testCase in cases {
+            let parsed = (try? PureYAML.parseStream(testCase.input)) != nil
+            if testCase.expectsError {
+                errExpected += 1
+                if parsed { shouldRejectFailures.append(testCase.id) } else { errPassed += 1 }
+            } else {
+                okExpected += 1
+                if parsed { okPassed += 1 } else { shouldParseFailures.append(testCase.id) }
+            }
+        }
+        let total = okExpected + errExpected
+        let passed = okPassed + errPassed
+        report("""
+        YAML test-suite parse-level: \(passed)/\(total) (\(percent(passed, total)))
+          should-parse handled:  \(okPassed)/\(okExpected) (\(percent(okPassed, okExpected)))
+          should-error rejected: \(errPassed)/\(errExpected) (\(percent(errPassed, errExpected)))
+          should-parse failures (\(shouldParseFailures.count)): \(sample(shouldParseFailures))
+          should-error misses  (\(shouldRejectFailures.count)): \(sample(shouldRejectFailures))
+        """)
+        #expect(total > 0)
+    }
+
+    private func percent(_ part: Int, _ whole: Int) -> String {
+        whole == 0 ? "n/a" : String(format: "%.1f%%", Double(part) / Double(whole) * 100)
+    }
+
+    private func sample(_ ids: [String]) -> String {
+        ids.prefix(25).joined(separator: " ") + (ids.count > 25 ? " …" : "")
+    }
+
+    private func report(_ text: String) {
+        FileHandle.standardError.write(Data("\n\(text)\n".utf8))
+    }
+}
diff --git a/scripts/fetch-yaml-test-suite.sh b/scripts/fetch-yaml-test-suite.sh
new file mode 100755
index 0000000..ebf26ef
--- /dev/null
+++ b/scripts/fetch-yaml-test-suite.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Fetch the official YAML test-suite (the canonical YAML 1.2 conformance corpus) into a gitignored
+# location for the opt-in conformance run. The data is not vendored, keeping the package free of
+# external content; this script puts it under .build/ (ignored) like the other fetched corpora.
+#
+# Usage:
+#   bash scripts/fetch-yaml-test-suite.sh
+#   PUREYAML_RUN_YAML_TEST_SUITE=1 swift test --filter YAMLTestSuiteConformanceTests
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+DEST="$ROOT_DIR/.build/yaml-test-suite"
+
+if [ -d "$DEST/.git" ]; then
+    git -C "$DEST" fetch --quiet --depth 1 origin data
+    git -C "$DEST" reset --hard --quiet FETCH_HEAD
+else
+    rm -rf "$DEST"
+    git clone --quiet --depth 1 --branch data https://github.com/yaml/yaml-test-suite "$DEST"
+fi
+
+cases="$(find "$DEST" -maxdepth 2 -name in.yaml | wc -l | tr -d ' ')"
+echo "yaml-test-suite ready at $DEST ($cases cases)"