YAMLTestSuiteConformanceTests resolved a JSON bool vs number with CFGetTypeID and CFBooleanGetTypeID, CoreFoundation functions absent on Linux and Windows, so the test target failed to compile there and the Linux and Windows CI gates were red. Make JSONNode Decodable and read the expected in.json with JSONDecoder, which distinguishes a bool from a number natively and reads top-level fragments via a single-value container. Value-level conformance is unchanged at 206/206 on macOS, and the parse no longer touches CoreFoundation.
216 lines
9.8 KiB
Swift
216 lines
9.8 KiB
Swift
import Foundation
|
|
@testable import PureYAML
|
|
import Testing
|
|
|
|
/// Measures PureYAML against the official YAML test-suite (https://github.com/yaml/yaml-test-suite),
|
|
/// the canonical YAML 1.2 conformance corpus. Opt-in, like the real-world corpus suite: the data is
|
|
/// large and fetched separately, so the run is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1` and
|
|
/// `bash scripts/fetch-yaml-test-suite.sh`. It reports two levels:
|
|
/// - parse-level: a should-parse case must parse and a should-error case must be rejected;
|
|
/// - value-level: a should-parse case must parse to the exact tree its `in.json` describes, the
|
|
/// honest metric, since a parser can accept a document and build the wrong tree.
|
|
/// Both print their pass rate to stderr and assert only that the corpus was found, so this is a
|
|
/// measurement harness, not yet a threshold gate (the threshold is set once the number is known).
|
|
@Suite("YAML test-suite conformance")
|
|
struct YAMLTestSuiteConformanceTests {
|
|
private struct Case {
|
|
let id: String
|
|
let input: String
|
|
let expectsError: Bool
|
|
let expectedJSON: String?
|
|
}
|
|
|
|
private func loadCases() -> [Case]? {
|
|
let manager = FileManager.default
|
|
let root = ProcessInfo.processInfo.environment["PUREYAML_YAML_TEST_SUITE_DIR"]
|
|
?? manager.currentDirectoryPath + "/.build/yaml-test-suite"
|
|
guard let names = try? manager.contentsOfDirectory(atPath: root) else {
|
|
return nil
|
|
}
|
|
var cases: [Case] = []
|
|
for name in names.sorted() {
|
|
let dir = root + "/" + name
|
|
guard let input = try? String(contentsOfFile: dir + "/in.yaml", encoding: .utf8) else {
|
|
continue
|
|
}
|
|
cases.append(Case(
|
|
id: name,
|
|
input: input,
|
|
expectsError: manager.fileExists(atPath: dir + "/error"),
|
|
expectedJSON: try? String(contentsOfFile: dir + "/in.json", encoding: .utf8),
|
|
))
|
|
}
|
|
return cases
|
|
}
|
|
|
|
@Test("Parse-level conformance against the official yaml-test-suite")
|
|
func parseLevelConformance() throws {
|
|
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
|
|
return
|
|
}
|
|
let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh")
|
|
var okExpected = 0, okPassed = 0, errExpected = 0, errPassed = 0
|
|
var shouldParseFailures: [String] = []
|
|
var shouldRejectFailures: [String] = []
|
|
var failureCauses: [String: Int] = [:]
|
|
for testCase in cases {
|
|
var thrown: Error?
|
|
do {
|
|
_ = try PureYAML.parseStream(testCase.input)
|
|
} catch {
|
|
thrown = error
|
|
}
|
|
let parsed = thrown == nil
|
|
if testCase.expectsError {
|
|
errExpected += 1
|
|
if parsed { shouldRejectFailures.append(testCase.id) } else { errPassed += 1 }
|
|
} else {
|
|
okExpected += 1
|
|
if parsed {
|
|
okPassed += 1
|
|
} else {
|
|
shouldParseFailures.append(testCase.id)
|
|
let cause = (thrown as? PureYAML.Parsing.ParseError)?.diagnosticCode ?? "other"
|
|
failureCauses[cause, default: 0] += 1
|
|
}
|
|
}
|
|
}
|
|
let total = okExpected + errExpected
|
|
let passed = okPassed + errPassed
|
|
let causes = failureCauses.sorted { $0.value > $1.value }.map { "\($0.key)=\($0.value)" }
|
|
report("""
|
|
YAML test-suite parse-level: \(passed)/\(total) (\(percent(passed, total)))
|
|
should-parse handled: \(okPassed)/\(okExpected) (\(percent(okPassed, okExpected)))
|
|
should-error rejected: \(errPassed)/\(errExpected) (\(percent(errPassed, errExpected)))
|
|
should-parse failures by cause: \(causes.joined(separator: " "))
|
|
should-parse failures (\(shouldParseFailures.count)): \(sample(shouldParseFailures))
|
|
should-error misses (\(shouldRejectFailures.count)): \(sample(shouldRejectFailures))
|
|
""")
|
|
#expect(total > 0)
|
|
}
|
|
|
|
/// The honest metric: a should-parse case must parse to the exact tree its `in.json` describes,
|
|
/// not merely parse without error. Restricted to single-document cases, since the suite's
|
|
/// `in.json` is one JSON value; multi-document and non-string-key cases (which JSON cannot
|
|
/// represent) are counted as skipped, not failed, so the rate reflects only what is comparable.
|
|
@Test("Value-level conformance: the parsed tree equals the expected in.json")
|
|
func valueLevelConformance() throws {
|
|
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
|
|
return
|
|
}
|
|
let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh")
|
|
var eligible = 0, matched = 0, parseFailed = 0, skipped = 0
|
|
var mismatches: [String] = []
|
|
for testCase in cases where !testCase.expectsError {
|
|
guard let json = testCase.expectedJSON else { continue }
|
|
guard let documents = try? PureYAML.parseStream(testCase.input) else {
|
|
parseFailed += 1
|
|
continue
|
|
}
|
|
// Compare only single-document cases against their single in.json value.
|
|
guard documents.count == 1,
|
|
let expected = jsonNode(fromJSONText: json),
|
|
let actual = jsonNode(from: documents[0].value)
|
|
else {
|
|
skipped += 1
|
|
continue
|
|
}
|
|
eligible += 1
|
|
if actual == expected { matched += 1 } else { mismatches.append(testCase.id) }
|
|
}
|
|
report("""
|
|
YAML test-suite value-level: \(matched)/\(eligible) (\(percent(matched, eligible))) of comparable cases
|
|
(parse-failed: \(parseFailed), skipped multi-doc/non-string-key: \(skipped))
|
|
value mismatches (\(mismatches.count)): \(sample(mismatches))
|
|
""")
|
|
#expect(eligible > 0)
|
|
}
|
|
|
|
/// A JSON-comparable normalization: ints and floats both become a number, so a value-level
|
|
/// match reflects structure and resolved value the way `in.json` records them. `Decodable` so the
|
|
/// expected `in.json` is read with `JSONDecoder`, which distinguishes a JSON bool from a number
|
|
/// natively. The previous `JSONSerialization` path needed CoreFoundation (`CFGetTypeID`) for that,
|
|
/// which is Darwin-only and failed to compile on Linux and Windows.
|
|
private indirect enum JSONNode: Equatable, Decodable {
|
|
case null
|
|
case bool(Bool)
|
|
case number(Double)
|
|
case string(String)
|
|
case array([JSONNode])
|
|
case object([String: JSONNode])
|
|
|
|
init(from decoder: Decoder) throws {
|
|
let container = try decoder.singleValueContainer()
|
|
// Order matters: a JSON bool must be tried before a number, since `true`/`false` are a
|
|
// distinct JSON type the number decode would otherwise not see.
|
|
if container.decodeNil() {
|
|
self = .null
|
|
} else if let bool = try? container.decode(Bool.self) {
|
|
self = .bool(bool)
|
|
} else if let number = try? container.decode(Double.self) {
|
|
self = .number(number)
|
|
} else if let string = try? container.decode(String.self) {
|
|
self = .string(string)
|
|
} else if let array = try? container.decode([JSONNode].self) {
|
|
self = .array(array)
|
|
} else if let object = try? container.decode([String: JSONNode].self) {
|
|
self = .object(object)
|
|
} else {
|
|
throw DecodingError.dataCorrupted(.init(codingPath: decoder.codingPath, debugDescription: "unrecognized JSON value"))
|
|
}
|
|
}
|
|
}
|
|
|
|
private func jsonNode(from value: PureYAML.Model.Value) -> JSONNode? {
|
|
switch value {
|
|
case .null: .null
|
|
case let .bool(flag): .bool(flag)
|
|
case let .int(integer): .number(Double(integer))
|
|
case let .double(double): .number(double)
|
|
case let .string(text): .string(text)
|
|
case let .sequence(items): jsonArray(items)
|
|
case let .mapping(mapping): jsonObject(mapping)
|
|
}
|
|
}
|
|
|
|
private func jsonArray(_ items: [PureYAML.Model.Value]) -> JSONNode? {
|
|
var nodes: [JSONNode] = []
|
|
for item in items {
|
|
guard let node = jsonNode(from: item) else { return nil }
|
|
nodes.append(node)
|
|
}
|
|
return .array(nodes)
|
|
}
|
|
|
|
private func jsonObject(_ mapping: PureYAML.Model.Mapping) -> JSONNode? {
|
|
var object: [String: JSONNode] = [:]
|
|
for pair in mapping.pairs {
|
|
guard let key = pair.keyNode.stringValue, let node = jsonNode(from: pair.value) else { return nil }
|
|
object[key] = node
|
|
}
|
|
return .object(object)
|
|
}
|
|
|
|
private func jsonNode(fromJSONText text: String) -> JSONNode? {
|
|
guard let data = text.data(using: .utf8) else {
|
|
return nil
|
|
}
|
|
// The suite's `in.json` may be a top-level fragment (a bare string, number, or null), which
|
|
// `JSONDecoder` accepts in Swift 6. `JSONNode`'s decoder uses a single-value container, so it
|
|
// reads fragments and containers alike, cross-platform and without CoreFoundation.
|
|
return try? JSONDecoder().decode(JSONNode.self, from: data)
|
|
}
|
|
|
|
private func percent(_ part: Int, _ whole: Int) -> String {
|
|
whole == 0 ? "n/a" : String(format: "%.1f%%", Double(part) / Double(whole) * 100)
|
|
}
|
|
|
|
private func sample(_ ids: [String]) -> String {
|
|
ids.prefix(25).joined(separator: " ") + (ids.count > 25 ? " …" : "")
|
|
}
|
|
|
|
private func report(_ text: String) {
|
|
FileHandle.standardError.write(Data("\n\(text)\n".utf8))
|
|
}
|
|
}
|