Mihaela Mihaljevic
6fa49b7ffc
swiftformat --lint failed on these pre-existing files and the pre-push hook checks the whole tree, blocking every push until they conform. The wrapped if bodies pushed YAMLTestSuiteGapsTests over the 400-line lint limit, so its private JSONStreamScanner helper moves to its own support file.
224 lines
9.9 KiB
Swift
224 lines
9.9 KiB
Swift
import Foundation
|
|
@testable import PureYAML
|
|
import Testing
|
|
|
|
/// Measures PureYAML against the official YAML test-suite (https://github.com/yaml/yaml-test-suite),
|
|
/// the canonical YAML 1.2 conformance corpus. Opt-in, like the real-world corpus suite: the data is
|
|
/// large and fetched separately, so the run is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1` and
|
|
/// `bash scripts/fetch-yaml-test-suite.sh`. It reports two levels:
|
|
/// - parse-level: a should-parse case must parse and a should-error case must be rejected;
|
|
/// - value-level: a should-parse case must parse to the exact tree its `in.json` describes, the
|
|
/// honest metric, since a parser can accept a document and build the wrong tree.
|
|
/// Both print their pass rate to stderr and assert only that the corpus was found, so this is a
|
|
/// measurement harness, not yet a threshold gate (the threshold is set once the number is known).
|
|
@Suite("YAML test-suite conformance")
|
|
struct YAMLTestSuiteConformanceTests {
|
|
private struct Case {
|
|
let id: String
|
|
let input: String
|
|
let expectsError: Bool
|
|
let expectedJSON: String?
|
|
}
|
|
|
|
private func loadCases() -> [Case]? {
|
|
let manager = FileManager.default
|
|
let root = ProcessInfo.processInfo.environment["PUREYAML_YAML_TEST_SUITE_DIR"]
|
|
?? manager.currentDirectoryPath + "/.build/yaml-test-suite"
|
|
guard let names = try? manager.contentsOfDirectory(atPath: root) else {
|
|
return nil
|
|
}
|
|
var cases: [Case] = []
|
|
for name in names.sorted() {
|
|
let dir = root + "/" + name
|
|
guard let input = try? String(contentsOfFile: dir + "/in.yaml", encoding: .utf8) else {
|
|
continue
|
|
}
|
|
cases.append(Case(
|
|
id: name,
|
|
input: input,
|
|
expectsError: manager.fileExists(atPath: dir + "/error"),
|
|
expectedJSON: try? String(contentsOfFile: dir + "/in.json", encoding: .utf8),
|
|
))
|
|
}
|
|
return cases
|
|
}
|
|
|
|
@Test("Parse-level conformance against the official yaml-test-suite")
|
|
func parseLevelConformance() throws {
|
|
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
|
|
return
|
|
}
|
|
let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh")
|
|
var okExpected = 0, okPassed = 0, errExpected = 0, errPassed = 0
|
|
var shouldParseFailures: [String] = []
|
|
var shouldRejectFailures: [String] = []
|
|
var failureCauses: [String: Int] = [:]
|
|
for testCase in cases {
|
|
var thrown: Error?
|
|
do {
|
|
_ = try PureYAML.parseStream(testCase.input)
|
|
} catch {
|
|
thrown = error
|
|
}
|
|
let parsed = thrown == nil
|
|
if testCase.expectsError {
|
|
errExpected += 1
|
|
if parsed {
|
|
shouldRejectFailures.append(testCase.id)
|
|
} else {
|
|
errPassed += 1
|
|
}
|
|
} else {
|
|
okExpected += 1
|
|
if parsed {
|
|
okPassed += 1
|
|
} else {
|
|
shouldParseFailures.append(testCase.id)
|
|
let cause = (thrown as? PureYAML.Parsing.ParseError)?.diagnosticCode ?? "other"
|
|
failureCauses[cause, default: 0] += 1
|
|
}
|
|
}
|
|
}
|
|
let total = okExpected + errExpected
|
|
let passed = okPassed + errPassed
|
|
let causes = failureCauses.sorted { $0.value > $1.value }.map { "\($0.key)=\($0.value)" }
|
|
report("""
|
|
YAML test-suite parse-level: \(passed)/\(total) (\(percent(passed, total)))
|
|
should-parse handled: \(okPassed)/\(okExpected) (\(percent(okPassed, okExpected)))
|
|
should-error rejected: \(errPassed)/\(errExpected) (\(percent(errPassed, errExpected)))
|
|
should-parse failures by cause: \(causes.joined(separator: " "))
|
|
should-parse failures (\(shouldParseFailures.count)): \(sample(shouldParseFailures))
|
|
should-error misses (\(shouldRejectFailures.count)): \(sample(shouldRejectFailures))
|
|
""")
|
|
#expect(total > 0)
|
|
}
|
|
|
|
/// The honest metric: a should-parse case must parse to the exact tree its `in.json` describes,
|
|
/// not merely parse without error. Restricted to single-document cases, since the suite's
|
|
/// `in.json` is one JSON value; multi-document and non-string-key cases (which JSON cannot
|
|
/// represent) are counted as skipped, not failed, so the rate reflects only what is comparable.
|
|
@Test("Value-level conformance: the parsed tree equals the expected in.json")
|
|
func valueLevelConformance() throws {
|
|
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
|
|
return
|
|
}
|
|
let cases = try #require(loadCases(), "fetch the suite first: bash scripts/fetch-yaml-test-suite.sh")
|
|
var eligible = 0, matched = 0, parseFailed = 0, skipped = 0
|
|
var mismatches: [String] = []
|
|
for testCase in cases where !testCase.expectsError {
|
|
guard let json = testCase.expectedJSON else { continue }
|
|
guard let documents = try? PureYAML.parseStream(testCase.input) else {
|
|
parseFailed += 1
|
|
continue
|
|
}
|
|
// Compare only single-document cases against their single in.json value.
|
|
guard documents.count == 1,
|
|
let expected = jsonNode(fromJSONText: json),
|
|
let actual = jsonNode(from: documents[0].value)
|
|
else {
|
|
skipped += 1
|
|
continue
|
|
}
|
|
eligible += 1
|
|
if actual == expected {
|
|
matched += 1
|
|
} else {
|
|
mismatches.append(testCase.id)
|
|
}
|
|
}
|
|
report("""
|
|
YAML test-suite value-level: \(matched)/\(eligible) (\(percent(matched, eligible))) of comparable cases
|
|
(parse-failed: \(parseFailed), skipped multi-doc/non-string-key: \(skipped))
|
|
value mismatches (\(mismatches.count)): \(sample(mismatches))
|
|
""")
|
|
#expect(eligible > 0)
|
|
}
|
|
|
|
/// A JSON-comparable normalization: ints and floats both become a number, so a value-level
|
|
/// match reflects structure and resolved value the way `in.json` records them. `Decodable` so the
|
|
/// expected `in.json` is read with `JSONDecoder`, which distinguishes a JSON bool from a number
|
|
/// natively. The previous `JSONSerialization` path needed CoreFoundation (`CFGetTypeID`) for that,
|
|
/// which is Darwin-only and failed to compile on Linux and Windows.
|
|
private indirect enum JSONNode: Equatable, Decodable {
|
|
case null
|
|
case bool(Bool)
|
|
case number(Double)
|
|
case string(String)
|
|
case array([JSONNode])
|
|
case object([String: JSONNode])
|
|
|
|
init(from decoder: Decoder) throws {
|
|
let container = try decoder.singleValueContainer()
|
|
// Order matters: a JSON bool must be tried before a number, since `true`/`false` are a
|
|
// distinct JSON type the number decode would otherwise not see.
|
|
if container.decodeNil() {
|
|
self = .null
|
|
} else if let bool = try? container.decode(Bool.self) {
|
|
self = .bool(bool)
|
|
} else if let number = try? container.decode(Double.self) {
|
|
self = .number(number)
|
|
} else if let string = try? container.decode(String.self) {
|
|
self = .string(string)
|
|
} else if let array = try? container.decode([JSONNode].self) {
|
|
self = .array(array)
|
|
} else if let object = try? container.decode([String: JSONNode].self) {
|
|
self = .object(object)
|
|
} else {
|
|
throw DecodingError.dataCorrupted(.init(codingPath: decoder.codingPath, debugDescription: "unrecognized JSON value"))
|
|
}
|
|
}
|
|
}
|
|
|
|
private func jsonNode(from value: PureYAML.Model.Value) -> JSONNode? {
|
|
switch value {
|
|
case .null: .null
|
|
case let .bool(flag): .bool(flag)
|
|
case let .int(integer): .number(Double(integer))
|
|
case let .double(double): .number(double)
|
|
case let .string(text): .string(text)
|
|
case let .sequence(items): jsonArray(items)
|
|
case let .mapping(mapping): jsonObject(mapping)
|
|
}
|
|
}
|
|
|
|
private func jsonArray(_ items: [PureYAML.Model.Value]) -> JSONNode? {
|
|
var nodes: [JSONNode] = []
|
|
for item in items {
|
|
guard let node = jsonNode(from: item) else { return nil }
|
|
nodes.append(node)
|
|
}
|
|
return .array(nodes)
|
|
}
|
|
|
|
private func jsonObject(_ mapping: PureYAML.Model.Mapping) -> JSONNode? {
|
|
var object: [String: JSONNode] = [:]
|
|
for pair in mapping.pairs {
|
|
guard let key = pair.keyNode.stringValue, let node = jsonNode(from: pair.value) else { return nil }
|
|
object[key] = node
|
|
}
|
|
return .object(object)
|
|
}
|
|
|
|
private func jsonNode(fromJSONText text: String) -> JSONNode? {
|
|
guard let data = text.data(using: .utf8) else {
|
|
return nil
|
|
}
|
|
// The suite's `in.json` may be a top-level fragment (a bare string, number, or null), which
|
|
// `JSONDecoder` accepts in Swift 6. `JSONNode`'s decoder uses a single-value container, so it
|
|
// reads fragments and containers alike, cross-platform and without CoreFoundation.
|
|
return try? JSONDecoder().decode(JSONNode.self, from: data)
|
|
}
|
|
|
|
private func percent(_ part: Int, _ whole: Int) -> String {
|
|
whole == 0 ? "n/a" : String(format: "%.1f%%", Double(part) / Double(whole) * 100)
|
|
}
|
|
|
|
private func sample(_ ids: [String]) -> String {
|
|
ids.prefix(25).joined(separator: " ") + (ids.count > 25 ? " …" : "")
|
|
}
|
|
|
|
private func report(_ text: String) {
|
|
FileHandle.standardError.write(Data("\n\(text)\n".utf8))
|
|
}
|
|
}
|