Mihaela Mihaljevic
6fa49b7ffc
swiftformat --lint failed on these pre-existing files and the pre-push hook checks the whole tree, blocking every push until they conform. The wrapped if bodies pushed YAMLTestSuiteGapsTests over the 400-line lint limit, so its private JSONStreamScanner helper moves to its own support file.
339 lines
12 KiB
Swift
339 lines
12 KiB
Swift
import Foundation
|
|
@testable import PureYAML
|
|
import Testing
|
|
|
|
/// Full conformance gate against the official YAML test suite
|
|
/// (github.com/yaml/yaml-test-suite, `data` branch, commit
|
|
/// 6ad3d2c62885d82fc349026c136ef560838fdf3d). Every official case (numbered
|
|
/// sub-cases counted) is vendored under `Fixtures/yaml-test-suite/<id>/` and
|
|
/// recorded in `Fixtures/yaml-test-suite-manifest.yaml` with a `status`:
|
|
///
|
|
/// - `passing`: PureYAML already handles the case conformantly (it parses with a
|
|
/// value matching the JSON oracle when one is present, or it rejects an error
|
|
/// case). This is the regression guard: if a later change breaks the case, the
|
|
/// gated sweep fails.
|
|
/// - `gap`: PureYAML does not yet handle the case conformantly. The sweep asserts
|
|
/// the gap still reproduces, so a fix is forced to update the manifest rather
|
|
/// than silently drift.
|
|
///
|
|
/// The behavioral sweep is gated behind `PUREYAML_RUN_YAML_TEST_SUITE=1`, mirroring
|
|
/// the full real-corpus sweep, so the default suite stays fast while the whole suite
|
|
/// is guarded on demand. The manifest-integrity test below is always on.
|
|
@Suite("YAML Test Suite Gaps")
|
|
struct YAMLTestSuiteGapsTests {
|
|
@Test("Manifest is consistent with its vendored fixture files")
|
|
func test_manifestMatchesVendoredFixtureFiles() throws {
|
|
let manifest = try YAMLTestSuiteManifest.load()
|
|
|
|
#expect(manifest.version == 3)
|
|
#expect(manifest.cases.count == manifest.total)
|
|
#expect(manifest.total == 402)
|
|
#expect(Set(manifest.cases.map(\.id)).count == manifest.cases.count)
|
|
|
|
let passing = manifest.cases.count(where: { $0.status == "passing" })
|
|
let gaps = manifest.cases.count(where: { $0.status == "gap" })
|
|
#expect(passing == manifest.passing)
|
|
#expect(gaps == manifest.gap)
|
|
#expect(passing + gaps == manifest.total)
|
|
|
|
let rejects = manifest.cases.count(where: { $0.expected == "reject" })
|
|
let parseOK = manifest.cases.count(where: { $0.expected == "parse-ok" })
|
|
#expect(rejects + parseOK == manifest.total)
|
|
|
|
for testCase in manifest.cases {
|
|
#expect(testCase.status == "passing" || testCase.status == "gap", "\(testCase.id): invalid status")
|
|
#expect(testCase.expected == "reject" || testCase.expected == "parse-ok", "\(testCase.id): invalid expected")
|
|
#expect(testCase.oracle == "json" || testCase.oracle == "none", "\(testCase.id): invalid oracle")
|
|
|
|
let yaml = try testCase.loadInput()
|
|
#expect(yaml.utf8.count == testCase.byteCount, "\(testCase.id): byte count drift")
|
|
#expect(yaml.count(where: { $0 == "\n" }) == testCase.lineCount, "\(testCase.id): line count drift")
|
|
|
|
if testCase.expected == "reject" {
|
|
#expect(testCase.loadErrorMarker() != nil, "\(testCase.id): reject case is missing its error marker")
|
|
}
|
|
if testCase.oracle == "json" {
|
|
#expect(testCase.loadExpectedJSON() != nil, "\(testCase.id): json oracle case is missing in.json")
|
|
} else {
|
|
#expect(testCase.loadExpectedJSON() == nil, "\(testCase.id): oracle is none but an in.json is present")
|
|
}
|
|
}
|
|
}
|
|
|
|
@Test("Every case matches its recorded conformance status")
|
|
func test_everyCaseMatchesRecordedStatus() throws {
|
|
guard ProcessInfo.processInfo.environment["PUREYAML_RUN_YAML_TEST_SUITE"] == "1" else {
|
|
return
|
|
}
|
|
|
|
let manifest = try YAMLTestSuiteManifest.load()
|
|
for testCase in manifest.cases {
|
|
let observed = try testCase.observedStatus()
|
|
#expect(
|
|
observed == testCase.status,
|
|
"""
|
|
\(testCase.id) "\(testCase.name)": manifest records status '\(testCase.status)' \
|
|
but observed '\(observed)'. A 'passing' case that now fails is a regression; \
|
|
a 'gap' case that now succeeds should be promoted to 'passing' in the manifest.
|
|
""",
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
// MARK: - Manifest
|
|
|
|
struct YAMLTestSuiteManifest {
|
|
var version: Int
|
|
var total: Int
|
|
var passing: Int
|
|
var gap: Int
|
|
var cases: [YAMLTestSuiteCase]
|
|
|
|
static func load() throws -> Self {
|
|
let url = try #require(Bundle.module.url(
|
|
forResource: "yaml-test-suite-manifest",
|
|
withExtension: "yaml",
|
|
subdirectory: "Fixtures",
|
|
))
|
|
let value = try PureYAML.parse(String(contentsOf: url, encoding: .utf8))
|
|
let root = try #require(value.mapping)
|
|
let version = try #require(root["version"]?.intValue)
|
|
let total = try #require(root["total"]?.intValue)
|
|
let passing = try #require(root["passing"]?.intValue)
|
|
let gap = try #require(root["gap"]?.intValue)
|
|
let caseValues = try #require(root.sequence("cases"))
|
|
return try YAMLTestSuiteManifest(
|
|
version: version,
|
|
total: total,
|
|
passing: passing,
|
|
gap: gap,
|
|
cases: caseValues.map(YAMLTestSuiteCase.init(value:)),
|
|
)
|
|
}
|
|
}
|
|
|
|
struct YAMLTestSuiteCase {
|
|
var id: String
|
|
var name: String
|
|
var localPath: String
|
|
var expected: String
|
|
var oracle: String
|
|
var status: String
|
|
var byteCount: Int
|
|
var lineCount: Int
|
|
|
|
init(value: PureYAML.Model.Value) throws {
|
|
let mapping = try #require(value.mapping)
|
|
id = try mapping.requiredString("id")
|
|
name = try mapping.requiredString("name")
|
|
localPath = try mapping.requiredString("localPath")
|
|
expected = try mapping.requiredString("expected")
|
|
oracle = try mapping.requiredString("oracle")
|
|
status = try mapping.requiredString("status")
|
|
byteCount = try mapping.requiredInt("byteCount")
|
|
lineCount = try mapping.requiredInt("lineCount")
|
|
}
|
|
|
|
private var caseSubdirectory: String {
|
|
let directory = localPath.split(separator: "/").dropLast().joined(separator: "/")
|
|
return "Fixtures/\(directory)"
|
|
}
|
|
|
|
func loadInput() throws -> String {
|
|
let url = try #require(Bundle.module.url(forResource: "in", withExtension: "yaml", subdirectory: caseSubdirectory))
|
|
return try String(contentsOf: url, encoding: .utf8)
|
|
}
|
|
|
|
func loadErrorMarker() -> URL? {
|
|
Bundle.module.url(forResource: "error", withExtension: nil, subdirectory: caseSubdirectory)
|
|
}
|
|
|
|
func loadExpectedJSON() -> String? {
|
|
guard let url = Bundle.module.url(forResource: "in", withExtension: "json", subdirectory: caseSubdirectory) else {
|
|
return nil
|
|
}
|
|
return try? String(contentsOf: url, encoding: .utf8)
|
|
}
|
|
|
|
/// Recomputes PureYAML's present behavior for this case in the manifest's
|
|
/// `status` vocabulary, so any drift (a regression in a passing case, or a fix
|
|
/// that promotes a gap) is detectable.
|
|
///
|
|
/// Conformance is decided by parsing alone: an error case is `passing` when the
|
|
/// parser rejects it; a value case is `passing` when the parser accepts it and,
|
|
/// when an in.json oracle is present, the parsed value matches it. Strict
|
|
/// validation (e.g. duplicate-key detection) is a separate, orthogonal concern
|
|
/// and is deliberately not part of the conformance decision.
|
|
func observedStatus() throws -> String {
|
|
let yaml = try loadInput()
|
|
var documents: [PureYAML.Stream.Document] = []
|
|
var parseThrew = false
|
|
do {
|
|
documents = try PureYAML.parseStream(yaml)
|
|
} catch {
|
|
parseThrew = true
|
|
}
|
|
|
|
if expected == "reject" {
|
|
return parseThrew ? "passing" : "gap"
|
|
}
|
|
if parseThrew {
|
|
return "gap"
|
|
}
|
|
guard oracle == "json", let jsonText = loadExpectedJSON() else {
|
|
return "passing"
|
|
}
|
|
// An empty in.json oracle means the conformant output is an empty stream
|
|
// (no JSON value), as for a comment-only or document-end-only input. A parser
|
|
// that yields any document then differs from the oracle, so it is a gap.
|
|
let trimmed = jsonText.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
if trimmed.isEmpty {
|
|
return documents.isEmpty ? "passing" : "gap"
|
|
}
|
|
guard let expectedDocuments = decodeJSONStream(jsonText) else {
|
|
return "gap"
|
|
}
|
|
if expectedDocuments.count != documents.count {
|
|
return "gap"
|
|
}
|
|
let matches = zip(documents, expectedDocuments).allSatisfy { equalValue($0.value, $1) }
|
|
return matches ? "passing" : "gap"
|
|
}
|
|
}
|
|
|
|
// MARK: - JSON oracle (type-preserving, avoids NSNumber bool ambiguity)
|
|
|
|
indirect enum YAMLTestSuiteJSON: Decodable, Equatable {
|
|
case null
|
|
case bool(Bool)
|
|
case int(Int)
|
|
case double(Double)
|
|
case string(String)
|
|
case array([YAMLTestSuiteJSON])
|
|
case object([String: YAMLTestSuiteJSON])
|
|
|
|
init(from decoder: Decoder) throws {
|
|
let container = try decoder.singleValueContainer()
|
|
if container.decodeNil() {
|
|
self = .null
|
|
} else if let value = try? container.decode(Bool.self) {
|
|
self = .bool(value)
|
|
} else if let value = try? container.decode(Int.self) {
|
|
self = .int(value)
|
|
} else if let value = try? container.decode(Double.self) {
|
|
self = .double(value)
|
|
} else if let value = try? container.decode(String.self) {
|
|
self = .string(value)
|
|
} else if let value = try? container.decode([YAMLTestSuiteJSON].self) {
|
|
self = .array(value)
|
|
} else if let value = try? container.decode([String: YAMLTestSuiteJSON].self) {
|
|
self = .object(value)
|
|
} else {
|
|
throw DecodingError.dataCorrupted(.init(codingPath: container.codingPath, debugDescription: "unrecognized JSON"))
|
|
}
|
|
}
|
|
}
|
|
|
|
func decodeJSONStream(_ text: String) -> [YAMLTestSuiteJSON]? {
|
|
let decoder = JSONDecoder()
|
|
if let whole = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(text.utf8)) {
|
|
return [whole]
|
|
}
|
|
let parts = splitJSONStream(text)
|
|
guard !parts.isEmpty else {
|
|
return nil
|
|
}
|
|
var values: [YAMLTestSuiteJSON] = []
|
|
for part in parts {
|
|
guard let value = try? decoder.decode(YAMLTestSuiteJSON.self, from: Data(part.utf8)) else {
|
|
return nil
|
|
}
|
|
values.append(value)
|
|
}
|
|
return values
|
|
}
|
|
|
|
/// Splits a concatenated JSON stream (one value per YAML document) into top-level values.
|
|
func splitJSONStream(_ text: String) -> [String] {
|
|
var scanner = JSONStreamScanner(text: text)
|
|
return scanner.scan()
|
|
}
|
|
|
|
func equalValue(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool {
|
|
switch (value, json) {
|
|
case let (.sequence(lhs), .array(rhs)):
|
|
lhs.count == rhs.count && zip(lhs, rhs).allSatisfy { equalValue($0, $1) }
|
|
case let (.mapping(lhs), .object(rhs)):
|
|
equalMapping(lhs, rhs)
|
|
default:
|
|
equalScalar(value, json)
|
|
}
|
|
}
|
|
|
|
private func equalScalar(_ value: PureYAML.Model.Value, _ json: YAMLTestSuiteJSON) -> Bool {
|
|
switch (value, json) {
|
|
case (.null, .null):
|
|
true
|
|
case let (.bool(lhs), .bool(rhs)):
|
|
lhs == rhs
|
|
case let (.int(lhs), .int(rhs)):
|
|
lhs == rhs
|
|
case let (.int(lhs), .double(rhs)):
|
|
Double(lhs) == rhs
|
|
case let (.double(lhs), .int(rhs)):
|
|
lhs == Double(rhs)
|
|
case let (.double(lhs), .double(rhs)):
|
|
lhs == rhs || (lhs.isNaN && rhs.isNaN)
|
|
case let (.string(lhs), .string(rhs)):
|
|
lhs == rhs
|
|
default:
|
|
false
|
|
}
|
|
}
|
|
|
|
private func equalMapping(_ mapping: PureYAML.Model.Mapping, _ object: [String: YAMLTestSuiteJSON]) -> Bool {
|
|
guard mapping.pairs.count == object.count else {
|
|
return false
|
|
}
|
|
var dictionary: [String: PureYAML.Model.Value] = [:]
|
|
for pair in mapping.pairs {
|
|
dictionary[pair.key] = pair.value
|
|
}
|
|
guard dictionary.count == object.count else {
|
|
return false
|
|
}
|
|
for (key, expected) in object {
|
|
guard let actual = dictionary[key], equalValue(actual, expected) else {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
private extension PureYAML.Model.Mapping {
|
|
func requiredString(_ key: String) throws -> String {
|
|
try #require(self[key]?.stringValue)
|
|
}
|
|
|
|
func requiredInt(_ key: String) throws -> Int {
|
|
try #require(self[key]?.intValue)
|
|
}
|
|
}
|
|
|
|
private extension PureYAML.Model.Value {
|
|
var stringValue: String? {
|
|
guard case let .string(value) = self else {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
|
|
var intValue: Int? {
|
|
guard case let .int(value) = self else {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
}
|