From e127cc1f1ebff99eef61fd511bd78b8ae5363fd6 Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Fri, 5 Jun 2026 10:46:28 +0200 Subject: [PATCH] test(validation): harden diagnostics corpus --- Tests/ValidationModeTests.swift | 127 ++++++++++++++ Tests/ValidationModelTests.swift | 80 +++++++++ Tests/ValidationTestSupport.swift | 75 ++++++++ Tests/ValidationTests.swift | 283 +++++++++++++----------------- docs/rules/testing.md | 7 +- 5 files changed, 408 insertions(+), 164 deletions(-) create mode 100644 Tests/ValidationModeTests.swift create mode 100644 Tests/ValidationModelTests.swift create mode 100644 Tests/ValidationTestSupport.swift diff --git a/Tests/ValidationModeTests.swift b/Tests/ValidationModeTests.swift new file mode 100644 index 0000000..b5eb52c --- /dev/null +++ b/Tests/ValidationModeTests.swift @@ -0,0 +1,127 @@ +@testable import PureYAML +import Testing + +@Suite("Validation Modes") +struct ValidationModeTests { + @Test("Custom validation rules report exact errors") + func test_customValidationRuleErrors() throws { + let value = try PureYAML.parse( + """ + mode: legacy + nested: + mode: legacy + """, + ) + let validator = PureYAML.Validation.Validator.blank.validating(legacyModeRule(severity: .error)) + + expectValidationError(value, using: validator) { collection in + #expect(collection.issues == [ + .init( + severity: .error, + reason: "Legacy mode is not allowed", + path: .init([.key("mode")]), + ), + .init( + severity: .error, + reason: "Legacy mode is not allowed", + path: .init([.key("nested"), .key("mode")]), + ), + ]) + } + } + + @Test("Strict validation treats warnings as failures") + func test_strictValidationTreatsWarningsAsFailures() throws { + let value = try PureYAML.parse("mode: legacy") + let validator = PureYAML.Validation.Validator.blank.validating(legacyModeRule(severity: .warning)) + + expectValidationError(value, using: validator) { collection in + #expect(collection.issues == [ + .init( + severity: .warning, + reason: "Legacy mode is not allowed", + path: .init([.key("mode")]), + ), + ]) + } + } + + @Test("Non-strict validation returns warnings") + func test_nonStrictValidationReturnsWarnings() throws { + let value = try PureYAML.parse("mode: legacy") + let validator = PureYAML.Validation.Validator.blank.validating(legacyModeRule(severity: .warning)) + + let warnings = try PureYAML.validate(value, using: validator, strict: false) + + #expect(warnings == [ + PureYAML.Validation.Issue( + severity: .warning, + reason: "Legacy mode is not allowed", + path: .init([.key("mode")]), + ), + ]) + } + + @Test("Non-strict validation throws only errors when errors and warnings are mixed") + func test_nonStrictValidationThrowsOnlyErrorsWhenErrorsAndWarningsAreMixed() throws { + let value = try PureYAML.parse( + """ + title: First + title: Second + mode: legacy + """, + ) + let validator = PureYAML.Validation.Validator() + .validating(legacyModeRule(severity: .warning)) + + expectValidationError(value, using: validator, strict: false) { collection in + #expect(collection.issues == [ + .init( + severity: .error, + reason: "Duplicate mapping key 'title'", + path: .init([.key("title")]), + ), + ]) + #expect(!collection.issues.contains(.init( + severity: .warning, + reason: "Legacy mode is not allowed", + path: .init([.key("mode")]), + ))) + } + } + + @Test("Validation traversal preserves rule order before child paths") + func test_validationTraversalPreservesRuleOrderBeforeChildPaths() throws { + let value = try PureYAML.parse( + """ + root: + mode: legacy + list: + - mode: legacy + """, + ) + let validator = PureYAML.Validation.Validator.blank + .validating(traceRule(label: "first")) + .validating(traceRule(label: "second")) + .validating(legacyModeRule(severity: .warning)) + + let result = validator.collect(value) + + #expect(result.issues.map(\.description) == [ + "warning: first at root at root", + "warning: second at root at root", + "warning: first at $.root at $.root", + "warning: second at $.root at $.root", + "warning: first at $.root.mode at $.root.mode", + "warning: second at $.root.mode at $.root.mode", + "warning: Legacy mode is not allowed at $.root.mode", + "warning: first at $.list at $.list", + "warning: second at $.list at $.list", + "warning: first at $.list[0] at $.list[0]", + "warning: second at $.list[0] at $.list[0]", + "warning: first at $.list[0].mode at $.list[0].mode", + "warning: second at $.list[0].mode at $.list[0].mode", + "warning: Legacy mode is not allowed at $.list[0].mode", + ]) + } +} diff --git a/Tests/ValidationModelTests.swift b/Tests/ValidationModelTests.swift new file mode 100644 index 0000000..0f6d36c --- /dev/null +++ b/Tests/ValidationModelTests.swift @@ -0,0 +1,80 @@ +@testable import PureYAML +import Testing + +@Suite("Validation Model") +struct ValidationModelTests { + @Test("Validation result separates errors and warnings") + func test_validationResultSeparatesErrorsAndWarnings() { + let error = PureYAML.Validation.Issue(severity: .error, reason: "error") + let warning = PureYAML.Validation.Issue(severity: .warning, reason: "warning") + let result = PureYAML.Validation.Result([error, warning]) + + #expect(result.isValid == false) + #expect(result.errors == [error]) + #expect(result.warnings == [warning]) + #expect(!result.errors.contains(warning)) + #expect(!result.warnings.contains(error)) + } + + @Test("Validation paths describe roots keys and indexes", arguments: [ + ValidationPathCase( + name: "root", + path: .root, + expectedDescription: "$", + isRoot: true, + ), + ValidationPathCase( + name: "single key", + path: .root.appending(.key("title")), + expectedDescription: "$.title", + isRoot: false, + ), + ValidationPathCase( + name: "nested index", + path: .root + .appending(.key("routes")) + .appending(.index(1)) + .appending(.key("name")), + expectedDescription: "$.routes[1].name", + isRoot: false, + ), + ValidationPathCase( + name: "punctuated key", + path: .root + .appending(.key("/users")) + .appending(.key("get.method")), + expectedDescription: "$./users.get.method", + isRoot: false, + ), + ]) + func test_validationPathsDescribeRootsKeysAndIndexes(testCase: ValidationPathCase) { + #expect(testCase.path.description == testCase.expectedDescription) + #expect(testCase.path.isRoot == testCase.isRoot) + } + + @Test("Validation issue descriptions include severity reason and path") + func test_validationIssueDescriptionsIncludeSeverityReasonAndPath() { + let issue = PureYAML.Validation.Issue( + severity: .error, + reason: "Duplicate mapping key 'title'", + path: .init([.key("title")]), + ) + + #expect(issue.description == "error: Duplicate mapping key 'title' at $.title") + } + + @Test("Validation issue collection describes all failures") + func test_validationIssueCollectionDescription() { + let collection = PureYAML.Validation.Issue.Collection([ + .init(severity: .error, reason: "first", path: .root), + .init(severity: .warning, reason: "second", path: .init([.key("name")])), + ]) + + #expect(collection.description == """ + error: first at root + warning: second at $.name + """) + #expect(collection.description.contains("error: first")) + #expect(!collection.description.contains("info:")) + } +} diff --git a/Tests/ValidationTestSupport.swift b/Tests/ValidationTestSupport.swift new file mode 100644 index 0000000..217fe79 --- /dev/null +++ b/Tests/ValidationTestSupport.swift @@ -0,0 +1,75 @@ +@testable import PureYAML +import Testing + +struct ValidationValidDocumentCase: CustomStringConvertible { + var name: String + var yaml: String + + var description: String { + name + } +} + +struct ValidationDuplicateKeyCase: CustomStringConvertible { + var name: String + var yaml: String + var expectedIssues: [PureYAML.Validation.Issue] + + var description: String { + name + } +} + +struct ValidationPathCase: CustomStringConvertible { + var name: String + var path: PureYAML.Validation.Path + var expectedDescription: String + var isRoot: Bool + + var description: String { + name + } +} + +func legacyModeRule(severity: PureYAML.Validation.Severity) -> PureYAML.Validation.Rule { + PureYAML.Validation.Rule(description: "Mode must not be legacy") { context in + guard case .string("legacy") = context.subject else { + return [] + } + return [ + PureYAML.Validation.Issue( + severity: severity, + reason: "Legacy mode is not allowed", + path: context.path, + ), + ] + } +} + +func traceRule(label: String) -> PureYAML.Validation.Rule { + PureYAML.Validation.Rule(description: "Trace \(label)") { context in + [ + PureYAML.Validation.Issue( + severity: .warning, + reason: "\(label) at \(context.path.isRoot ? "root" : context.path.description)", + path: context.path, + ), + ] + } +} + +func expectValidationError( + _ value: PureYAML.Model.Value, + using validator: PureYAML.Validation.Validator = .init(), + strict: Bool = true, + check: (PureYAML.Validation.Issue.Collection) -> Void, +) { + do { + try PureYAML.validate(value, using: validator, strict: strict) + recordIssue("expected validation failure") + } catch let collection as PureYAML.Validation.Issue.Collection { + check(collection) + } catch { + recordIssue("unexpected error \(error)") + } +} diff --git a/Tests/ValidationTests.swift b/Tests/ValidationTests.swift index 872db99..a4955f1 100644 --- a/Tests/ValidationTests.swift +++ b/Tests/ValidationTests.swift @@ -1,70 +1,152 @@ @testable import PureYAML import Testing -@Suite("Validation") +@Suite("Validation Rules") struct ValidationTests { - @Test("Validates documents with default rules") - func test_documentsWithDefaultRules() throws { - let value = try PureYAML.parse( - """ + @Test("Valid documents produce no diagnostics", arguments: [ + ValidationValidDocumentCase( + name: "flat mapping", + yaml: """ title: Example active: true + retries: 3 """, - ) + ), + ValidationValidDocumentCase( + name: "nested mapping and sequence", + yaml: """ + routes: + - name: Users + methods: + - GET + - POST + - name: Status + methods: + - GET + """, + ), + ValidationValidDocumentCase( + name: "flow collections", + yaml: """ + values: [one, 2, false, {name: Example}] + """, + ), + ValidationValidDocumentCase( + name: "anchors aliases and block scalars", + yaml: """ + shared: &shared {title: Shared, active: true} + first: *shared + second: + title: Local + body: | + one + two + """, + ), + ]) + func test_validDocumentsProduceNoDiagnostics(testCase: ValidationValidDocumentCase) throws { + let value = try PureYAML.parse(testCase.yaml) + let validator = PureYAML.Validation.Validator() + let result = validator.collect(value) + #expect(result == PureYAML.Validation.Result()) + #expect(result.isValid) + #expect(result.errors.isEmpty) + #expect(result.warnings.isEmpty) #expect(try PureYAML.validate(value).isEmpty) } - @Test("Reports duplicate mapping keys") - func test_duplicateMappingKeyError() { - let value = PureYAML.Model.Value.mapping(.init([ - .init(key: "title", value: .string("First")), - .init(key: "title", value: .string("Second")), - ])) - - expectValidationError(value) { collection in - #expect(collection.issues == [ - PureYAML.Validation.Issue( + @Test("Reports duplicate mapping keys with exact diagnostics", arguments: [ + ValidationDuplicateKeyCase( + name: "root duplicate key", + yaml: """ + title: First + title: Second + """, + expectedIssues: [ + .init( severity: .error, reason: "Duplicate mapping key 'title'", path: .init([.key("title")]), ), - ]) - } - } - - @Test("Reports nested validation paths") - func test_nestedValidationPaths() throws { - let value = try PureYAML.parse( - """ + ], + ), + ValidationDuplicateKeyCase( + name: "nested duplicate key", + yaml: """ routes: - name: Users name: People """, - ) - - let result = PureYAML.Validation.Validator().collect(value) - - #expect(result.errors.map(\.path.description) == ["$.routes[0].name"]) - } - - @Test("Rejects duplicate mapping keys parsed through the event composer") - func test_parsedDuplicateMappingKeyError() throws { - let value = try PureYAML.parse( - """ + expectedIssues: [ + .init( + severity: .error, + reason: "Duplicate mapping key 'name'", + path: .init([.key("routes"), .index(0), .key("name")]), + ), + ], + ), + ValidationDuplicateKeyCase( + name: "multiple duplicate keys preserve traversal order", + yaml: """ + root: + title: First + title: Second + routes: + - name: Users + name: People + - method: GET + method: POST + """, + expectedIssues: [ + .init( + severity: .error, + reason: "Duplicate mapping key 'title'", + path: .init([.key("root"), .key("title")]), + ), + .init( + severity: .error, + reason: "Duplicate mapping key 'name'", + path: .init([.key("routes"), .index(0), .key("name")]), + ), + .init( + severity: .error, + reason: "Duplicate mapping key 'method'", + path: .init([.key("routes"), .index(1), .key("method")]), + ), + ], + ), + ValidationDuplicateKeyCase( + name: "third repeated key reports each repeated occurrence", + yaml: """ title: First title: Second + title: Third """, - ) - - expectValidationError(value) { collection in - #expect(collection.issues == [ - PureYAML.Validation.Issue( + expectedIssues: [ + .init( severity: .error, reason: "Duplicate mapping key 'title'", path: .init([.key("title")]), ), - ]) + .init( + severity: .error, + reason: "Duplicate mapping key 'title'", + path: .init([.key("title")]), + ), + ], + ), + ]) + func test_duplicateMappingKeyDiagnostics(testCase: ValidationDuplicateKeyCase) throws { + let value = try PureYAML.parse(testCase.yaml) + let result = PureYAML.Validation.Validator().collect(value) + + #expect(result.issues == testCase.expectedIssues) + #expect(result.errors == testCase.expectedIssues) + #expect(result.warnings.isEmpty) + expectValidationError(value) { collection in + #expect(collection.issues == testCase.expectedIssues) + #expect(collection.description == testCase.expectedIssues.map(\.description).joined(separator: "\n")) } } @@ -77,123 +159,4 @@ struct ValidationTests { #expect(try PureYAML.validate(value, using: .blank).isEmpty) } - - @Test("Custom validation rules can be added") - func test_customValidationRuleError() throws { - let value = try PureYAML.parse( - """ - mode: legacy - """, - ) - let validator = PureYAML.Validation.Validator.blank.validating(legacyModeRule(severity: .error)) - - expectValidationError(value, using: validator) { collection in - #expect(collection.issues.map(\.path.description) == ["$.mode"]) - #expect(collection.issues.map(\.reason) == ["Legacy mode is not allowed"]) - } - } - - @Test("Strict validation treats warnings as failures") - func test_strictValidationTreatsWarningsAsFailures() throws { - let value = try PureYAML.parse("mode: legacy") - let validator = PureYAML.Validation.Validator.blank.validating(legacyModeRule(severity: .warning)) - - expectValidationError(value, using: validator) { collection in - #expect(collection.issues.map(\.severity) == [.warning]) - } - } - - @Test("Non-strict validation returns warnings") - func test_nonStrictValidationReturnsWarnings() throws { - let value = try PureYAML.parse("mode: legacy") - let validator = PureYAML.Validation.Validator.blank.validating(legacyModeRule(severity: .warning)) - - let warnings = try PureYAML.validate(value, using: validator, strict: false) - - #expect(warnings == [ - PureYAML.Validation.Issue( - severity: .warning, - reason: "Legacy mode is not allowed", - path: .init([.key("mode")]), - ), - ]) - } - - @Test("Validation result separates errors and warnings") - func test_validationResultSeparatesErrorsAndWarnings() { - let result = PureYAML.Validation.Result([ - .init(severity: .error, reason: "error"), - .init(severity: .warning, reason: "warning"), - ]) - - #expect(result.isValid == false) - #expect(result.errors.map(\.reason) == ["error"]) - #expect(result.warnings.map(\.reason) == ["warning"]) - } - - @Test("Validation paths describe roots, keys, and indexes") - func test_validationPathsDescribeRootsKeysAndIndexes() { - let path = PureYAML.Validation.Path.root - .appending(.key("routes")) - .appending(.index(1)) - .appending(.key("name")) - - #expect(PureYAML.Validation.Path.root.description == "$") - #expect(PureYAML.Validation.Path.root.isRoot) - #expect(path.description == "$.routes[1].name") - } - - @Test("Validation issue descriptions include severity reason and path") - func test_validationIssueDescriptionsIncludeSeverityReasonAndPath() { - let issue = PureYAML.Validation.Issue( - severity: .error, - reason: "Duplicate mapping key 'title'", - path: .init([.key("title")]), - ) - - #expect(issue.description == "error: Duplicate mapping key 'title' at $.title") - } - - @Test("Validation issue collection describes all failures") - func test_validationIssueCollectionDescription() { - let collection = PureYAML.Validation.Issue.Collection([ - .init(severity: .error, reason: "first", path: .root), - .init(severity: .warning, reason: "second", path: .init([.key("name")])), - ]) - - #expect(collection.description == """ - error: first at root - warning: second at $.name - """) - } - - private func legacyModeRule(severity: PureYAML.Validation.Severity) -> PureYAML.Validation.Rule { - PureYAML.Validation.Rule(description: "Mode must not be legacy") { context in - guard case .string("legacy") = context.subject else { - return [] - } - return [ - PureYAML.Validation.Issue( - severity: severity, - reason: "Legacy mode is not allowed", - path: context.path, - ), - ] - } - } -} - -private func expectValidationError( - _ value: PureYAML.Model.Value, - using validator: PureYAML.Validation.Validator = .init(), - check: (PureYAML.Validation.Issue.Collection) -> Void, -) { - do { - try PureYAML.validate(value, using: validator) - recordIssue("expected validation failure") - } catch let collection as PureYAML.Validation.Issue.Collection { - check(collection) - } catch { - recordIssue("unexpected error \(error)") - } } diff --git a/docs/rules/testing.md b/docs/rules/testing.md index 5fb1bd2..a243a71 100644 --- a/docs/rules/testing.md +++ b/docs/rules/testing.md @@ -8,10 +8,9 @@ The `withDependencies` overrides shown in the patterns below assume the Point-Fr ## Core rules -### Rule 0: OpenAPIKit discipline, Swift Testing syntax +### Rule 0: Exact Diagnostics Discipline -PureYAML tests must follow the OpenAPIKit testing approach, translated to Swift -Testing instead of XCTest: +PureYAML tests must use Swift Testing and pin exact behavior: - Use focused `test_...` method names inside `@Suite` groups. - Use direct equality checks for concrete values, rendered output, token/event @@ -49,7 +48,7 @@ Control dependencies in tests: Structure tests clearly: - MUST use `@Suite` for logical grouping -- MUST name test methods with the `test_...` convention used by OpenAPIKit +- MUST name test methods with the `test_...` convention - MUST keep `@Test("...")` descriptions clear and user-readable - MUST test one behavior per test - MUST NOT create kitchen sink tests