From 00995071b05c71f568d84625ce0bfe35a2bad75f Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Thu, 18 Jun 2026 22:37:49 +0200 Subject: [PATCH] Reject mixedness-changing complexContent extensions --- CHANGELOG.md | 2 ++ Sources/Schema/SchemaExtensionAllGroup.swift | 28 ++++++++--------- Tests/SchemaExtensionAllGroupTests.swift | 33 ++++++++++++++++++++ Tests/XSTSSuiteTests.swift | 11 +++---- docs/plans/2026-06-18-false-negative-tail.md | 16 +++++----- docs/release-hurdles.md | 7 +++-- docs/schema-validity-burndown.md | 8 ++--- docs/xsts-deviations.md | 15 ++++++--- 8 files changed, 80 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55d090c..8883675 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- XSTS invalid-schemas-accepted (#145): a `complexContent` extension that explicitly changes mixedness, or that adds element content while defaulting to `mixed="false"`, must match the base type's mixed setting. This rejects ctZ010d (mixed base extended with new element content by an omitted/default-false derived type) while preserving the valid ctZ012b near-miss (empty extension inherits the mixed base) and the invalid ctF008 explicit mixed mismatch. invalid-schemas-accepted 47 → 46, no false positive (valid-schemas-rejected and valid-instances-rejected held at 0), no other bucket moved. + - XSTS invalid-schemas-accepted (#145): a `simpleContent` restriction that supplies an inline `simpleType` must use a type that is itself a valid restriction of the base complex type's simple content. A list of `xs:int` is not a restriction of an atomic `xs:decimal` content type, so particlesZ018 is now rejected. Valid inline atomic restrictions become the effective simpleContent type for instance validation, including facets declared beside the inline type. invalid-schemas-accepted 48 → 47, no false positive (valid-schemas-rejected and valid-instances-rejected held at 0), no other bucket moved. - XSTS invalid-schemas-accepted (#145): a constraining facet may not be applied where a `simpleContent` restriction's base chain resolves to `xs:anySimpleType`, the ur simple type, which carries no constraining facets (XSD Part 2 §3.2.1); stZ010 (a complex type extending `xs:anySimpleType`, restricted by another that adds `minLength`) is now rejected. The base chain is followed through local complex types' own `simpleContent` bases; a base that bottoms at any other built-in, a simple type, or an external/unresolved type is left alone, and a restriction supplying its own nested `simpleType` base is skipped (the facet applies to that), so a facet on a string-, int-, or other real-typed simpleContent stays accepted. invalid-schemas-accepted 49 → 48, no false positive (valid-schemas-rejected and valid-instances-rejected held at 0), no other bucket moved. diff --git a/Sources/Schema/SchemaExtensionAllGroup.swift b/Sources/Schema/SchemaExtensionAllGroup.swift index c8dc4b6..ad23bb1 100644 --- a/Sources/Schema/SchemaExtensionAllGroup.swift +++ b/Sources/Schema/SchemaExtensionAllGroup.swift @@ -282,12 +282,13 @@ extension PureXML.Schema.XSDParser { } } - /// XSD 1.0 `cos-ct-extends.1.4.3.2.2.1`: a `complexContent` extension's mixedness - /// must match its base type's. The effective content joins the base content and - /// the extension's, so a mixed extension may extend only a mixed base and an - /// element-only extension only an element-only base. Checked for a self-contained - /// schema, and only when the base content is definitely element-only or mixed (an - /// empty or simple-content base is governed by other rules and is left alone). + /// XSD 1.0 `cos-ct-extends.1.4.3.2.2.1`: an explicit `mixed` on a + /// `complexContent` extension must match the base. If `mixed` is omitted, an + /// extension with no element particle inherits the base content unchanged + /// (`ctZ012b`), while an extension that adds element content uses the schema + /// default `false` (`ctZ010d`). Checked for a self-contained schema, and only + /// when the base content is definitely element-only or mixed (an empty or + /// simple-content base is governed by other rules and is left alone). static func extensionMixedAgreementErrors(_ schema: XSDTree, _ context: PureXML.Schema.XSDContext, _ types: [String: PureXML.Schema.ElementType]) -> [String] { guard !skipsCrossDocumentRules(schema, compositionLoaded: context.compositionLoaded) else { return [] } let bindings = context.namespaceBindings @@ -304,9 +305,7 @@ extension PureXML.Schema.XSDParser { case .elementOnly: baseIsMixed = false case .empty, .simpleContent: continue } - // An extension that does not state `mixed` inherits the base's, so there - // is no conflict; only an EXPLICIT `mixed` that disagrees is invalid. - guard let derivedMixed = explicitMixed(content) else { continue } + guard let derivedMixed = declaredMixed(content, addsElementContent: extensionAddsElementContent(ext)) else { continue } if baseIsMixed != derivedMixed { errors.append("a complexContent extension of '\(ExtAllNode.stripPrefix(base))' must have the same mixed setting as its base") } @@ -314,14 +313,13 @@ extension PureXML.Schema.XSDParser { return errors } - /// The EXPLICIT `mixed` of a `complexContent` derivation: its own `mixed` - /// attribute when present, otherwise the enclosing `complexType`'s, or nil when - /// neither states it (an extension then inherits its base's mixedness). `true` - /// and `1` are mixed. - private static func explicitMixed(_ complexContent: XSDTree) -> Bool? { + /// The declared `mixed` value of a `complexContent` derivation. An omitted + /// value matters only when the extension adds element content; otherwise the + /// base content is inherited unchanged and there is no mixedness conflict. + private static func declaredMixed(_ complexContent: XSDTree, addsElementContent: Bool) -> Bool? { if let own = ExtAllNode.attribute(complexContent, "mixed") { return own == "true" || own == "1" } if let parent = complexContent.parent, let outer = ExtAllNode.attribute(parent, "mixed") { return outer == "true" || outer == "1" } - return nil + return addsElementContent ? false : nil } /// Whether a `complexContent` extension adds element content: an `element` or diff --git a/Tests/SchemaExtensionAllGroupTests.swift b/Tests/SchemaExtensionAllGroupTests.swift index fb0c130..7aec463 100644 --- a/Tests/SchemaExtensionAllGroupTests.swift +++ b/Tests/SchemaExtensionAllGroupTests.swift @@ -35,6 +35,39 @@ struct SchemaExtensionAllGroupTests { #expect(compiles(seqBase + ext(base: "myType", content: ""))) } + @Test("A complexContent extension must match its base mixedness") + func test_complexContentExtensionMixedAgreement() { + let mixedBase = "" + + "" + + "" + let extensionParticle = "" + + "" + + "" + + // XSTS ctZ010d: omitting mixed on the derived complex type defaults it to + // false, which disagrees with the mixed base. + #expect(!compiles(mixedBase + ext(base: "base", content: extensionParticle))) + + let explicitMixedDerived = "" + + "\(extensionParticle)" + + "" + #expect(compiles(mixedBase + explicitMixedDerived)) + + // XSTS ctZ012b: without an extension particle, the base content is + // inherited unchanged and the omitted mixed setting is valid. + #expect(compiles(mixedBase + "")) + + let elementOnlyBase = "" + + "" + #expect(compiles(elementOnlyBase + ext(base: "elementOnly", content: extensionParticle))) + + // XSTS ctF008: explicit mixed=true conflicts with an element-only base, + // even when the extension adds no element particle. + let emptyMixedExtension = "" + + "" + #expect(!compiles(elementOnlyBase + emptyMixedExtension)) + } + @Test("An all group as the whole content of a complex type compiles") func test_allAsWholeContentAccepted() { #expect(compiles("\(allContent)")) diff --git a/Tests/XSTSSuiteTests.swift b/Tests/XSTSSuiteTests.swift index 6bbfe3d..fac45b0 100644 --- a/Tests/XSTSSuiteTests.swift +++ b/Tests/XSTSSuiteTests.swift @@ -142,13 +142,12 @@ struct XSTSSuiteTests { /// non-starter and under-rejection is recoverable; this debt is recovered by the /// full effective-total-range Particle-Valid-Restriction (tracked). /// Further rules, each with valid-schemas-rejected held at 0 (see CHANGELOG and - /// docs/xsts-deviations.md), drove invalid-schemas-accepted 130 -> 69: model-group - /// occurrence, identity-constraint content model and xpath prefixes, derivation - /// base-type kind and value-constraint content type, attribute/simple-type rules, - /// and the cross-document composition family (src-resolve, src-import.3, - /// src-redefine.5/6.1/7.2.1, redefine cross-namespace, required schemaLocation). + /// docs/xsts-deviations.md), drove invalid-schemas-accepted 130 -> 46 across + /// model-group occurrence, identity-constraint content, derivation base kind, + /// attribute/simple-type rules, composition, simpleContent inline-type checks, + /// and the complexContent mixedness tail. private let knownSchemaValidRejected = 0 - private let knownSchemaInvalidAccepted = 47 + private let knownSchemaInvalidAccepted = 46 private let knownInstanceValidRejected = 0 /// Instance bucket (133 -> 31), per-step deltas in CHANGELOG.md: xsi:type must derive from the /// declared type (complex, then atomic and list/union, resolved through the element-ref chain); diff --git a/docs/plans/2026-06-18-false-negative-tail.md b/docs/plans/2026-06-18-false-negative-tail.md index e3c1eb6..0b2d101 100644 --- a/docs/plans/2026-06-18-false-negative-tail.md +++ b/docs/plans/2026-06-18-false-negative-tail.md @@ -8,7 +8,7 @@ ratchet one clause at a time. ## Current measurement -Measured from `/tmp/xsts-failures.txt`, last written 2026-06-18 21:31 local time +Measured from `/tmp/xsts-failures.txt`, last written 2026-06-18 22:27 local time by `XSTSSuiteTests`. ```sh @@ -20,7 +20,7 @@ Result: | Bucket | Count | |---|---:| | valid schema rejected | 0 | -| invalid schema accepted | 47 | +| invalid schema accepted | 46 | | valid instance rejected | 0 | | invalid instance accepted | 31 | @@ -28,8 +28,9 @@ The false-positive buckets are already zero. That is the non-negotiable gate: every task below must keep them at zero. Progress since the starting 48/31 snapshot: `particlesZ018` is closed by the -simpleContent inline type-restriction rule, dropping the schema false-negative -bucket by one with both false-positive buckets still at zero. +simpleContent inline type-restriction rule, and `ctZ010d` is closed by the +complexContent mixed-agreement rule. The schema false-negative bucket is down by +two with both false-positive buckets still at zero. ## File map @@ -63,7 +64,7 @@ not a proof. Before code, read the XSTS fixture, the matching PureXMLResearch reference, and the XSD 1.0 rule text; update the lane if the evidence says this classification is wrong. -### Invalid schemas accepted: 47 +### Invalid schemas accepted: 46 | Lane | Cases | First owning surface | |---|---|---| @@ -72,7 +73,6 @@ classification is wrong. | Attribute group restriction triage | `attgC028` | `SchemaAttributeRestriction`, redefine machinery | | Attribute type/use triage | `attKa015`, `attKb018a` | `SchemaAttributeApplicability`, `SchemaAttributeRestriction` | | Attribute identity/composition triage | `attQ011`, `attQ016`, `attQ017`, `attQ018` | Attribute restrictions and per-document composition | -| Complex/simple derivation edge | `ctZ010d` | `SchemaStructureDerivation`, `SimpleType` | | Substitution-group head/member modeling | `elemZ026`, `elemZ027_c`, `elemZ028e` | `SchemaSubstitutionType`, `XSDSubstitutionMembers` | | ParticleRestriction exactness | `particlesEa025`, `particlesFb003`, `particlesHa161`, `particlesHb011`, `particlesIg004`, `particlesIj008`, `particlesIk011`, `particlesIk025`, `particlesIk027`, `particlesK006`, `particlesM034`, `particlesV020` | `ParticleRestriction*` | | UPA / wildcard determinism | `particlesZ022`, `particlesZ030_d`, `particlesZ039` | `ContentModelDeterminism` | @@ -126,8 +126,8 @@ awk -F': ' '/invalid schema accepted/ {schema++} /invalid instance accepted/ {in `Sources/Schema/SchemaSimpleTypeValidity.swift`, focused `Tests/Schema*Tests.swift` **Does:** Start with cases that do not route through `ParticleRestriction`: -`stZ048`, `ctZ010d`, datatype instance cases, and regex cases. `particlesZ018` -is closed by the first pass. +`stZ048`, datatype instance cases, and regex cases. `particlesZ018` and +`ctZ010d` are closed by the first passes. For regex, first decide whether each case is a genuine unsupported-pattern under-rejection or a spec-backed error. Disclosed cases must be documented, not silently left in the bucket. diff --git a/docs/release-hurdles.md b/docs/release-hurdles.md index 998fadc..18ae69f 100644 --- a/docs/release-hurdles.md +++ b/docs/release-hurdles.md @@ -6,7 +6,7 @@ evidence, what actually stands between the current engine and a 1.0 fit to put i front of a developer as the authority on XML/XSD correctness. Snapshot at time of writing: v0.2.0 released; full 2006-11-06 XSTS archive (14383 -groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 47`, +groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 46`, `valid-instances-rejected = 0`, `invalid-instances-accepted = 31`. ## Status against the four stoppers @@ -16,7 +16,7 @@ groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 47`, libxml2 differential over 9618 schemas. **Met.** This is the doc's bar to be "an authority"; it is cleared on the measured corpora. 2. **Silently accepting faulty input (false negatives).** `invalid-schemas-accepted - = 47`, `invalid-instances-accepted = 31`. **Not at 0.** This is the bulk of named + = 46`, `invalid-instances-accepted = 31`. **Not at 0.** This is the bulk of named milestone work (M2, M3) and the subject of Part 1 below. 3. **Uncharacterized correctness.** Differential harness vs libxml2 and a fuzz suite exist (M1, ~done). Gap: real-world corpora beyond XSTS, and the silent @@ -28,7 +28,7 @@ groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 47`, ## The biggest hurdle, stated plainly -Driving the false-negative buckets (`invalid-schemas-accepted = 47`, +Driving the false-negative buckets (`invalid-schemas-accepted = 46`, `invalid-instances-accepted = 31`) to zero is the largest remaining body of named work, and it is hard for a specific reason: **the ~95% already closed were the tractable cases; every case left fights the cardinal rule.** The remainder sits in @@ -58,6 +58,7 @@ chose to under-approximate to stay false-positive-free. | particlesEa025, Fb003, Ha161, Hb011 | pointless-particle / extension-from-`any` / forbidden sequence:elt | partial | | elemZ026/027/028 | element-decl restriction where base is a substitution-group head | not modelled | | particlesZ018 | list-of-int ⊄ decimal (simple-type derivation, a *different* oracle) | closed by inline simpleContent type restriction check | +| ctZ010d | mixed base extended with new element content by a default-false derived type (a *different* oracle) | closed by complexContent mixed-agreement check | | particlesZ022/030 | non-deterministic wildcards (UPA, not restriction) | determinism check is deliberately QName-only | **Why this is the risk, by fan-in.** `ParticleRestriction.violation` is called from diff --git a/docs/schema-validity-burndown.md b/docs/schema-validity-burndown.md index 904a35f..f9a2723 100644 --- a/docs/schema-validity-burndown.md +++ b/docs/schema-validity-burndown.md @@ -9,10 +9,10 @@ irreducible tail. Do not deviate to new features while this is open. | Bucket | Count | Reading | |---|---|---| -| valid-schemas-rejected | **72 -> 4** | production target is 0: rejecting a correct schema is the most user-hostile defect (#148). Particle-restriction over-rejection fixed: pointless `maxOccurs=0` particles removed, content-free restriction of an EMPTY base, empty-derived emptiable-base, and (#163) the pairwise check over the normalized content model with occurrence subsumption applied only to same-kind pairings (cross-kind element/group, group/element, group/wildcard match occurrence inside, by effective total range / member). #164 resolved element-ref namespaces by prefix binding (the `particlesJj/Jk/Jm/Jn/Js`, `particlesQ/R` cluster, -28). Content-free (`maxOccurs=0`) derived groups and substitution-group declaration order closed `particlesW006`/`elemZ027_a`; the W3C MapAndSum rule (Sequence:Choice, #165) closed `particlesV003`. Remaining 1 (`particlesZ001`): a documented spec ambiguity (RecurseAsIfGroup; the reference rejects it), not a defect. | -| **invalid-schemas-accepted** | **2461 -> 264** | we rarely catch a bad one (facet, default/fixed value, particle-restriction name+type, id, structural content-model, UPA determinism, content order, identity-constraint XPath-subset, wildcard-namespace, final/block, simpleType content, group/wildcard and group/element cardinality, element-ref namespace, sequence/choice MapAndSum, circular type derivation, circular group/attributeGroup/substitution references, attribute-use uniqueness and single-ID, type-excludes-inline-type, ID-typed value-constraint exclusion, all-group reference placement, substitution-member type derivation, default/fixed mutual exclusion, elementTypeRestrictionOK NameAndTypeOK union checks, prohibited attribute type-check bypass, keyref arity verification, top-level group/attributeGroup local-attribute checks, simpleType restriction structure and prefix binding checks, namespace-aware reference resolution checks, simpleType variety constraints on list itemType and union memberTypes, targetNamespace scope matching on local schema references, substitution group exclusions based on element final and schema finalDefault attributes, cross-container component-name and identity-constraint uniqueness checks, notation declarations and notation enumeration restrictions, xml:lang and xml:space on documentation/appinfo elements, duplicate schema container validation across separate files with identical content, schema element order, import constraints validation, facet applicability per base type, fractionDigits constraints on integer types, ordered duration range bounds, and built-in list non-emptiness constraints landed). | -| valid-instances-rejected | **233 -> 173** | (instance side; #164's element-ref namespace fix is shared with instance validation, complexContent restriction attributes inheritance and namespace fixes) | -| invalid-instances-accepted | **160 -> 155** | (instance side; resolving inline `` restriction base, complexContent restriction attributes prohibition) | +| valid-schemas-rejected | **72 -> 0** | production target is 0: rejecting a correct schema is the most user-hostile defect (#148). Particle-restriction over-rejection fixed: pointless `maxOccurs=0` particles removed, content-free restriction of an EMPTY base, empty-derived emptiable-base, and (#163) the pairwise check over the normalized content model with occurrence subsumption applied only to same-kind pairings (cross-kind element/group, group/element, group/wildcard match occurrence inside, by effective total range / member). #164 resolved element-ref namespaces by prefix binding (the `particlesJj/Jk/Jm/Jn/Js`, `particlesQ/R` cluster, -28). Content-free (`maxOccurs=0`) derived groups and substitution-group declaration order closed `particlesW006`/`elemZ027_a`; the W3C MapAndSum rule (Sequence:Choice, #165) closed `particlesV003`. The false-positive schema bucket is now zero. | +| **invalid-schemas-accepted** | **2461 -> 46** | we rarely catch a bad one (facet, default/fixed value, particle-restriction name+type, id, structural content-model, UPA determinism, content order, identity-constraint XPath-subset, wildcard-namespace, final/block, simpleType content, group/wildcard and group/element cardinality, element-ref namespace, sequence/choice MapAndSum, circular type derivation, circular group/attributeGroup/substitution references, attribute-use uniqueness and single-ID, type-excludes-inline-type, ID-typed value-constraint exclusion, all-group reference placement, substitution-member type derivation, default/fixed mutual exclusion, elementTypeRestrictionOK NameAndTypeOK union checks, prohibited attribute type-check bypass, keyref arity verification, top-level group/attributeGroup local-attribute checks, simpleType restriction structure and prefix binding checks, namespace-aware reference resolution checks, simpleType variety constraints on list itemType and union memberTypes, targetNamespace scope matching on local schema references, substitution group exclusions based on element final and schema finalDefault attributes, cross-container component-name and identity-constraint uniqueness checks, notation declarations and notation enumeration restrictions, xml:lang and xml:space on documentation/appinfo elements, duplicate schema container validation across separate files with identical content, schema element order, import constraints validation, facet applicability per base type, fractionDigits constraints on integer types, ordered duration range bounds, built-in list non-emptiness constraints, simpleContent inline type restriction, and complexContent mixed-agreement checks landed). | +| valid-instances-rejected | **233 -> 0** | false-positive instance bucket is now zero. | +| invalid-instances-accepted | **160 -> 31** | instance side remains open; see the XSTS false-negative tail plan for the current 31-case split. | One-sentence diagnosis: *the validator runs its instance-validity rules but not its schema-validity rules.* It compiles what it understands and ignores the diff --git a/docs/xsts-deviations.md b/docs/xsts-deviations.md index ae4eef3..1dfbd69 100644 --- a/docs/xsts-deviations.md +++ b/docs/xsts-deviations.md @@ -40,6 +40,12 @@ entries removed about 70 apparent deviations that were never bugs. Counts are against *settled* W3C expectations (queried/disputed entries excluded). +Current ratchet (2026-06-18): `valid schema rejected = 0`, +`invalid schema accepted = 46`, `valid instance rejected = 0`, +`invalid instance accepted = 31`. The historical table above is retained as the +published baseline snapshot; the active burn-down is in +`docs/plans/2026-06-18-false-negative-tail.md`. + XSTS is a deliberately adversarial, exhaustive corpus; mature validators (Xerces, libxml2) do not pass it fully. The goal is not a green number. It is to fix the deviations that are genuine correctness bugs affecting real documents, @@ -164,13 +170,14 @@ ratchet the baseline in `Tests/XSTSSuiteTests.swift` down to the new value. Tracking issues: #145 (invalid schema accepted), #146 (valid instance rejected), #147 (invalid instance accepted), #148 (valid schema rejected). -## Autopilot accounting (2026-06-18): labelled-suite invalid-schemas-accepted = 74 +## Historical autopilot accounting (2026-06-18): labelled-suite invalid-schemas-accepted = 74 -The `XSTSSuiteTests` ratchet (settled-expectation labelled subset) now stands at +At this earlier snapshot, the `XSTSSuiteTests` ratchet +(settled-expectation labelled subset) stood at **0 valid-schemas-rejected, 74 invalid-schemas-accepted, 0 valid-instances-rejected, 133 invalid-instances-accepted**. False positives have been held at 0 across the -whole campaign (the hard gate). The remaining 74 are the hard tail; every clean, -zero-false-positive structural rule has been harvested. They partition as: +whole campaign (the hard gate). The remaining 74 were the hard tail at that +snapshot. They partitioned as: ### Fixable debt (genuine bugs, but intricate and false-positive-prone)