diff --git a/CHANGELOG.md b/CHANGELOG.md index 8883675..ab029fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- XSTS invalid-schemas-accepted (#145): a substitution-group head typed `xs:anySimpleType` now admits simple type members and complex types with `simpleContent`, but rejects element-only, empty, and mixed complex-type members. The shared derivation oracle treats `anySimpleType` as a broad ur-type for other restriction checks, so the substitution-group rule now adds its own value-space guard before accepting the affiliation. stZ048 is rejected while the valid simple/simpleContent near-misses remain accepted. invalid-schemas-accepted 46 → 45, no false positive (valid-schemas-rejected and valid-instances-rejected held at 0), no other bucket moved. + - XSTS invalid-schemas-accepted (#145): a `complexContent` extension that explicitly changes mixedness, or that adds element content while defaulting to `mixed="false"`, must match the base type's mixed setting. This rejects ctZ010d (mixed base extended with new element content by an omitted/default-false derived type) while preserving the valid ctZ012b near-miss (empty extension inherits the mixed base) and the invalid ctF008 explicit mixed mismatch. invalid-schemas-accepted 47 → 46, no false positive (valid-schemas-rejected and valid-instances-rejected held at 0), no other bucket moved. - XSTS invalid-schemas-accepted (#145): a `simpleContent` restriction that supplies an inline `simpleType` must use a type that is itself a valid restriction of the base complex type's simple content. A list of `xs:int` is not a restriction of an atomic `xs:decimal` content type, so particlesZ018 is now rejected. Valid inline atomic restrictions become the effective simpleContent type for instance validation, including facets declared beside the inline type. invalid-schemas-accepted 48 → 47, no false positive (valid-schemas-rejected and valid-instances-rejected held at 0), no other bucket moved. diff --git a/Sources/Schema/SchemaSubstitutionType.swift b/Sources/Schema/SchemaSubstitutionType.swift index 60a5036..750661c 100644 --- a/Sources/Schema/SchemaSubstitutionType.swift +++ b/Sources/Schema/SchemaSubstitutionType.swift @@ -79,7 +79,7 @@ extension PureXML.Schema.XSDParser { guard let headType = globalElementType[headKey] else { return [] } if isInlineListOrUnion(element) { return [] } let inlineDeriv = inlineTypeDerivation(element) - let isDerived: Bool + var isDerived: Bool let memberType: String if let inlineDeriv { memberType = "anonymous" @@ -88,9 +88,13 @@ extension PureXML.Schema.XSDParser { memberType = typeAttr isDerived = PureXML.Schema.ParticleRestriction.typeDerivesOrEqual(memberType, headType, tables.typeDerivation, types) } else { + // A member without its own type uses the substitution head's type definition. memberType = headType isDerived = true } + if isBuiltinAnySimpleType(headType, types) { + isDerived = inlineMemberHasSimpleContent(element) ?? memberTypeHasSimpleContent(memberType, types) ?? isDerived + } // A member whose type is a list or union derives only from anySimpleType, so // it may affiliate only to a head typed anySimpleType (or to the same type, or // as a recorded restriction of the head's type, both of which `isDerived` @@ -230,4 +234,43 @@ extension PureXML.Schema.XSDParser { case .list, .union: return true } } + + private static func isBuiltinAnySimpleType(_ name: String, _ types: [String: PureXML.Schema.ElementType]) -> Bool { + name == "anySimpleType" && types[name] == nil + } + + /// Whether a member type can derive from the simple ur-type. Unknown types are + /// left undecided so the reference-resolution rule, not this check, owns them. + private static func memberTypeHasSimpleContent(_ name: String, _ types: [String: PureXML.Schema.ElementType]) -> Bool? { + if name == "anySimpleType" { return true } + if name == "anyType" { return false } + if PureXML.Schema.BuiltinType(rawValue: name) != nil { return true } + var current: PureXML.Schema.ElementType? = types[name] + var steps = 0 + while let resolved = current, steps <= types.count { + switch resolved { + case .simple: + return true + case let .complex(complex): + if case .simpleContent = complex.content { return true } + return false + case let .typeReference(key): + current = types[key] + steps += 1 + } + } + return nil + } + + private static func inlineMemberHasSimpleContent(_ element: XSDTree) -> Bool? { + if SubTypeNode.elementChildren(element).contains(where: { + $0.name?.namespaceURI == xsdNamespace && SubTypeNode.localName($0) == "simpleType" + }) { + return true + } + guard let complex = SubTypeNode.elementChildren(element).first(where: { + $0.name?.namespaceURI == xsdNamespace && SubTypeNode.localName($0) == "complexType" + }) else { return nil } + return SubTypeNode.firstChild(complex, named: "simpleContent") != nil + } } diff --git a/Tests/SchemaSubstitutionTypeTests.swift b/Tests/SchemaSubstitutionTypeTests.swift index ff59996..327560d 100644 --- a/Tests/SchemaSubstitutionTypeTests.swift +++ b/Tests/SchemaSubstitutionTypeTests.swift @@ -45,6 +45,47 @@ struct SchemaSubstitutionTypeTests { )) } + @Test("An anySimpleType substitution head admits only simple-content members") + func test_anySimpleTypeHeadRequiresSimpleContentMember() { + #expect(compiles( + "" + + "" + + "" + + "" + + "", + )) + #expect(!compiles( + "" + + "" + + "", + )) + #expect(!compiles( + "" + + "" + + "", + )) + #expect(compiles( + "" + + "", + )) + } + + @Test("An inline anySimpleType substitution member must also have simple content") + func test_inlineAnySimpleTypeMemberRequiresSimpleContent() { + #expect(compiles( + "" + + "" + + "" + + "", + )) + #expect(!compiles( + "" + + "" + + "" + + "", + )) + } + /// `cos-st-derived-ok` clause 2.4: a type validly derives from a union when it /// derives from one of the union's member types. List/union derivation is not /// modelled, so the check stands down: an integer member of a union-typed head diff --git a/Tests/XSTSSuiteTests.swift b/Tests/XSTSSuiteTests.swift index fac45b0..e57cb53 100644 --- a/Tests/XSTSSuiteTests.swift +++ b/Tests/XSTSSuiteTests.swift @@ -142,12 +142,12 @@ struct XSTSSuiteTests { /// non-starter and under-rejection is recoverable; this debt is recovered by the /// full effective-total-range Particle-Valid-Restriction (tracked). /// Further rules, each with valid-schemas-rejected held at 0 (see CHANGELOG and - /// docs/xsts-deviations.md), drove invalid-schemas-accepted 130 -> 46 across + /// docs/xsts-deviations.md), drove invalid-schemas-accepted 130 -> 45 across /// model-group occurrence, identity-constraint content, derivation base kind, /// attribute/simple-type rules, composition, simpleContent inline-type checks, - /// and the complexContent mixedness tail. + /// complexContent mixedness, and substitution-group anySimpleType checks. private let knownSchemaValidRejected = 0 - private let knownSchemaInvalidAccepted = 46 + private let knownSchemaInvalidAccepted = 45 private let knownInstanceValidRejected = 0 /// Instance bucket (133 -> 31), per-step deltas in CHANGELOG.md: xsi:type must derive from the /// declared type (complex, then atomic and list/union, resolved through the element-ref chain); diff --git a/docs/plans/2026-06-18-false-negative-tail.md b/docs/plans/2026-06-18-false-negative-tail.md index 0b2d101..db1cc4b 100644 --- a/docs/plans/2026-06-18-false-negative-tail.md +++ b/docs/plans/2026-06-18-false-negative-tail.md @@ -8,7 +8,7 @@ ratchet one clause at a time. ## Current measurement -Measured from `/tmp/xsts-failures.txt`, last written 2026-06-18 22:27 local time +Measured from `/tmp/xsts-failures.txt`, last written 2026-06-18 22:59 local time by `XSTSSuiteTests`. ```sh @@ -20,7 +20,7 @@ Result: | Bucket | Count | |---|---:| | valid schema rejected | 0 | -| invalid schema accepted | 46 | +| invalid schema accepted | 45 | | valid instance rejected | 0 | | invalid instance accepted | 31 | @@ -29,8 +29,9 @@ every task below must keep them at zero. Progress since the starting 48/31 snapshot: `particlesZ018` is closed by the simpleContent inline type-restriction rule, and `ctZ010d` is closed by the -complexContent mixed-agreement rule. The schema false-negative bucket is down by -two with both false-positive buckets still at zero. +complexContent mixed-agreement rule. `stZ048` is closed by the substitution-head +`xs:anySimpleType` member content check. The schema false-negative bucket is down +by three with both false-positive buckets still at zero. ## File map @@ -64,7 +65,7 @@ not a proof. Before code, read the XSTS fixture, the matching PureXMLResearch reference, and the XSD 1.0 rule text; update the lane if the evidence says this classification is wrong. -### Invalid schemas accepted: 46 +### Invalid schemas accepted: 45 | Lane | Cases | First owning surface | |---|---|---| @@ -78,7 +79,6 @@ classification is wrong. | UPA / wildcard determinism | `particlesZ022`, `particlesZ030_d`, `particlesZ039` | `ContentModelDeterminism` | | Regex disclosed-vs-fixable triage | `reK88`, `RegexTest_993`, `RegexTest_1477` | `Sources/Regex/*`, `SimpleType` | | Redefine / composition | `schG10`, `schM4`, `schM8`, `schN10`, `schN12`, `schZ011_a`, `schZ011_b`, `schZ011_c`, `schZ011_d` | `SchemaReferences`, `SchemaRedefineSelfReference`, `ParticleRestriction` | -| Simple-type validity | `stZ048` | `SchemaSimpleTypeValidity`, `SimpleType` | | Wildcard UPA | `wildI009`, `wildI013`, `wildI014`, `wildZ013` | `ContentModelDeterminism` | ### Invalid instances accepted: 31 @@ -126,8 +126,8 @@ awk -F': ' '/invalid schema accepted/ {schema++} /invalid instance accepted/ {in `Sources/Schema/SchemaSimpleTypeValidity.swift`, focused `Tests/Schema*Tests.swift` **Does:** Start with cases that do not route through `ParticleRestriction`: -`stZ048`, datatype instance cases, and regex cases. `particlesZ018` and -`ctZ010d` are closed by the first passes. +datatype instance cases and regex cases. `particlesZ018`, `ctZ010d`, and +`stZ048` are closed by the first passes. For regex, first decide whether each case is a genuine unsupported-pattern under-rejection or a spec-backed error. Disclosed cases must be documented, not silently left in the bucket. diff --git a/docs/release-hurdles.md b/docs/release-hurdles.md index 18ae69f..1de786c 100644 --- a/docs/release-hurdles.md +++ b/docs/release-hurdles.md @@ -6,7 +6,7 @@ evidence, what actually stands between the current engine and a 1.0 fit to put i front of a developer as the authority on XML/XSD correctness. Snapshot at time of writing: v0.2.0 released; full 2006-11-06 XSTS archive (14383 -groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 46`, +groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 45`, `valid-instances-rejected = 0`, `invalid-instances-accepted = 31`. ## Status against the four stoppers @@ -16,7 +16,7 @@ groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 46`, libxml2 differential over 9618 schemas. **Met.** This is the doc's bar to be "an authority"; it is cleared on the measured corpora. 2. **Silently accepting faulty input (false negatives).** `invalid-schemas-accepted - = 46`, `invalid-instances-accepted = 31`. **Not at 0.** This is the bulk of named + = 45`, `invalid-instances-accepted = 31`. **Not at 0.** This is the bulk of named milestone work (M2, M3) and the subject of Part 1 below. 3. **Uncharacterized correctness.** Differential harness vs libxml2 and a fuzz suite exist (M1, ~done). Gap: real-world corpora beyond XSTS, and the silent @@ -28,7 +28,7 @@ groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 46`, ## The biggest hurdle, stated plainly -Driving the false-negative buckets (`invalid-schemas-accepted = 46`, +Driving the false-negative buckets (`invalid-schemas-accepted = 45`, `invalid-instances-accepted = 31`) to zero is the largest remaining body of named work, and it is hard for a specific reason: **the ~95% already closed were the tractable cases; every case left fights the cardinal rule.** The remainder sits in diff --git a/docs/schema-validity-burndown.md b/docs/schema-validity-burndown.md index f9a2723..a37439d 100644 --- a/docs/schema-validity-burndown.md +++ b/docs/schema-validity-burndown.md @@ -10,7 +10,7 @@ irreducible tail. Do not deviate to new features while this is open. | Bucket | Count | Reading | |---|---|---| | valid-schemas-rejected | **72 -> 0** | production target is 0: rejecting a correct schema is the most user-hostile defect (#148). Particle-restriction over-rejection fixed: pointless `maxOccurs=0` particles removed, content-free restriction of an EMPTY base, empty-derived emptiable-base, and (#163) the pairwise check over the normalized content model with occurrence subsumption applied only to same-kind pairings (cross-kind element/group, group/element, group/wildcard match occurrence inside, by effective total range / member). #164 resolved element-ref namespaces by prefix binding (the `particlesJj/Jk/Jm/Jn/Js`, `particlesQ/R` cluster, -28). Content-free (`maxOccurs=0`) derived groups and substitution-group declaration order closed `particlesW006`/`elemZ027_a`; the W3C MapAndSum rule (Sequence:Choice, #165) closed `particlesV003`. The false-positive schema bucket is now zero. | -| **invalid-schemas-accepted** | **2461 -> 46** | we rarely catch a bad one (facet, default/fixed value, particle-restriction name+type, id, structural content-model, UPA determinism, content order, identity-constraint XPath-subset, wildcard-namespace, final/block, simpleType content, group/wildcard and group/element cardinality, element-ref namespace, sequence/choice MapAndSum, circular type derivation, circular group/attributeGroup/substitution references, attribute-use uniqueness and single-ID, type-excludes-inline-type, ID-typed value-constraint exclusion, all-group reference placement, substitution-member type derivation, default/fixed mutual exclusion, elementTypeRestrictionOK NameAndTypeOK union checks, prohibited attribute type-check bypass, keyref arity verification, top-level group/attributeGroup local-attribute checks, simpleType restriction structure and prefix binding checks, namespace-aware reference resolution checks, simpleType variety constraints on list itemType and union memberTypes, targetNamespace scope matching on local schema references, substitution group exclusions based on element final and schema finalDefault attributes, cross-container component-name and identity-constraint uniqueness checks, notation declarations and notation enumeration restrictions, xml:lang and xml:space on documentation/appinfo elements, duplicate schema container validation across separate files with identical content, schema element order, import constraints validation, facet applicability per base type, fractionDigits constraints on integer types, ordered duration range bounds, built-in list non-emptiness constraints, simpleContent inline type restriction, and complexContent mixed-agreement checks landed). | +| **invalid-schemas-accepted** | **2461 -> 45** | we rarely catch a bad one (facet, default/fixed value, particle-restriction name+type, id, structural content-model, UPA determinism, content order, identity-constraint XPath-subset, wildcard-namespace, final/block, simpleType content, group/wildcard and group/element cardinality, element-ref namespace, sequence/choice MapAndSum, circular type derivation, circular group/attributeGroup/substitution references, attribute-use uniqueness and single-ID, type-excludes-inline-type, ID-typed value-constraint exclusion, all-group reference placement, substitution-member type derivation, default/fixed mutual exclusion, elementTypeRestrictionOK NameAndTypeOK union checks, prohibited attribute type-check bypass, keyref arity verification, top-level group/attributeGroup local-attribute checks, simpleType restriction structure and prefix binding checks, namespace-aware reference resolution checks, simpleType variety constraints on list itemType and union memberTypes, targetNamespace scope matching on local schema references, substitution group exclusions based on element final and schema finalDefault attributes, cross-container component-name and identity-constraint uniqueness checks, notation declarations and notation enumeration restrictions, xml:lang and xml:space on documentation/appinfo elements, duplicate schema container validation across separate files with identical content, schema element order, import constraints validation, facet applicability per base type, fractionDigits constraints on integer types, ordered duration range bounds, built-in list non-emptiness constraints, simpleContent inline type restriction, complexContent mixed-agreement checks, and substitution-head anySimpleType member-content checks landed). | | valid-instances-rejected | **233 -> 0** | false-positive instance bucket is now zero. | | invalid-instances-accepted | **160 -> 31** | instance side remains open; see the XSTS false-negative tail plan for the current 31-case split. | diff --git a/docs/xsts-deviations.md b/docs/xsts-deviations.md index 1dfbd69..8c5b328 100644 --- a/docs/xsts-deviations.md +++ b/docs/xsts-deviations.md @@ -41,7 +41,7 @@ entries removed about 70 apparent deviations that were never bugs. Counts are against *settled* W3C expectations (queried/disputed entries excluded). Current ratchet (2026-06-18): `valid schema rejected = 0`, -`invalid schema accepted = 46`, `valid instance rejected = 0`, +`invalid schema accepted = 45`, `valid instance rejected = 0`, `invalid instance accepted = 31`. The historical table above is retained as the published baseline snapshot; the active burn-down is in `docs/plans/2026-06-18-false-negative-tail.md`. @@ -53,7 +53,7 @@ and to document the remainder as spec-justified, deliberate exclusions. ## Category A: genuine correctness bugs -### Fixed (thirty-one root causes, ~2255 deviations cleared) +### Fixed (thirty-two root causes, ~2256 deviations cleared) list-datatype length facets (#146) · union pattern/enumeration facets · duration partial-order facets · QName length non-constraining · the full XSD `\p{Is...}` @@ -68,8 +68,9 @@ identity fields comparing in value space when the node carries an `xsi:type`) · compile-time constraining-facet definition validity (length-family facets are `nonNegativeInteger`, co-occurrence and range order) · value-bound facet validity (`min`/`maxInclusive`, `min`/`maxExclusive`, `enumeration` values valid -in the base space; inclusive/exclusive exclusions; bound ordering) · `gMonth` -`--MM--` lexical form · schema `id` attribute validity (`xs:ID`: NCName and +in the base space; inclusive/exclusive exclusions; bound ordering) · +substitution-group heads typed `xs:anySimpleType` only admitting simple and +simpleContent members · `gMonth` `--MM--` lexical form · schema `id` attribute validity (`xs:ID`: NCName and unique within the document). The schema-validity work brought invalid schemas accepted 2461 to 264 (schema-document structural validity: content model + complexType content shape, attribute values, name/reference + pattern-regex lexical validity, and reference resolution, against the schema-for-schemas, plus simpleType variety constraints on list itemType and union memberTypes, plus targetNamespace scope matching on local schema references, plus substitution group exclusions based on element final and schema finalDefault attributes, duplicate schema container validation across separate files with identical content, schema element order, and import constraints validation); see `schema-validity-burndown.md`. Plus the measurement fix (exclude W3C-disputed `status="queried"` entries).