diff --git a/CHANGELOG.md b/CHANGELOG.md index fef28b9..2f4c238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- XSTS invalid-schemas-accepted (#145): global element references now carry the + referenced declaration's type name, fixed/default constraint, nillable flag, + and `block` metadata into Particle-Valid-Restriction. A local element with an + implicit `xs:anyType` therefore no longer satisfies NameAndTypeOK against a + same-named global ref typed as `xs:decimal`, closing elemZ028e while preserving + a valid narrowing to `xs:short`. invalid-schemas-accepted 44 -> 43, no false + positive (valid-schemas-rejected and valid-instances-rejected held at 0), no + other bucket moved. + - XSTS invalid-schemas-accepted (#145): substitution-group closure now stops chaining through an intermediate member whose own `block` contains `substitution`. The blocked element itself may still substitute for its direct diff --git a/Sources/Schema/XSDContext.swift b/Sources/Schema/XSDContext.swift index 878c07b..d0c5a8c 100644 --- a/Sources/Schema/XSDContext.swift +++ b/Sources/Schema/XSDContext.swift @@ -276,6 +276,11 @@ extension PureXML.Schema { /// Top-level (global) attribute declaration nodes by name, so an /// `` resolves to the global declaration's type. var globalAttributes: [String: XSDTree] = [:] + /// Top-level (global) element declaration nodes keyed by namespaced + /// component identity (`{namespace}local`), so an `` + /// particle can carry the referenced declaration's type and metadata into + /// the restriction oracle. + var globalElements: [String: XSDTree] = [:] var groups: [String: XSDTree] /// Model-group definitions before a `redefine` overlay. var baseGroups: [String: XSDTree] = [:] diff --git a/Sources/Schema/XSDParser.swift b/Sources/Schema/XSDParser.swift index 3f76307..7b0ba46 100644 --- a/Sources/Schema/XSDParser.swift +++ b/Sources/Schema/XSDParser.swift @@ -104,6 +104,7 @@ extension PureXML.Schema.XSDParser { context.namespaceBindings = XSDNode.namespaceBindings(of: schema) context.complexTypeNodes = indexByName(allChildren(containers, named: "complexType")) context.globalAttributes = indexByName(allChildren(containers, named: "attribute")) + context.globalElements = globalElementNodes(containers, namespaceMap: namespaceMap, mainTargetNamespace: mainTargetNamespace) let redefinedAttributeGroups = PureXML.Schema.XSDParser.redefinedNames(containers, "attributeGroup") let redefinedGroups = PureXML.Schema.XSDParser.redefinedNames(containers, "group") let redefinedComplexTypes = PureXML.Schema.XSDParser.redefinedNames(containers, "complexType") @@ -313,6 +314,24 @@ extension PureXML.Schema.XSDParser { containers.flatMap { XSDNode.children($0, named: name) } } + private static func globalElementNodes( + _ containers: [XSDTree], + namespaceMap: [Int: String?], + mainTargetNamespace: String?, + ) -> [String: XSDTree] { + var index: [String: XSDTree] = [:] + for containerIndex in containers.indices { + let container = containers[containerIndex] + guard XSDNode.localName(container) == "schema" else { continue } + let namespace = namespaceMap[containerIndex] ?? mainTargetNamespace + for node in XSDNode.children(container, named: "element") { + guard let name = XSDNode.attribute(node, "name"), XSDNode.attribute(node, "ref") == nil else { continue } + index[derivationKey(name, in: namespace)] = node + } + } + return index + } + /// Named component definitions from every container except `redefine` /// overlays, so a redefinition's self-reference resolves to the included /// definition rather than the overlay being built. diff --git a/Sources/Schema/XSDParserParticles.swift b/Sources/Schema/XSDParserParticles.swift index 647cdc6..f755600 100644 --- a/Sources/Schema/XSDParserParticles.swift +++ b/Sources/Schema/XSDParserParticles.swift @@ -125,19 +125,22 @@ extension PureXML.Schema.XSDParser { // member's own namespace, even across imported namespaces. let alternatives = head + (context.substitutions[headKey] ?? []).map { PureXML.Schema.XSDParser.unpackElementName($0) } if alternatives.count == 1 { - return PureXML.Schema.Particle(minOccurs: minimum, maxOccurs: maximum, term: elementReferenceTerm(alternatives[0].0, alternatives[0].1)) + return PureXML.Schema.Particle(minOccurs: minimum, maxOccurs: maximum, term: elementReferenceTerm(alternatives[0].0, alternatives[0].1, context)) } - let members = alternatives.map { PureXML.Schema.Particle(term: elementReferenceTerm($0.0, $0.1)) } + let members = alternatives.map { PureXML.Schema.Particle(term: elementReferenceTerm($0.0, $0.1, context)) } return PureXML.Schema.Particle(minOccurs: minimum, maxOccurs: maximum, term: .group(.init(compositor: .choice, particles: members))) } /// The term for a reference to a global element, in the resolved `namespace`. - static func elementReferenceTerm(_ name: String, _ namespace: String?) -> PureXML.Schema.Term { - .element( + static func elementReferenceTerm(_ name: String, _ namespace: String?, _ context: PureXML.Schema.XSDContext) -> PureXML.Schema.Term { + let definition = context.globalElements[derivationKey(name, in: namespace)] + return .element( name: PureXML.Model.QualifiedName(localName: name, namespaceURI: namespace), type: .typeReference(elementKey(name)), - typeName: nil, - valueConstraint: nil, + typeName: definition.flatMap(elementTypeName), + valueConstraint: definition.flatMap(valueConstraint), + block: definition.map { blockMethods(of: $0, admitting: [.extension, .restriction, .substitution]) } ?? [], + nillable: definition.map { ["true", "1"].contains(XSDNode.attribute($0, "nillable")) } ?? false, ) } } diff --git a/Tests/XSDParticleReferenceRestrictionTests.swift b/Tests/XSDParticleReferenceRestrictionTests.swift new file mode 100644 index 0000000..0760b55 --- /dev/null +++ b/Tests/XSDParticleReferenceRestrictionTests.swift @@ -0,0 +1,30 @@ +import Testing +@testable import PureXML + +@Suite("XSD Particle Restriction References") +struct XSDParticleReferenceRestrictionTests { + @Test("NameAndTypeOK: element refs keep the global declaration's type") + func test_globalRefCarriesTypeMetadata() { + let invalid = """ + + + + + + + + """ + #expect((try? PureXML.Schema.Document(invalid)) == nil) + + let valid = """ + + + + + + + + """ + #expect((try? PureXML.Schema.Document(valid)) != nil) + } +} diff --git a/Tests/XSTSSuiteTests.swift b/Tests/XSTSSuiteTests.swift index 3546789..68a48a7 100644 --- a/Tests/XSTSSuiteTests.swift +++ b/Tests/XSTSSuiteTests.swift @@ -146,8 +146,9 @@ struct XSTSSuiteTests { /// model-group occurrence, identity-constraint content, derivation base kind, /// attribute/simple-type rules, composition, simpleContent inline-type checks, /// complexContent mixedness, and substitution-group anySimpleType checks. + /// Substitution closure barriers and element-ref metadata moved 45 -> 43. private let knownSchemaValidRejected = 0 - private let knownSchemaInvalidAccepted = 44 + private let knownSchemaInvalidAccepted = 43 private let knownInstanceValidRejected = 0 /// Instance bucket (133 -> 31), per-step deltas in CHANGELOG.md: xsi:type must derive from the /// declared type (complex, then atomic and list/union, resolved through the element-ref chain); diff --git a/docs/plans/2026-06-18-false-negative-tail.md b/docs/plans/2026-06-18-false-negative-tail.md index abc6234..5e0d0c9 100644 --- a/docs/plans/2026-06-18-false-negative-tail.md +++ b/docs/plans/2026-06-18-false-negative-tail.md @@ -8,7 +8,7 @@ ratchet one clause at a time. ## Current measurement -Measured from `/tmp/xsts-failures.txt`, last written 2026-06-19 08:20 local time +Measured from `/tmp/xsts-failures.txt`, last written 2026-06-19 08:47 local time by `XSTSSuiteTests`. ```sh @@ -20,7 +20,7 @@ Result: | Bucket | Count | |---|---:| | valid schema rejected | 0 | -| invalid schema accepted | 44 | +| invalid schema accepted | 43 | | valid instance rejected | 0 | | invalid instance accepted | 31 | @@ -32,8 +32,10 @@ simpleContent inline type-restriction rule, and `ctZ010d` is closed by the complexContent mixed-agreement rule. `stZ048` is closed by the substitution-head `xs:anySimpleType` member content check. `elemZ027_c` is closed by making substitution-group closure stop chaining through members whose own `block` -contains `substitution`. The schema false-negative bucket is down by four with -both false-positive buckets still at zero. +contains `substitution`. `elemZ028e` is closed by carrying global element-ref +declaration metadata into NameAndTypeOK, so a local implicit-`anyType` element +cannot widen a typed global ref. The schema false-negative bucket is down by +five with both false-positive buckets still at zero. ## File map @@ -67,7 +69,7 @@ not a proof. Before code, read the XSTS fixture, the matching PureXMLResearch reference, and the XSD 1.0 rule text; update the lane if the evidence says this classification is wrong. -### Invalid schemas accepted: 44 +### Invalid schemas accepted: 43 | Lane | Cases | First owning surface | |---|---|---| @@ -76,7 +78,7 @@ classification is wrong. | Attribute group restriction triage | `attgC028` | `SchemaAttributeRestriction`, redefine machinery | | Attribute type/use triage | `attKa015`, `attKb018a` | `SchemaAttributeApplicability`, `SchemaAttributeRestriction` | | Attribute identity/composition triage | `attQ011`, `attQ016`, `attQ017`, `attQ018` | Attribute restrictions and per-document composition | -| Substitution-group head/member modeling | `elemZ026`, `elemZ028e` | `SchemaSubstitutionType`, `XSDSubstitutionMembers` | +| Substitution-group head/member modeling | `elemZ026` | `SchemaSubstitutionType`, `XSDSubstitutionMembers` | | ParticleRestriction exactness | `particlesEa025`, `particlesFb003`, `particlesHa161`, `particlesHb011`, `particlesIg004`, `particlesIj008`, `particlesIk011`, `particlesIk025`, `particlesIk027`, `particlesK006`, `particlesM034`, `particlesV020` | `ParticleRestriction*` | | UPA / wildcard determinism | `particlesZ022`, `particlesZ030_d`, `particlesZ039` | `ContentModelDeterminism` | | Regex disclosed-vs-fixable triage | `reK88`, `RegexTest_993`, `RegexTest_1477` | `Sources/Regex/*`, `SimpleType` | @@ -170,7 +172,7 @@ XSTS_ROOT=/private/tmp/xsts/xmlschema2006-11-06 swift test -c release --filter X `Tests/SchemaSubstitutionTypeTests.swift`, `Tests/SchemaSubstitutionBlockTests.swift` -**Does:** Model the remaining `elemZ026`/`elemZ028e` schema failures and +**Does:** Model the remaining `elemZ026` schema failure and the `disallowedsubst*` instance failures without changing the general MapAndSum/RecurseAsIfGroup logic. The key safety check is that a substitution head/member graph must be namespace-exact and must not conflate same-local names. diff --git a/docs/release-hurdles.md b/docs/release-hurdles.md index db3103b..13ceae2 100644 --- a/docs/release-hurdles.md +++ b/docs/release-hurdles.md @@ -6,7 +6,7 @@ evidence, what actually stands between the current engine and a 1.0 fit to put i front of a developer as the authority on XML/XSD correctness. Snapshot at time of writing: v0.2.0 released; full 2006-11-06 XSTS archive (14383 -groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 44`, +groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 43`, `valid-instances-rejected = 0`, `invalid-instances-accepted = 31`. ## Status against the four stoppers @@ -16,7 +16,7 @@ groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 44`, libxml2 differential over 9618 schemas. **Met.** This is the doc's bar to be "an authority"; it is cleared on the measured corpora. 2. **Silently accepting faulty input (false negatives).** `invalid-schemas-accepted - = 44`, `invalid-instances-accepted = 31`. **Not at 0.** This is the bulk of named + = 43`, `invalid-instances-accepted = 31`. **Not at 0.** This is the bulk of named milestone work (M2, M3) and the subject of Part 1 below. 3. **Uncharacterized correctness.** Differential harness vs libxml2 and a fuzz suite exist (M1, ~done). Gap: real-world corpora beyond XSTS, and the silent @@ -28,7 +28,7 @@ groups) with `valid-schemas-rejected = 0`, `invalid-schemas-accepted = 44`, ## The biggest hurdle, stated plainly -Driving the false-negative buckets (`invalid-schemas-accepted = 44`, +Driving the false-negative buckets (`invalid-schemas-accepted = 43`, `invalid-instances-accepted = 31`) to zero is the largest remaining body of named work, and it is hard for a specific reason: **the ~95% already closed were the tractable cases; every case left fights the cardinal rule.** The remainder sits in @@ -56,7 +56,7 @@ chose to under-approximate to stay false-positive-free. | particlesV020 | MapAndSum (Sequence:Choice) | effective-total-range "approximated by per-particle checks, a documented simplification" (ParticleRestriction.swift:8) | | particlesK006, M034 | RecurseAsIfGroup (Elt:All, Elt:Sequence) | "a documented, bounded under-rejection… over-rejection is the non-starter" (ParticleRestriction.swift:298) | | particlesEa025, Fb003, Ha161, Hb011 | pointless-particle / extension-from-`any` / forbidden sequence:elt | partial | -| elemZ026/028 | element-decl restriction where base is a substitution-group head | partially modelled; elemZ027_c is closed | +| elemZ026 | element-decl restriction where base is a substitution-group head | partially modelled; elemZ027_c and elemZ028e are closed | | particlesZ018 | list-of-int ⊄ decimal (simple-type derivation, a *different* oracle) | closed by inline simpleContent type restriction check | | ctZ010d | mixed base extended with new element content by a default-false derived type (a *different* oracle) | closed by complexContent mixed-agreement check | | particlesZ022/030 | non-deterministic wildcards (UPA, not restriction) | determinism check is deliberately QName-only | diff --git a/docs/schema-validity-burndown.md b/docs/schema-validity-burndown.md index 3ddcbe2..e223c74 100644 --- a/docs/schema-validity-burndown.md +++ b/docs/schema-validity-burndown.md @@ -10,7 +10,7 @@ irreducible tail. Do not deviate to new features while this is open. | Bucket | Count | Reading | |---|---|---| | valid-schemas-rejected | **72 -> 0** | production target is 0: rejecting a correct schema is the most user-hostile defect (#148). Particle-restriction over-rejection fixed: pointless `maxOccurs=0` particles removed, content-free restriction of an EMPTY base, empty-derived emptiable-base, and (#163) the pairwise check over the normalized content model with occurrence subsumption applied only to same-kind pairings (cross-kind element/group, group/element, group/wildcard match occurrence inside, by effective total range / member). #164 resolved element-ref namespaces by prefix binding (the `particlesJj/Jk/Jm/Jn/Js`, `particlesQ/R` cluster, -28). Content-free (`maxOccurs=0`) derived groups and substitution-group declaration order closed `particlesW006`/`elemZ027_a`; the W3C MapAndSum rule (Sequence:Choice, #165) closed `particlesV003`. The false-positive schema bucket is now zero. | -| **invalid-schemas-accepted** | **2461 -> 44** | we rarely catch a bad one (facet, default/fixed value, particle-restriction name+type, id, structural content-model, UPA determinism, content order, identity-constraint XPath-subset, wildcard-namespace, final/block, simpleType content, group/wildcard and group/element cardinality, element-ref namespace, sequence/choice MapAndSum, circular type derivation, circular group/attributeGroup/substitution references, attribute-use uniqueness and single-ID, type-excludes-inline-type, ID-typed value-constraint exclusion, all-group reference placement, substitution-member type derivation, default/fixed mutual exclusion, elementTypeRestrictionOK NameAndTypeOK union checks, prohibited attribute type-check bypass, keyref arity verification, top-level group/attributeGroup local-attribute checks, simpleType restriction structure and prefix binding checks, namespace-aware reference resolution checks, simpleType variety constraints on list itemType and union memberTypes, targetNamespace scope matching on local schema references, substitution group exclusions based on element final and schema finalDefault attributes, cross-container component-name and identity-constraint uniqueness checks, notation declarations and notation enumeration restrictions, xml:lang and xml:space on documentation/appinfo elements, duplicate schema container validation across separate files with identical content, schema element order, import constraints validation, facet applicability per base type, fractionDigits constraints on integer types, ordered duration range bounds, built-in list non-emptiness constraints, simpleContent inline type restriction, complexContent mixed-agreement checks, substitution-head anySimpleType member-content checks, and substitution-group closure barriers for `block="substitution"` landed). | +| **invalid-schemas-accepted** | **2461 -> 43** | we rarely catch a bad one (facet, default/fixed value, particle-restriction name+type, id, structural content-model, UPA determinism, content order, identity-constraint XPath-subset, wildcard-namespace, final/block, simpleType content, group/wildcard and group/element cardinality, element-ref namespace, sequence/choice MapAndSum, circular type derivation, circular group/attributeGroup/substitution references, attribute-use uniqueness and single-ID, type-excludes-inline-type, ID-typed value-constraint exclusion, all-group reference placement, substitution-member type derivation, default/fixed mutual exclusion, elementTypeRestrictionOK NameAndTypeOK union checks, prohibited attribute type-check bypass, keyref arity verification, top-level group/attributeGroup local-attribute checks, simpleType restriction structure and prefix binding checks, namespace-aware reference resolution checks, simpleType variety constraints on list itemType and union memberTypes, targetNamespace scope matching on local schema references, substitution group exclusions based on element final and schema finalDefault attributes, cross-container component-name and identity-constraint uniqueness checks, notation declarations and notation enumeration restrictions, xml:lang and xml:space on documentation/appinfo elements, duplicate schema container validation across separate files with identical content, schema element order, import constraints validation, facet applicability per base type, fractionDigits constraints on integer types, ordered duration range bounds, built-in list non-emptiness constraints, simpleContent inline type restriction, complexContent mixed-agreement checks, substitution-head anySimpleType member-content checks, substitution-group closure barriers for `block="substitution"`, and global element-ref declaration metadata in NameAndTypeOK landed). | | valid-instances-rejected | **233 -> 0** | false-positive instance bucket is now zero. | | invalid-instances-accepted | **160 -> 31** | instance side remains open; see the XSTS false-negative tail plan for the current 31-case split. | diff --git a/docs/xsts-deviations.md b/docs/xsts-deviations.md index 774e6d6..7c7d2cf 100644 --- a/docs/xsts-deviations.md +++ b/docs/xsts-deviations.md @@ -41,7 +41,7 @@ entries removed about 70 apparent deviations that were never bugs. Counts are against *settled* W3C expectations (queried/disputed entries excluded). Current ratchet (2026-06-19): `valid schema rejected = 0`, -`invalid schema accepted = 44`, `valid instance rejected = 0`, +`invalid schema accepted = 43`, `valid instance rejected = 0`, `invalid instance accepted = 31`. The historical table above is retained as the published baseline snapshot; the active burn-down is in `docs/plans/2026-06-18-false-negative-tail.md`. @@ -53,7 +53,7 @@ and to document the remainder as spec-justified, deliberate exclusions. ## Category A: genuine correctness bugs -### Fixed (thirty-three root causes, ~2257 deviations cleared) +### Fixed (thirty-four root causes, ~2258 deviations cleared) list-datatype length facets (#146) · union pattern/enumeration facets · duration partial-order facets · QName length non-constraining · the full XSD `\p{Is...}` @@ -71,9 +71,10 @@ compile-time constraining-facet definition validity (length-family facets are validity (`min`/`maxInclusive`, `min`/`maxExclusive`, `enumeration` values valid in the base space; inclusive/exclusive exclusions; bound ordering) · substitution-group heads typed `xs:anySimpleType` only admitting simple and -simpleContent members · `gMonth` `--MM--` lexical form · schema `id` attribute validity (`xs:ID`: NCName and +simpleContent members · global element refs carrying declaration metadata into +Particle-Valid-Restriction · `gMonth` `--MM--` lexical form · schema `id` attribute validity (`xs:ID`: NCName and unique within the document). The schema-validity work brought invalid schemas -accepted 2461 to 44 (schema-document structural validity: content model + complexType content shape, attribute values, name/reference + pattern-regex lexical validity, and reference resolution, against the schema-for-schemas, plus simpleType variety constraints on list itemType and union memberTypes, plus targetNamespace scope matching on local schema references, plus substitution group exclusions based on element final and schema finalDefault attributes, duplicate schema container validation across separate files with identical content, schema element order, import constraints validation, and the later tail fixes); see `schema-validity-burndown.md`. +accepted 2461 to 43 (schema-document structural validity: content model + complexType content shape, attribute values, name/reference + pattern-regex lexical validity, and reference resolution, against the schema-for-schemas, plus simpleType variety constraints on list itemType and union memberTypes, plus targetNamespace scope matching on local schema references, plus substitution group exclusions based on element final and schema finalDefault attributes, duplicate schema container validation across separate files with identical content, schema element order, import constraints validation, and the later tail fixes); see `schema-validity-burndown.md`. Plus the measurement fix (exclude W3C-disputed `status="queried"` entries). ### Remaining, root-caused (the hard tail)