From 4df9eda52e21add8dfea6e4d998e09f079378f12 Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Tue, 16 Jun 2026 23:16:26 +0200 Subject: [PATCH] feat(schema): compile import-chained documents as a union (#161) Schema documents that form an import chain are now compiled as a single union (Document.init(composing:)) rather than as separately compiled documents merged after the fact. Merging froze each document's content models with only the substitution-group members visible in its own import closure, so a substitution member declared in an importing document never reached the head's content model in the imported document. This clears XSTS ctZ007, a cross-document substitution group, taking valid-instances-rejected (#146) from 4 to 3 with no other bucket moving (now 1/262/3/147). The union pools each document's container closure into one context, deduplicating a document supplied directly and also reached through another's import by target namespace and component-name set. The XSTS runner uses the union only for import chains, detected by matching import/include/redefine schemaLocation basenames against the listed documents; independent roots stay on the lenient per-document merge, which a union would wrongly trip with cross-document checks. The remaining three valid-instance rejections are the Ethiopic-digit \d cases (reS17, reS38, reZ004v), which contradict \d = \p{Nd}. --- CHANGELOG.md | 2 +- Sources/Schema/SchemaDocument.swift | 21 ++++++- Sources/Schema/XSDParser.swift | 12 ++-- Sources/Schema/XSDParserUnion.swift | 76 ++++++++++++++++++++++++ Tests/XSTSSuiteTests.swift | 91 ++++++++++++++++++++--------- 5 files changed, 165 insertions(+), 37 deletions(-) create mode 100644 Sources/Schema/XSDParserUnion.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 416bf80..345df06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ Second pre-release. Continues the W3C XSTS schema-validity campaign: **invalid-s ### Fixed -- XSTS valid-instances-rejected (#146): **173 → 4** on the 2006-11-06 archive (settled-status count; the four buckets now read 1/262/4/147). Regex patterns match Unicode scalars (not Swift grapheme clusters), fixing `\p{Is…}` block tests and combining-mark cases; empty patterns match only the empty string. The regex repertoire is pinned to Unicode 3.2, the version the corpus is consistent at: code points added in 3.2 (Tagalog, Cyrillic Komi) are word characters, while later additions (the Tamil digit zero) stay unassigned and match `\D` (`XSDCategory`, `Unicode32Assigned`). Per-child content assessment is driven by the content-model particle the automaton matched, not a by-name lookup (#180), so two same-named particles keep their own `fixed` values and an element matching a `skip` wildcard is not validated against its global declaration. The XSTS runner compiles and merges every `schemaDocument` in a `schemaTest` (not just the first), which also surfaced one previously-masked invalid-instance acceptance. Plus attribute-group `anyAttribute` inheritance, extension wildcard union, namespace-qualified type tables for multi-document `xsi:schemaLocation`, per-container `elementFormDefault`, strict/lax attribute wildcard validation, built-in type resolution, and `xs:redefine` simpleType overrides. +- XSTS valid-instances-rejected (#146): **173 → 3** on the 2006-11-06 archive (settled-status count; the four buckets now read 1/262/3/147). Regex patterns match Unicode scalars (not Swift grapheme clusters), fixing `\p{Is…}` block tests and combining-mark cases; empty patterns match only the empty string. The regex repertoire is pinned to Unicode 3.2, the version the corpus is consistent at: code points added in 3.2 (Tagalog, Cyrillic Komi) are word characters, while later additions (the Tamil digit zero) stay unassigned and match `\D` (`XSDCategory`, `Unicode32Assigned`). Per-child content assessment is driven by the content-model particle the automaton matched, not a by-name lookup (#180), so two same-named particles keep their own `fixed` values and an element matching a `skip` wildcard is not validated against its global declaration. Schema documents that form an import chain are compiled as a union rather than merged as separately compiled documents, so cross-document substitution groups resolve (#161, `Document.init(composing:)`). Plus attribute-group `anyAttribute` inheritance, extension wildcard union, namespace-qualified type tables for multi-document `xsi:schemaLocation`, per-container `elementFormDefault`, strict/lax attribute wildcard validation, built-in type resolution, and `xs:redefine` simpleType overrides. - Enforce element, attribute, attributeGroup, and group child-content shape (schema-for-schemas cardinality and order): at most one inline type per element/attribute, inline type before identity constraints, at most one compositor per group, at most one `anyAttribute` per attributeGroup with attribute references preceding it (`SchemaStructureHelpers.swift`). diff --git a/Sources/Schema/SchemaDocument.swift b/Sources/Schema/SchemaDocument.swift index 55eecc6..9446f49 100644 --- a/Sources/Schema/SchemaDocument.swift +++ b/Sources/Schema/SchemaDocument.swift @@ -123,8 +123,7 @@ public extension PureXML.Schema { /// schema source; it returns nil (the default) when external schemas are /// not available, which keeps compilation from reaching the filesystem or /// network by default. - public init(_ xsd: String, schemaLoader: @escaping (String) -> String? = { _ in nil }) throws { - let compiled = try XSDParser.parse(xsd, loader: schemaLoader) + private init(compiled: XSDCompiled) throws { elements = compiled.elements types = compiled.types constraints = compiled.constraints @@ -374,3 +373,21 @@ public extension PureXML.Schema { } } } + +public extension PureXML.Schema.Document { + /// Compiles a schema document. `schemaLoader` resolves the `schemaLocation` of + /// `xs:include`, `xs:import`, and `xs:redefine` to schema source; it returns + /// nil (the default) when external schemas are not available, which keeps + /// compilation from reaching the filesystem or network by default. + init(_ xsd: String, schemaLoader: @escaping (String) -> String? = { _ in nil }) throws { + try self.init(compiled: PureXML.Schema.XSDParser.parse(xsd, loader: schemaLoader)) + } + + /// Compiles several schema documents that jointly form one schema as a single + /// union (#161), so cross-document substitution-group membership is resolved. + /// Use this rather than compiling each document and merging the results, which + /// freezes each document's content models before the others are known. + init(composing sources: [String], schemaLoader: @escaping (String) -> String? = { _ in nil }) throws { + try self.init(compiled: PureXML.Schema.XSDParser.parse(union: sources, loader: schemaLoader)) + } +} diff --git a/Sources/Schema/XSDParser.swift b/Sources/Schema/XSDParser.swift index 1daea6d..3232ace 100644 --- a/Sources/Schema/XSDParser.swift +++ b/Sources/Schema/XSDParser.swift @@ -57,7 +57,7 @@ extension PureXML.Schema { } extension PureXML.Schema.XSDParser { - private static func containerLocationMap( + static func containerLocationMap( _ containerTuples: [(location: String?, tree: XSDTree)], rootLocation: String?, ) -> [ObjectIdentifier: String?] { @@ -68,13 +68,13 @@ extension PureXML.Schema.XSDParser { return containerLocations } - private static func createContext( + static func createContext( schema: XSDTree, containers: [XSDTree], derivation: DerivationTables, containerLocations: [ObjectIdentifier: String?], compositionLoaded: Bool, - ) -> XSDContext { + ) -> PureXML.Schema.XSDContext { var context = XSDContext( simpleTypes: [:], attributeGroups: indexByName(allChildren(containers, named: "attributeGroup")), @@ -137,12 +137,12 @@ extension PureXML.Schema.XSDParser { var identityFieldTypes: [String: SimpleType] } - private static func finishCompile( + static func finishCompile( schema: XSDTree, containers: [XSDTree], derivation: DerivationTables, - context: inout XSDContext, - ) -> XSDCompiled { + context: inout PureXML.Schema.XSDContext, + ) -> PureXML.Schema.XSDCompiled { for error in PureXML.Validation.SchemaCompile.preCompileErrors(schema: schema, context: context, containers: containers) { context.diagnostics.report(error) } diff --git a/Sources/Schema/XSDParserUnion.swift b/Sources/Schema/XSDParserUnion.swift new file mode 100644 index 0000000..4636791 --- /dev/null +++ b/Sources/Schema/XSDParserUnion.swift @@ -0,0 +1,76 @@ +private typealias XSDNode = PureXML.Schema.XSDNode + +extension PureXML.Schema.XSDParser { + /// Compiles several schema documents that jointly form one schema (a + /// multi-document `schemaTest`, or any set assembled outside the + /// `import`/`include` graph) as a single union: their container closures are + /// pooled into one context so cross-document facts (substitution-group + /// membership above all) are global, exactly as if one document imported the + /// rest. Merging separately compiled `Document`s cannot do this, since each + /// document's content models are already frozen with only the substitution + /// members visible in its own closure (#161). + /// + /// A document supplied directly and also reachable through another's `import` + /// is pooled once (deduplicated by target namespace and the set of its + /// top-level component names). The document with the largest closure, i.e. the + /// one that imports the others, is the primary, matching the result of + /// compiling that document alone. + static func parse(union sources: [String], loader: (String) -> String? = { _ in nil }) throws -> PureXML.Schema.XSDCompiled { + var parsed: [(schema: XSDTree, tuples: [(location: String?, tree: XSDTree)])] = [] + for source in sources { + var visited: Set = [] + guard let root = try? PureXML.parseTree(source), + let schema = XSDNode.elementChildren(root).first(where: { XSDNode.localName($0) == "schema" }) + else { continue } + parsed.append((schema, XSDNode.collectContainers(schema, loader, &visited))) + } + guard !parsed.isEmpty else { throw PureXML.Schema.SchemaError.notASchema } + // The widest closure is the importing document; compile as if it were the + // single entry point so the primary target namespace matches. Tie-break by + // target namespace so the choice is deterministic for independent roots. + parsed.sort { + $0.tuples.count != $1.tuples.count + ? $0.tuples.count > $1.tuples.count + : (XSDNode.attribute($0.schema, "targetNamespace") ?? "") < (XSDNode.attribute($1.schema, "targetNamespace") ?? "") + } + guard let schema = parsed.first?.schema else { throw PureXML.Schema.SchemaError.notASchema } + + var seenSignatures: Set = [] + var containerTuples: [(location: String?, tree: XSDTree)] = [] + for tuple in parsed.flatMap(\.tuples) { + if let signature = unionDedupSignature(tuple.tree) { + guard seenSignatures.insert(signature).inserted else { continue } + } + containerTuples.append(tuple) + } + + let containers = containerTuples.map(\.tree) + let derivation = derivationTables(containers) + try checkRedefine(containers) + try checkAllGroups(containers) + let containerLocations = containerLocationMap(containerTuples, rootLocation: nil) + let compositionLoaded = XSDNode.compositionLoaded(from: containerTuples) + var context = createContext( + schema: schema, + containers: containers, + derivation: derivation, + containerLocations: containerLocations, + compositionLoaded: compositionLoaded, + ) + return finishCompile(schema: schema, containers: containers, derivation: derivation, context: &context) + } + + /// A dedup key for a schema container in a union: its target namespace plus its + /// sorted top-level component names. The same document parsed twice (as a + /// direct source and through another's `import`) collapses to one. Only + /// namespaced `schema` containers are keyed; chameleon (no-namespace) includes + /// and `redefine` wrappers are parent-specific and always kept. + private static func unionDedupSignature(_ tree: XSDTree) -> String? { + guard XSDNode.localName(tree) == "schema" else { return nil } + guard let namespace = XSDNode.attribute(tree, "targetNamespace"), !namespace.isEmpty else { return nil } + let components = XSDNode.elementChildren(tree) + .map { "\(XSDNode.localName($0)):\(XSDNode.attribute($0, "name") ?? "")" } + .sorted() + return "\(namespace)\u{1}\(components.joined(separator: "\u{1}"))" + } +} diff --git a/Tests/XSTSSuiteTests.swift b/Tests/XSTSSuiteTests.swift index 5d55a81..e5785f6 100644 --- a/Tests/XSTSSuiteTests.swift +++ b/Tests/XSTSSuiteTests.swift @@ -45,12 +45,16 @@ struct XSTSSuiteTests { /// than a by-name lookup (#180) then cleared isDefault079 (two same-named /// particles with different `fixed` values) and QFE1700c2 (an element matching /// a `skip` wildcard must not be validated against its global declaration): - /// instances rejected 6 → 4, no other bucket moved. The remaining four are the - /// three Ethiopic-digit `\d` cases (reS17, reS38, reZ004v) and ctZ007 (a - /// cross-document substitution group, tracked under #180/#161). + /// instances rejected 6 → 4, no other bucket moved. + /// + /// Compiling an import chain of schema documents as a union rather than merging + /// separately compiled documents (#161) then cleared ctZ007, a cross-document + /// substitution group: instances rejected 4 → 3, no other bucket moved. The + /// remaining three are the Ethiopic-digit `\d` cases (reS17, reS38, reZ004v), + /// which contradict `\d` = `\p{Nd}` and are tracked as suspect tests. private let knownSchemaValidRejected = 1 private let knownSchemaInvalidAccepted = 262 - private let knownInstanceValidRejected = 4 + private let knownInstanceValidRejected = 3 private let knownInstanceInvalidAccepted = 147 @Test("Every XSTS case behaves: compile, reject, validate, invalidate") @@ -122,30 +126,7 @@ struct XSTSSuiteTests { let hrefs = references(in: .element(schemaTest), named: "schemaDocument") guard !hrefs.isEmpty else { continue } schemaExpectedValid = expectedValidity(schemaTest) != "invalid" - // A schemaTest's documents jointly form the schema under test (one is - // typically the entry point that imports the others; their order in the - // metadata is not significant). Compile each and merge so the schema is - // complete regardless of which document declares the root. - var compiled: PureXML.Schema.Document? - var allCompiled = true - for href in hrefs { - let schemaPath = resolve(href, against: directory) - let schemaDirectory = self.directory(of: schemaPath) - let loader: (String) -> String? = { location in - (try? String(contentsOfFile: resolve(location, against: schemaDirectory), encoding: .utf8)) - } - guard let source = try? String(contentsOfFile: schemaPath, encoding: .utf8) else { - counters.note("\(name): unreadable schema \(href)") - allCompiled = false - continue - } - guard let document = try? PureXML.Schema.Document(source, schemaLoader: loader) else { - allCompiled = false - continue - } - compiled = compiled.map { $0.merged(with: document) } ?? document - } - if !allCompiled { compiled = nil } + let compiled = compileSchemaDocuments(hrefs, directory: directory, name: name, into: &counters) switch (schemaExpectedValid, compiled == nil) { case (true, true): counters.schemaValidRejected += 1 @@ -161,6 +142,60 @@ struct XSTSSuiteTests { return (schema, schemaExpectedValid) } + /// Compiles a schemaTest's documents into one schema. When one document + /// imports another listed document they form an import chain and are compiled + /// as a union so cross-document substitution groups resolve (#161); otherwise + /// they are independent roots, composed through the lenient per-document merge + /// (a union of independent roots wrongly trips cross-document checks). + private func compileSchemaDocuments( + _ hrefs: [String], + directory: String, + name: String, + into counters: inout Counters, + ) -> PureXML.Schema.Document? { + let schemaDirectory = self.directory(of: resolve(hrefs[0], against: directory)) + let loader: (String) -> String? = { location in + (try? String(contentsOfFile: resolve(location, against: schemaDirectory), encoding: .utf8)) + } + let basenames = hrefs.map { String($0.split(separator: "/").last ?? "") } + var sources: [(name: String, text: String)] = [] + for (index, href) in hrefs.enumerated() { + guard let source = try? String(contentsOfFile: resolve(href, against: directory), encoding: .utf8) else { + counters.note("\(name): unreadable schema \(href)") + return nil + } + sources.append((basenames[index], source)) + } + let isImportChain = sources.contains { source in + importedBasenames(of: source.text).contains { $0 != source.name && basenames.contains($0) } + } + if isImportChain { + return try? PureXML.Schema.Document(composing: sources.map(\.text), schemaLoader: loader) + } + var merged: PureXML.Schema.Document? + for source in sources { + guard let document = try? PureXML.Schema.Document(source.text, schemaLoader: loader) else { return nil } + merged = merged.map { $0.merged(with: document) } ?? document + } + return merged + } + + /// The basenames a schema source imports, includes, or redefines, used to tell + /// an import chain (compile as a union) from independent roots (merge). Matches + /// the `schemaLocation` attribute, not a raw substring, so a name appearing in + /// documentation or as a suffix of another name does not misclassify. + private func importedBasenames(of source: String) -> Set { + guard let node = try? PureXML.parse(source, limits: .init(allowDoctype: true)) else { return [] } + var result: Set = [] + for kind in ["import", "include", "redefine"] { + for element in elements(named: kind, under: node) { + guard let location = element.attributes.first(where: { $0.name.localName == "schemaLocation" })?.value else { continue } + result.insert(String(location.split(separator: "/").last ?? "")) + } + } + return result + } + private func runInstanceTests( _ group: PureXML.Model.Element, name: String,