diff --git a/CHANGELOG.md b/CHANGELOG.md index fd03580..315a26f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Replaced XSD content-model occurrence unrolling with a counted automaton. + `ContentMatcher` now tracks occurrence counters in active configurations instead + of copying particle bodies into states, preserving exact finite `{m,n}` behavior + for high numeric bounds and removing the instance-time silent widening at the old + state ceilings. XSD occurrence attributes are preserved as decimal magnitudes + beyond `Int` range so `maxOccurs="100000000000000000000"` no longer falls back + to the default `1`. + - XSTS invalid-schemas-accepted (#145): global element references now carry the referenced declaration's type name, fixed/default constraint, nillable flag, and `block` metadata into Particle-Valid-Restriction. A local element with an diff --git a/Sources/Schema/ComplexType.swift b/Sources/Schema/ComplexType.swift index 45e9b31..5bb8772 100644 --- a/Sources/Schema/ComplexType.swift +++ b/Sources/Schema/ComplexType.swift @@ -179,13 +179,26 @@ public extension PureXML.Schema { /// A particle: a term with an occurrence range. `maxOccurs` nil means /// unbounded. struct Particle: Sendable { - public var minOccurs: Int - public var maxOccurs: Int? + public var occurrenceRange: OccurrenceRange public var term: Term + public var minOccurs: Int { + get { occurrenceRange.minimum.clamped(to: Int.max) } + set { occurrenceRange.minimum = NonNegativeDecimal(newValue) } + } + + public var maxOccurs: Int? { + get { occurrenceRange.maximum.clamped(to: Int.max) } + set { occurrenceRange.maximum = OccurrenceUpper(newValue) } + } + public init(minOccurs: Int = 1, maxOccurs: Int? = 1, term: Term) { - self.minOccurs = minOccurs - self.maxOccurs = maxOccurs + occurrenceRange = OccurrenceRange(minimum: minOccurs, maximum: maxOccurs) + self.term = term + } + + public init(occurrenceRange: OccurrenceRange, term: Term) { + self.occurrenceRange = occurrenceRange self.term = term } } diff --git a/Sources/Schema/ComplexValidatorContentModel.swift b/Sources/Schema/ComplexValidatorContentModel.swift index 61e6a3b..3ee5049 100644 --- a/Sources/Schema/ComplexValidatorContentModel.swift +++ b/Sources/Schema/ComplexValidatorContentModel.swift @@ -9,9 +9,10 @@ extension PureXML.Schema.ComplexValidator { let steps = Self.childSteps(children) // Advance one active state-set across the children rather than re-walking // the prefix per child (which is quadratic over the content model, #129). - var current = nfa.startStates() + let inputLength = children.count + var current = nfa.startStates(inputLength: inputLength) for (index, child) in children.enumerated() { - guard let next = nfa.step(current, over: child.name) else { + guard let next = nfa.step(current, over: child.name, inputLength: inputLength) else { let allowed = nfa.admissible(from: current) errors.append(XSDFailure(reason: "element '\(child.name.localName)' is not allowed here\(Self.expectation(allowed))", at: path + [steps[index]])) return diff --git a/Sources/Schema/ContentMatcher.swift b/Sources/Schema/ContentMatcher.swift index 09a6454..cc85d3b 100644 --- a/Sources/Schema/ContentMatcher.swift +++ b/Sources/Schema/ContentMatcher.swift @@ -14,9 +14,54 @@ extension PureXML.Schema { } } - /// One state of the content-model automaton. + /// A guard on a counted epsilon transition. + enum ContentCounterGuard: Sendable { + case canExit(counter: Int, minimum: NonNegativeDecimal, nullableBody: Bool) + case belowMaximum(counter: Int, maximum: OccurrenceUpper) + + func accepts(_ values: [Int]) -> Bool { + switch self { + case let .canExit(counter, minimum, nullableBody): + nullableBody || minimum.isLessThanOrEqual(to: values[counter]) + case let .belowMaximum(counter, maximum): + maximum.isGreaterThan(values[counter]) + } + } + } + + /// An action on a counted epsilon transition. + enum ContentCounterAction: Sendable { + case reset(Int) + case increment(Int) + + func apply(to values: inout [Int], limit: Int) { + switch self { + case let .reset(counter): + values[counter] = 0 + case let .increment(counter): + if values[counter] < limit { + values[counter] += 1 + } + } + } + } + + /// One epsilon transition in the counted content-model automaton. + struct ContentEpsilonEdge: Sendable { + var target: Int + var guards: [ContentCounterGuard] = [] + var actions: [ContentCounterAction] = [] + } + + /// One occurrence counter in the counted content-model automaton. + struct ContentCounterScope: Sendable { + var range: OccurrenceRange + var nullableBody: Bool + } + + /// One state of the counted content-model automaton. struct ContentState: Sendable { - var epsilon: [Int] = [] + var epsilon: [ContentEpsilonEdge] = [] var label: TermLabel? var target: Int? /// The declared type and value constraint of the element particle this @@ -36,17 +81,26 @@ extension PureXML.Schema { case wildcard(Wildcard) } - /// A Thompson NFA over element names compiled from a ``Particle``. `all` groups - /// are validated separately by ``ContentMatcher`` and are not represented here. + /// One active configuration in the counted content-model automaton. + struct ContentConfiguration: Sendable, Hashable { + var state: Int + var counters: [Int] + } + + /// A counted automaton over element names compiled from a ``Particle``. `all` + /// groups are validated separately by ``ContentMatcher`` and are not represented + /// here. struct ContentNFA: Sendable { var states: [ContentState] + var counters: [ContentCounterScope] var start: Int var accept: Int func matchesWhole(_ names: [PureXML.Model.QualifiedName]) -> Bool { - var current = startStates() + let inputLength = names.count + var current = startStates(inputLength: inputLength) for name in names { - guard let next = step(current, over: name) else { return false } + guard let next = step(current, over: name, inputLength: inputLength) else { return false } current = next } return isComplete(current) @@ -58,9 +112,10 @@ extension PureXML.Schema { /// whether something is still required. Returns an empty, not-complete /// result when the prefix is already invalid. func follow(after names: [PureXML.Model.QualifiedName]) -> (allowed: [TermLabel], complete: Bool) { - var current = startStates() + let inputLength = names.count + var current = startStates(inputLength: inputLength) for name in names { - guard let next = step(current, over: name) else { return ([], false) } + guard let next = step(current, over: name, inputLength: inputLength) else { return ([], false) } current = next } return (admissible(from: current), isComplete(current)) @@ -69,22 +124,23 @@ extension PureXML.Schema { // MARK: Incremental matching // - // The active set is the epsilon-closure of the states reachable after some + // The active set is the epsilon-closure of the configurations reachable after some // prefix of children. Callers that scan a child sequence advance it one // child at a time (`startStates` then `step` per child) rather than calling // `follow(after:)` on a growing prefix, which re-walks the whole prefix each // time and is quadratic in the child count over a content model (#129). /// The active set before any child: the closure of the start state. - func startStates() -> Set { - closure([start]) + func startStates(inputLength: Int = 0) -> Set { + let seed = ContentConfiguration(state: start, counters: Array(repeating: 0, count: counters.count)) + return closure([seed], counterLimit: counterLimit(for: inputLength)) } /// The labels admissible from `current`, for diagnostics and completions. - func admissible(from current: Set) -> [TermLabel] { + func admissible(from current: Set) -> [TermLabel] { var labels: [TermLabel] = [] - for state in current where states[state].target != nil { - if let label = states[state].label { labels.append(label) } + for configuration in current where states[configuration.state].target != nil { + if let label = states[configuration.state].label { labels.append(label) } } return labels } @@ -100,207 +156,95 @@ extension PureXML.Schema { /// reproducible rather than left to `Set` iteration order: a named particle /// is preferred over a wildcard, then the lowest state index wins. func matchedParticles(_ names: [PureXML.Model.QualifiedName]) -> [MatchedParticle?] { - var current = startStates() + let inputLength = names.count + var current = startStates(inputLength: inputLength) var result: [MatchedParticle?] = [] for name in names { - var next: Set = [] - var namedMatch: Int? - var anyMatch: Int? - for state in current { + var next: Set = [] + var namedMatch: ContentConfiguration? + var namedState: Int? + var anyMatch: ContentConfiguration? + var anyState: Int? + for configuration in current { + let state = configuration.state guard let label = states[state].label, let target = states[state].target, label.matches(name) else { continue } - next.insert(target) - if anyMatch.map({ state < $0 }) ?? true { anyMatch = state } - if case .name = label, namedMatch.map({ state < $0 }) ?? true { namedMatch = state } + next.insert(ContentConfiguration(state: target, counters: configuration.counters)) + if anyState.map({ state < $0 }) ?? true { + anyState = state + anyMatch = configuration + } + if case .name = label, namedState.map({ state < $0 }) ?? true { + namedState = state + namedMatch = configuration + } } - guard let chosen = namedMatch ?? anyMatch, !next.isEmpty else { + guard let chosenState = namedState ?? anyState, (namedMatch ?? anyMatch) != nil, !next.isEmpty else { result.append(contentsOf: Array(repeating: nil, count: names.count - result.count)) return result } - let matched = states[chosen] + let matched = states[chosenState] if case let .wildcard(wildcard) = matched.label { result.append(.wildcard(wildcard)) } else { result.append(.element(type: matched.elementType, valueConstraint: matched.valueConstraint)) } - current = closure(next) + current = closure(next, counterLimit: counterLimit(for: inputLength)) } return result } /// Advances `current` by consuming one element `name`, or nil when `name` /// is not admissible there. - func step(_ current: Set, over name: PureXML.Model.QualifiedName) -> Set? { - var next: Set = [] - for state in current { + func step(_ current: Set, over name: PureXML.Model.QualifiedName, inputLength: Int = 0) -> Set? { + var next: Set = [] + for configuration in current { + let state = configuration.state if let label = states[state].label, let target = states[state].target, label.matches(name) { - next.insert(target) + next.insert(ContentConfiguration(state: target, counters: configuration.counters)) } } if next.isEmpty { return nil } - return closure(next) + return closure(next, counterLimit: counterLimit(for: inputLength)) } /// Whether the content may legally end at `current`. - func isComplete(_ current: Set) -> Bool { - current.contains(accept) + func isComplete(_ current: Set) -> Bool { + current.contains { $0.state == accept } } - private func closure(_ seed: Set) -> Set { + private func closure(_ seed: Set, counterLimit: Int) -> Set { var seen = seed var stack = Array(seed) - while let state = stack.popLast() { - for next in states[state].epsilon where seen.insert(next).inserted { + while let configuration = stack.popLast() { + for edge in states[configuration.state].epsilon { + guard let next = apply(edge, to: configuration, counterLimit: counterLimit), + seen.insert(next).inserted + else { + continue + } stack.append(next) } } return seen } - } - /// Builds a ``ContentNFA`` from a particle by Thompson construction. - struct ContentNFABuilder { - private var states: [ContentState] = [] - - static func build(_ particle: PureXML.Schema.Particle) -> ContentNFA { - var builder = ContentNFABuilder() - let fragment = builder.particle(particle) - return ContentNFA(states: builder.states, start: fragment.start, accept: fragment.accept) - } - - private mutating func addState() -> Int { - states.append(ContentState()) - return states.count - 1 - } - - /// Occurrence counts unroll into automaton states, so they are - /// capped: beyond the cap a repetition is treated as unbounded, the - /// libxml2 posture (its cap is 16384). Without this, a schema with - /// maxOccurs in the trillions allocates until the process dies - /// (found by XSTS groupF009v, #129). - private static let occursUnrollCap = 16384 - - /// The per-particle cap bounds one repetition, but nested particles - /// multiply: a sequence of 16384 holding a choice of 16384 is 2.7e8 - /// states even though each particle is individually capped. So the - /// total state count is also capped; once an NFA reaches the ceiling - /// every further repetition degrades to star, the same posture the - /// per-particle cap already takes. Legitimate content models are tens - /// of states, so only the pathological XSTS particle schemas (the - /// msMeta Particles set, which OOM-killed the suite at 8 GB, #129) - /// reach it. The ceiling sits well above a single maxed particle - /// (~33k states) so capped-but-not-nested schemas are unaffected. - private static let totalStateCap = 1 << 20 - - private var stateBudgetExhausted: Bool { - states.count >= Self.totalStateCap - } - - private mutating func particle(_ particle: PureXML.Schema.Particle) -> (start: Int, accept: Int) { - let start = addState() - var current = start - let boundedMin = Swift.min(particle.minOccurs, Self.occursUnrollCap) - var unrolledMin = 0 - for _ in 0 ..< boundedMin { - if stateBudgetExhausted { break } - let part = term(particle.term) - states[current].epsilon.append(part.start) - current = part.accept - unrolledMin += 1 + private func apply( + _ edge: ContentEpsilonEdge, + to configuration: ContentConfiguration, + counterLimit: Int, + ) -> ContentConfiguration? { + for guardCondition in edge.guards where !guardCondition.accepts(configuration.counters) { + return nil } - let accept = addState() - if stateBudgetExhausted || unrolledMin < boundedMin || particle.minOccurs > Self.occursUnrollCap { - // The tail of an absurd minOccurs, or a run that hit the total - // state ceiling mid-unroll, is approximated as star. - appendStar(particle.term, from: current, to: accept) - } else if let maximum = particle.maxOccurs, maximum - boundedMin <= Self.occursUnrollCap { - appendOptional(particle.term, count: maximum - boundedMin, from: current, to: accept) - } else { - appendStar(particle.term, from: current, to: accept) + var values = configuration.counters + for action in edge.actions { + action.apply(to: &values, limit: counterLimit) } - return (start, accept) + return ContentConfiguration(state: edge.target, counters: values) } - private mutating func appendOptional(_ term: PureXML.Schema.Term, count: Int, from: Int, to accept: Int) { - var current = from - for _ in 0 ..< Swift.max(0, count) { - if stateBudgetExhausted { - // Ran out of state budget: let the remainder repeat freely - // (star) rather than keep allocating bounded optionals. - appendStar(term, from: current, to: accept) - return - } - let part = self.term(term) - states[current].epsilon.append(part.start) - states[current].epsilon.append(accept) - current = part.accept - } - states[current].epsilon.append(accept) - } - - private mutating func appendStar(_ term: PureXML.Schema.Term, from current: Int, to accept: Int) { - let part = self.term(term) - states[current].epsilon.append(part.start) - states[current].epsilon.append(accept) - states[part.accept].epsilon.append(part.start) - states[part.accept].epsilon.append(accept) - } - - private mutating func term(_ term: PureXML.Schema.Term) -> (start: Int, accept: Int) { - switch term { - case let .element(name, type, _, constraint, _, _): - let fragment = labeled(.name(name)) - states[fragment.start].elementType = type - states[fragment.start].valueConstraint = constraint - return fragment - case let .wildcard(wildcard): - return labeled(.wildcard(wildcard)) - case let .group(group): - return self.group(group) - } - } - - private mutating func labeled(_ label: TermLabel) -> (start: Int, accept: Int) { - let start = addState() - let accept = addState() - states[start].label = label - states[start].target = accept - return (start, accept) - } - - private mutating func group(_ group: PureXML.Schema.Group) -> (start: Int, accept: Int) { - // `all` is validated by counting, not in the NFA; treat it as a - // sequence here so a nested occurrence still parses. - group.compositor == .choice ? choice(group.particles) : sequence(group.particles) - } - - private mutating func sequence(_ particles: [PureXML.Schema.Particle]) -> (start: Int, accept: Int) { - guard !particles.isEmpty else { return labeledEmpty() } - var first: Int? - var last: Int? - for member in particles { - let part = particle(member) - if let previous = last { states[previous].epsilon.append(part.start) } else { first = part.start } - last = part.accept - } - return (first ?? addState(), last ?? addState()) - } - - private mutating func choice(_ particles: [PureXML.Schema.Particle]) -> (start: Int, accept: Int) { - let start = addState() - let accept = addState() - for member in particles { - let part = particle(member) - states[start].epsilon.append(part.start) - states[part.accept].epsilon.append(accept) - } - return (start, accept) - } - - private mutating func labeledEmpty() -> (start: Int, accept: Int) { - let start = addState() - let accept = addState() - states[start].epsilon.append(accept) - return (start, accept) + private func counterLimit(for inputLength: Int) -> Int { + inputLength == Int.max ? Int.max : inputLength + 1 } } } diff --git a/Sources/Schema/ContentNFABuilder.swift b/Sources/Schema/ContentNFABuilder.swift new file mode 100644 index 0000000..b355201 --- /dev/null +++ b/Sources/Schema/ContentNFABuilder.swift @@ -0,0 +1,169 @@ +extension PureXML.Schema { + struct ContentNFAFragment { + var start: Int + var accept: Int + var counters: [Int] + var nullable: Bool + } + + /// Builds a counted ``ContentNFA`` from a particle. + struct ContentNFABuilder { + private var states: [ContentState] = [] + private var counters: [ContentCounterScope] = [] + + static func build(_ particle: PureXML.Schema.Particle) -> ContentNFA { + var builder = ContentNFABuilder() + let fragment = builder.particle(particle) + return ContentNFA( + states: builder.states, + counters: builder.counters, + start: fragment.start, + accept: fragment.accept, + ) + } + + private mutating func addState() -> Int { + states.append(ContentState()) + return states.count - 1 + } + + private mutating func addEpsilon( + from source: Int, + to target: Int, + guards: [ContentCounterGuard] = [], + actions: [ContentCounterAction] = [], + ) { + states[source].epsilon.append(ContentEpsilonEdge(target: target, guards: guards, actions: actions)) + } + + private mutating func particle(_ particle: PureXML.Schema.Particle) -> ContentNFAFragment { + let start = addState() + let loop = addState() + let accept = addState() + let counter = counters.count + let bodyNullable = termIsNullable(particle.term) + counters.append(ContentCounterScope(range: particle.occurrenceRange, nullableBody: bodyNullable)) + let body = term(particle.term) + let resetAll = [ContentCounterAction.reset(counter)] + body.counters.map(ContentCounterAction.reset) + addEpsilon(from: start, to: loop, actions: resetAll) + addEpsilon( + from: loop, + to: accept, + guards: [ + .canExit( + counter: counter, + minimum: particle.occurrenceRange.minimum, + nullableBody: bodyNullable, + ), + ], + ) + addEpsilon( + from: loop, + to: body.start, + guards: [.belowMaximum(counter: counter, maximum: particle.occurrenceRange.maximum)], + actions: body.counters.map(ContentCounterAction.reset), + ) + addEpsilon(from: body.accept, to: loop, actions: [.increment(counter)]) + return ContentNFAFragment( + start: start, + accept: accept, + counters: [counter] + body.counters, + nullable: particleIsNullable(particle), + ) + } + + private mutating func term(_ term: PureXML.Schema.Term) -> ContentNFAFragment { + switch term { + case let .element(name, type, _, constraint, _, _): + let fragment = labeled(.name(name)) + states[fragment.start].elementType = type + states[fragment.start].valueConstraint = constraint + return fragment + case let .wildcard(wildcard): + return labeled(.wildcard(wildcard)) + case let .group(group): + return self.group(group) + } + } + + private mutating func labeled(_ label: TermLabel) -> ContentNFAFragment { + let start = addState() + let accept = addState() + states[start].label = label + states[start].target = accept + return ContentNFAFragment(start: start, accept: accept, counters: [], nullable: false) + } + + private mutating func group(_ group: PureXML.Schema.Group) -> ContentNFAFragment { + // `all` is validated by counting, not in the NFA; treat it as a + // sequence here so a nested occurrence still parses. + group.compositor == .choice ? choice(group.particles) : sequence(group.particles) + } + + private mutating func sequence(_ particles: [PureXML.Schema.Particle]) -> ContentNFAFragment { + guard !particles.isEmpty else { return labeledEmpty() } + var first: Int? + var last: Int? + var fragmentCounters: [Int] = [] + var nullable = true + for member in particles { + let part = particle(member) + if let previous = last { addEpsilon(from: previous, to: part.start) } else { first = part.start } + last = part.accept + fragmentCounters.append(contentsOf: part.counters) + nullable = nullable && part.nullable + } + return ContentNFAFragment( + start: first ?? addState(), + accept: last ?? addState(), + counters: fragmentCounters, + nullable: nullable, + ) + } + + private mutating func choice(_ particles: [PureXML.Schema.Particle]) -> ContentNFAFragment { + let start = addState() + let accept = addState() + var fragmentCounters: [Int] = [] + var nullable = false + for member in particles { + let part = particle(member) + addEpsilon(from: start, to: part.start) + addEpsilon(from: part.accept, to: accept) + fragmentCounters.append(contentsOf: part.counters) + nullable = nullable || part.nullable + } + return ContentNFAFragment( + start: start, + accept: accept, + counters: fragmentCounters, + nullable: nullable, + ) + } + + private mutating func labeledEmpty() -> ContentNFAFragment { + let start = addState() + let accept = addState() + addEpsilon(from: start, to: accept) + return ContentNFAFragment(start: start, accept: accept, counters: [], nullable: true) + } + + private func particleIsNullable(_ particle: PureXML.Schema.Particle) -> Bool { + particle.occurrenceRange.minimum.isZero || termIsNullable(particle.term) + } + + private func termIsNullable(_ term: PureXML.Schema.Term) -> Bool { + switch term { + case .element, .wildcard: + false + case let .group(group): + switch group.compositor { + case .sequence, .all: + group.particles.allSatisfy(particleIsNullable) + case .choice: + group.particles.contains(where: particleIsNullable) + } + } + } + } +} diff --git a/Sources/Schema/OccurrenceRange.swift b/Sources/Schema/OccurrenceRange.swift new file mode 100644 index 0000000..1b7d26b --- /dev/null +++ b/Sources/Schema/OccurrenceRange.swift @@ -0,0 +1,106 @@ +public extension PureXML.Schema { + /// A non-negative integer magnitude preserved as normalized decimal text. + /// + /// XSD occurrence attributes are `nonNegativeInteger` values and may exceed + /// the platform `Int` range. This type compares such values without converting + /// them to machine integers. + struct NonNegativeDecimal: Sendable, Hashable, Comparable, CustomStringConvertible { + private var digits: String + + public init(_ value: Int) { + precondition(value >= 0, "occurrence values must be non-negative") + digits = String(value) + } + + public init?(lexical value: String) { + var raw = Substring(value.trimmingXMLWhitespace()) + if raw.first == "+" { raw = raw.dropFirst() } + guard !raw.isEmpty, raw.allSatisfy({ $0.isASCII && $0.isNumber }) else { + return nil + } + let trimmed = raw.drop { $0 == "0" } + digits = trimmed.isEmpty ? "0" : String(trimmed) + } + + public var description: String { + digits + } + + var isZero: Bool { + digits == "0" + } + + func clamped(to limit: Int) -> Int { + precondition(limit >= 0, "occurrence clamp limit must be non-negative") + if isGreaterThan(limit) { return limit } + return Int(digits) ?? limit + } + + func isGreaterThan(_ value: Int) -> Bool { + precondition(value >= 0, "occurrence comparison value must be non-negative") + let other = String(value) + if digits.count != other.count { return digits.count > other.count } + return digits > other + } + + func isLessThanOrEqual(to value: Int) -> Bool { + !isGreaterThan(value) + } + + public static func < (lhs: Self, rhs: Self) -> Bool { + if lhs.digits.count != rhs.digits.count { + return lhs.digits.count < rhs.digits.count + } + return lhs.digits < rhs.digits + } + } + + /// The upper occurrence bound of an XSD particle. + enum OccurrenceUpper: Sendable, Hashable, Equatable { + case finite(NonNegativeDecimal) + case unbounded + + public init(_ value: Int?) { + if let value { + self = .finite(NonNegativeDecimal(value)) + } else { + self = .unbounded + } + } + + public var isZero: Bool { + if case let .finite(value) = self { return value.isZero } + return false + } + + func clamped(to limit: Int) -> Int? { + switch self { + case let .finite(value): value.clamped(to: limit) + case .unbounded: nil + } + } + + func isGreaterThan(_ value: Int) -> Bool { + switch self { + case let .finite(upper): upper.isGreaterThan(value) + case .unbounded: true + } + } + } + + /// The full occurrence range of an XSD particle. + struct OccurrenceRange: Sendable, Hashable, Equatable { + public var minimum: NonNegativeDecimal + public var maximum: OccurrenceUpper + + public init(minimum: NonNegativeDecimal = NonNegativeDecimal(1), maximum: OccurrenceUpper = .finite(NonNegativeDecimal(1))) { + self.minimum = minimum + self.maximum = maximum + } + + public init(minimum: Int = 1, maximum: Int? = 1) { + self.minimum = NonNegativeDecimal(minimum) + self.maximum = OccurrenceUpper(maximum) + } + } +} diff --git a/Sources/Schema/XSDContext.swift b/Sources/Schema/XSDContext.swift index d0c5a8c..3eb7952 100644 --- a/Sources/Schema/XSDContext.swift +++ b/Sources/Schema/XSDContext.swift @@ -83,10 +83,23 @@ extension PureXML.Schema { } static func occurrence(_ node: XSDTree) -> (min: Int, max: Int?) { - let minimum = attribute(node, "minOccurs").flatMap(Int.init) ?? 1 - let maximumText = attribute(node, "maxOccurs") - let maximum: Int? = maximumText == "unbounded" ? nil : (maximumText.flatMap(Int.init) ?? 1) - return (minimum, maximum) + let range = occurrenceRange(node) + return ( + range.minimum.clamped(to: Int.max), + range.maximum.clamped(to: Int.max), + ) + } + + static func occurrenceRange(_ node: XSDTree) -> OccurrenceRange { + let minimum = attribute(node, "minOccurs") + .flatMap { NonNegativeDecimal(lexical: $0) } ?? NonNegativeDecimal(1) + let maximumText = attribute(node, "maxOccurs")?.trimmingXMLWhitespace() + let maximum: OccurrenceUpper = if maximumText == "unbounded" { + .unbounded + } else { + .finite(maximumText.flatMap { NonNegativeDecimal(lexical: $0) } ?? NonNegativeDecimal(1)) + } + return OccurrenceRange(minimum: minimum, maximum: maximum) } /// Collects the schema's own element and a transitive closure of the diff --git a/Sources/Schema/XSDParserParticles.swift b/Sources/Schema/XSDParserParticles.swift index f755600..a2774f1 100644 --- a/Sources/Schema/XSDParserParticles.swift +++ b/Sources/Schema/XSDParserParticles.swift @@ -29,22 +29,26 @@ extension PureXML.Schema.XSDParser { for child in XSDNode.elementChildren(node) { if let member = particle(child, context) { particles.append(member) } } - let (minimum, maximum) = XSDNode.occurrence(node) - if compositor == .all, minimum == 0 { - particles = particles.map { Particle(minOccurs: 0, maxOccurs: $0.maxOccurs, term: $0.term) } + let occurrence = XSDNode.occurrenceRange(node) + if compositor == .all, occurrence.minimum.isZero { + particles = particles.map { + Particle( + occurrenceRange: .init(minimum: .init(0), maximum: $0.occurrenceRange.maximum), + term: $0.term, + ) + } } return Particle( - minOccurs: minimum, - maxOccurs: maximum, + occurrenceRange: occurrence, term: .group(Group(compositor: compositor, particles: particles)), ) } private static func particle(_ node: PureXML.Model.TreeNode, _ context: XSDContext) -> Particle? { - let (minimum, maximum) = XSDNode.occurrence(node) + let occurrence = XSDNode.occurrenceRange(node) switch XSDNode.localName(node) { case "element": - return elementParticle(node, minimum, maximum, context) + return elementParticle(node, occurrence, context) case "sequence": return groupParticle(node, .sequence, context) case "choice": @@ -52,22 +56,21 @@ extension PureXML.Schema.XSDParser { case "all": return groupParticle(node, .all, context) case "group": - return groupReferenceParticle(node, minimum, maximum, context) + return groupReferenceParticle(node, occurrence, context) case "any": - return Particle(minOccurs: minimum, maxOccurs: maximum, term: .wildcard(wildcard(node, context))) + return Particle(occurrenceRange: occurrence, term: .wildcard(wildcard(node, context))) default: return nil } } - private static func elementParticle(_ node: PureXML.Model.TreeNode, _ minimum: Int, _ maximum: Int?, _ context: XSDContext) -> Particle { + private static func elementParticle(_ node: PureXML.Model.TreeNode, _ occurrence: PureXML.Schema.OccurrenceRange, _ context: XSDContext) -> Particle { if let ref = XSDNode.attribute(node, "ref") { - return referenceParticle(ref, minimum, maximum, context, node: node) + return referenceParticle(ref, occurrence, context, node: node) } let name = XSDNode.attribute(node, "name") ?? "" return Particle( - minOccurs: minimum, - maxOccurs: maximum, + occurrenceRange: occurrence, term: .element( name: localElementName(name, XSDNode.attribute(node, "form"), context), type: elementType(node, context), @@ -79,7 +82,7 @@ extension PureXML.Schema.XSDParser { ) } - private static func groupReferenceParticle(_ node: PureXML.Model.TreeNode, _ minimum: Int, _ maximum: Int?, _ context: XSDContext) -> Particle? { + private static func groupReferenceParticle(_ node: PureXML.Model.TreeNode, _ occurrence: PureXML.Schema.OccurrenceRange, _ context: XSDContext) -> Particle? { guard let ref = XSDNode.attribute(node, "ref") else { return nil } let name = XSDNode.stripPrefix(ref) if context.visitingGroups.contains(name) { @@ -92,7 +95,7 @@ extension PureXML.Schema.XSDParser { guard let inner = modelGroup(in: base, scoped.visiting(name)) else { return nil } - return Particle(minOccurs: minimum, maxOccurs: maximum, term: inner.term) + return Particle(occurrenceRange: occurrence, term: inner.term) } guard let definition = context.groups[name] else { return nil @@ -101,7 +104,7 @@ extension PureXML.Schema.XSDParser { guard let inner = modelGroup(in: definition, scoped.visiting(name)) else { return nil } - return Particle(minOccurs: minimum, maxOccurs: maximum, term: inner.term) + return Particle(occurrenceRange: occurrence, term: inner.term) } /// The particle for an ``. An abstract head may not appear @@ -113,6 +116,15 @@ extension PureXML.Schema.XSDParser { _ maximum: Int?, _ context: PureXML.Schema.XSDContext, node: XSDTree? = nil, + ) -> PureXML.Schema.Particle { + referenceParticle(ref, .init(minimum: minimum, maximum: maximum), context, node: node) + } + + static func referenceParticle( + _ ref: String, + _ occurrence: PureXML.Schema.OccurrenceRange, + _ context: PureXML.Schema.XSDContext, + node: XSDTree? = nil, ) -> PureXML.Schema.Particle { let name = PureXML.Schema.XSDNode.stripPrefix(ref) let bindings = node.map { PureXML.Schema.XSDParser.namespaceBindingsInScope(of: $0, defaultBindings: context.namespaceBindings) } @@ -125,10 +137,10 @@ extension PureXML.Schema.XSDParser { // member's own namespace, even across imported namespaces. let alternatives = head + (context.substitutions[headKey] ?? []).map { PureXML.Schema.XSDParser.unpackElementName($0) } if alternatives.count == 1 { - return PureXML.Schema.Particle(minOccurs: minimum, maxOccurs: maximum, term: elementReferenceTerm(alternatives[0].0, alternatives[0].1, context)) + return PureXML.Schema.Particle(occurrenceRange: occurrence, term: elementReferenceTerm(alternatives[0].0, alternatives[0].1, context)) } let members = alternatives.map { PureXML.Schema.Particle(term: elementReferenceTerm($0.0, $0.1, context)) } - return PureXML.Schema.Particle(minOccurs: minimum, maxOccurs: maximum, term: .group(.init(compositor: .choice, particles: members))) + return PureXML.Schema.Particle(occurrenceRange: occurrence, term: .group(.init(compositor: .choice, particles: members))) } /// The term for a reference to a global element, in the resolved `namespace`. diff --git a/Tests/ContentNFAStateBudgetTests.swift b/Tests/ContentNFAStateBudgetTests.swift index 051c9aa..ca5c52f 100644 --- a/Tests/ContentNFAStateBudgetTests.swift +++ b/Tests/ContentNFAStateBudgetTests.swift @@ -1,52 +1,103 @@ import Testing @testable import PureXML -/// Guards the content-model NFA builder against occurrence-explosion (#129). -/// The per-particle unroll cap bounds one repetition, but nested high-`maxOccurs` -/// particles multiply: the XSTS msMeta Particles set drove the builder to 8 GB -/// and OOM-killed the suite. A total state ceiling caps the whole automaton, -/// degrading further repetition to star. This locks that ceiling in: the -/// pathological model must build, stay bounded, and still match without -/// crashing, while ordinary models are untouched. -@Suite("Content NFA state budget") +/// Guards the counted content-model automaton against occurrence explosion (#129). +/// Occurrence bounds must affect counters, not program size, and finite bounds +/// must stay exact instead of widening to `*`. +@Suite("Counted content automaton") struct ContentNFAStateBudgetTests { - private func element(_ name: String, min: Int, max: Int?) -> PureXML.Schema.Particle { + private typealias Schema = PureXML.Schema + private typealias Name = PureXML.Model.QualifiedName + + private func decimal(_ value: String) throws -> Schema.NonNegativeDecimal { + try #require(Schema.NonNegativeDecimal(lexical: value)) + } + + private func element(_ name: String, min: Int = 1, max: Int? = 1) -> Schema.Particle { .init(minOccurs: min, maxOccurs: max, term: .element(name: .init(name), type: nil, typeName: nil)) } - /// A sequence repeated `min` times, each repeat holding an element repeated - /// `min` times: naively `min * min` automaton states. - private func nested(min: Int) -> PureXML.Schema.Particle { - let inner = element("a", min: min, max: min) - return .init( - minOccurs: min, - maxOccurs: min, - term: .group(.init(compositor: .sequence, particles: [inner])), - ) + private func element(_ name: String, range: Schema.OccurrenceRange) -> Schema.Particle { + .init(occurrenceRange: range, term: .element(name: .init(name), type: nil, typeName: nil)) } - @Test("A nested occurrence explosion stays bounded instead of exhausting memory") + private func group(_ compositor: Schema.Compositor, _ particles: [Schema.Particle], min: Int = 1, max: Int? = 1) -> Schema.Particle { + .init(minOccurs: min, maxOccurs: max, term: .group(.init(compositor: compositor, particles: particles))) + } + + private func names(_ values: String...) -> [Name] { + values.map(Name.init) + } + + @Test("Finite exact occurrence rejects too few and too many children") + func test_exactFiniteOccurrence() { + let nfa = Schema.ContentNFABuilder.build(element("a", min: 2, max: 2)) + #expect(!nfa.matchesWhole(names("a"))) + #expect(nfa.matchesWhole(names("a", "a"))) + #expect(!nfa.matchesWhole(names("a", "a", "a"))) + } + + @Test("Finite optional tail does not widen to star") + func test_finiteOptionalTailDoesNotWiden() { + let nfa = Schema.ContentNFABuilder.build(element("a", min: 0, max: 2)) + #expect(nfa.matchesWhole([])) + #expect(nfa.matchesWhole(names("a"))) + #expect(nfa.matchesWhole(names("a", "a"))) + #expect(!nfa.matchesWhole(names("a", "a", "a"))) + } + + @Test("Huge finite upper bound stays exact without proportional states") + func test_hugeFiniteUpperBound() throws { + let huge = try decimal("100000000000000000000") + let particle = element( + "a", + range: .init(minimum: .init(2), maximum: .finite(huge)), + ) + let nfa = Schema.ContentNFABuilder.build(particle) + #expect(nfa.states.count < 16) + #expect(!nfa.matchesWhole(names("a"))) + #expect(nfa.matchesWhole(names("a", "a"))) + #expect(nfa.matchesWhole(names("a", "a", "a", "a"))) + } + + @Test("Nested finite occurrences compose exactly") + func test_nestedFiniteOccurrences() { + let inner = element("a", min: 2, max: 3) + let outer = group(.sequence, [inner], min: 2, max: 2) + let nfa = Schema.ContentNFABuilder.build(outer) + #expect(!nfa.matchesWhole(names("a", "a", "a"))) + #expect(nfa.matchesWhole(names("a", "a", "a", "a"))) + #expect(nfa.matchesWhole(names("a", "a", "a", "a", "a"))) + #expect(nfa.matchesWhole(names("a", "a", "a", "a", "a", "a"))) + #expect(!nfa.matchesWhole(names("a", "a", "a", "a", "a", "a", "a"))) + } + + @Test("A huge required prefix does not offer the following particle early") + func test_hugeMinimumFollowSet() throws { + let huge = try decimal("1000000000000") + let requiredA = element( + "a", + range: .init(minimum: huge, maximum: .finite(huge)), + ) + let particle = group(.sequence, [requiredA, element("b")]) + let nfa = Schema.ContentNFABuilder.build(particle) + + let follow = nfa.follow(after: names("a", "a")) + let allowedNames = follow.allowed.compactMap { label -> String? in + if case let .name(name) = label { return name.localName } + return nil + } + + #expect(Set(allowedNames) == ["a"]) + #expect(!follow.complete) + } + + @Test("A nested occurrence explosion stays structurally bounded") func test_nestedExplosionBounded() { - // 16384 * 16384 ~ 2.7e8 states without a total cap; the ceiling is 2^20. - let nfa = PureXML.Schema.ContentNFABuilder.build(nested(min: 16384)) - // Bounded to the ceiling plus at most one in-flight term expansion, - // far below the naive product that previously allocated until death. - #expect(nfa.states.count < 1_500_000) - // The degraded automaton must still answer queries without crashing. - let names = Array(repeating: PureXML.Model.QualifiedName("a"), count: 32) - _ = nfa.matchesWhole(names) - } - - @Test("Ordinary content models are unaffected by the ceiling") - func test_ordinaryModelUnchanged() { - let particle = PureXML.Schema.Particle( - minOccurs: 1, - maxOccurs: nil, - term: .group(.init(compositor: .sequence, particles: [element("a", min: 1, max: 1)])), - ) - let nfa = PureXML.Schema.ContentNFABuilder.build(particle) - #expect(nfa.states.count < 64) - #expect(nfa.matchesWhole([.init("a"), .init("a"), .init("a")])) - #expect(!nfa.matchesWhole([.init("b")])) + let inner = element("a", min: 16384, max: 16384) + let nested = group(.sequence, [inner], min: 16384, max: 16384) + let nfa = Schema.ContentNFABuilder.build(nested) + #expect(nfa.states.count < 32) + #expect(!nfa.matchesWhole(Array(repeating: Name("a"), count: 32))) } } diff --git a/Tests/XSDSchemaTests.swift b/Tests/XSDSchemaTests.swift index 66a7306..641f544 100644 --- a/Tests/XSDSchemaTests.swift +++ b/Tests/XSDSchemaTests.swift @@ -73,6 +73,23 @@ struct XSDSchemaTests { #expect(try !validate(xsd, "Tnope").isEmpty) } + @Test("A huge finite maxOccurs is preserved beyond Int range") + func test_hugeFiniteMaxOccurs() throws { + let xsd = """ + + + + + + + + + + """ + #expect(try validate(xsd, "").isEmpty) + #expect(try !validate(xsd, "").isEmpty) + } + @Test("A named complex type referenced by elements") func test_namedComplexType() throws { let xsd = """ diff --git a/docs/design/counted-content-automaton.md b/docs/design/counted-content-automaton.md index c3176da..7c95b1b 100644 --- a/docs/design/counted-content-automaton.md +++ b/docs/design/counted-content-automaton.md @@ -48,8 +48,9 @@ The relevant literature line is: - Kilpeläinen 2011: the principled XML Schema determinism tool is a counting automaton, not occurrence unrolling. -These are witnesses for the algorithm family. PureXML's implementation still has -to be derived against its own `Particle` model and verified locally. +These are witnesses for the algorithm family. PureXML's instance-time matcher now +implements the counted shape against its own `Particle` model; counted UPA remains +separate. ## Correctness claims @@ -57,9 +58,9 @@ Every claim below is labeled per proof discipline. | Claim | Status | Meaning | |---|---|---| -| Counted matcher accepts exactly the same child-name language as the XSD particle tree for `element`, `wildcard`, `sequence`, and `choice` with `minOccurs`/`maxOccurs`. | theorem target | Proved by structural induction over the particle tree once the implementation replaces `ContentNFA`. | -| Counted matcher keeps per-child matched-particle attribution. | theorem target under UPA | For UPA-valid content models, each consumed element is matched to exactly one particle; that particle's type/value metadata is the one used by child validation. | -| Counted matcher build size is independent of numeric occurrence magnitudes. | theorem target | Program states and counters are proportional to particle nodes, not to `minOccurs` or `maxOccurs` values. | +| Counted matcher accepts exactly the same child-name language as the XSD particle tree for `element`, `wildcard`, `sequence`, and `choice` with `minOccurs`/`maxOccurs`. | theorem target, implemented | `ContentMatcher.swift` now uses counted configurations; the remaining work is writing the full structural proof against nullable nested particles. | +| Counted matcher keeps per-child matched-particle attribution. | theorem target under UPA, implemented | For UPA-valid content models, each consumed element is matched to exactly one particle; that particle's type/value metadata is the one used by child validation. | +| Counted matcher build size is independent of numeric occurrence magnitudes. | theorem, implemented | `ContentNFABuilder` allocates one counter scope per particle occurrence and a constant number of states/edges per particle; high-bound tests assert state count stays structural. | | Counted matcher runtime is bounded for interactive validation. | theorem target | For `N` child elements, `S` program states, and `D` counter scopes, validation is bounded by `O(N * S * D)` time and `O(S * D)` live matcher memory. The constants are schema-structural, not occurrence-magnitude structural. | | `all` groups remain exact. | theorem already implemented, out of scope for NFA replacement | `all` is validated by direct member counts in `ComplexValidator.matchesAll` and `allStructureErrors`; the counted automaton does not need to encode it initially. | | Counted UPA/determinism replaces `positionCap`. | blocked design frontier | The instance matcher can be replaced first. Deleting `positionCap` requires a separate counted determinism pass or a proof that the existing clamp is exact for XSD's admitted content models. | @@ -296,23 +297,24 @@ current caps. ## Migration plan -1. Add `NonNegativeDecimal`, `OccurrenceUpper`, and `OccurrenceRange` while +1. Done 2026-06-19: add `NonNegativeDecimal`, `OccurrenceUpper`, and `OccurrenceRange` while preserving the existing `Particle` API for call sites that only need small integer comparisons. -2. Add `CountedContentProgram` and a builder beside `ContentNFABuilder`. -3. Add differential tests comparing old and counted matchers on ordinary models +2. Done 2026-06-19: replace `ContentNFABuilder`'s unrolled body copies with a + counted program behind the existing `ContentNFA` surface. +3. Keep differential tests comparing DTD and XSD matchers on ordinary models below the current caps. -4. Add high-bound tests that are exact and currently impossible without language +4. Done 2026-06-19: add high-bound tests that are exact and previously impossible without language widening: - `a{2,2}` rejects one `a` and three `a`s; - `a{2,100000000000000000000}` accepts two `a`s and rejects one; - `a{0,2}` rejects three `a`s; - `(a{2,3}){2,2}` accepts four to six `a`s and rejects three/seven; - `sequence(a{1000000000000}, b)` never offers `b` before enough `a`s. -5. Switch `ComplexValidatorContentModel`, `XSDCompletions`, and +5. Done 2026-06-19: switch `ComplexValidatorContentModel`, `XSDCompletions`, and `matchedParticles` to the counted program. -6. Delete `occursUnrollCap` and `totalStateCap`. -7. Replace `ContentNFAStateBudgetTests` with exact high-bound tests. The old +6. Done 2026-06-19: delete `occursUnrollCap` and `totalStateCap`. +7. Done 2026-06-19: replace `ContentNFAStateBudgetTests` with exact high-bound tests. The old state-ceiling assertion should disappear because there is no state ceiling. 8. Design and land the counted UPA pass; then delete `positionCap`. @@ -331,10 +333,10 @@ current caps. This arc is complete only when all are true: -- `ContentMatcher.swift` contains no `occursUnrollCap`, `totalStateCap`, or +- Done 2026-06-19: `ContentMatcher.swift` contains no `occursUnrollCap`, `totalStateCap`, or cap-triggered "degrade to star" path. - `ContentModelDeterminism.swift` contains no `positionCap` skip. -- Tests include exact high-bound acceptance/rejection cases whose numeric bounds +- Done 2026-06-19: tests include exact high-bound acceptance/rejection cases whose numeric bounds exceed any practical unroll count. - The proof labels in this document are upgraded from "theorem target" to "theorem" with code references. diff --git a/docs/plans/2026-06-18-false-negative-tail.md b/docs/plans/2026-06-18-false-negative-tail.md index 2ad3b6e..8b9e237 100644 --- a/docs/plans/2026-06-18-false-negative-tail.md +++ b/docs/plans/2026-06-18-false-negative-tail.md @@ -278,7 +278,7 @@ XSTS_ROOT=/private/tmp/xsts/xmlschema2006-11-06 swift test -c release --filter X **Commit:** `fix(schema): close identity constraint instance tail` -### T9. Design the counting automaton separately +### T9. Land the counted instance matcher, then counted UPA **Files:** new `docs/design/` or `docs/plans/` design artifact, `Sources/Schema/ContentMatcher.swift`, @@ -293,6 +293,9 @@ state a worst-case time and memory bound; prove high numeric occurrence bounds do not create proportional states. The design artifact is `docs/design/counted-content-automaton.md`; implementation must follow that proof plan rather than adding another cap. +Status 2026-06-19: the `ContentMatcher` instance-time automaton now uses counted +configurations and exact decimal occurrence bounds. `ContentModelDeterminism` still +has `positionCap`, so this arc is not complete. **Verifies:** @@ -302,7 +305,8 @@ swift test --filter ContentModelDeterminism swift test --filter ContentNFAStateBudgetTests ``` -**Commit:** `docs(schema): design counted content automaton` +**Commits:** `docs(schema): design counted content automaton`, +`fix(schema): count content-model occurrences` ## Constraints diff --git a/docs/release-hurdles.md b/docs/release-hurdles.md index 525cef4..b91c75a 100644 --- a/docs/release-hurdles.md +++ b/docs/release-hurdles.md @@ -82,7 +82,7 @@ critic for latent FPs the corpus misses), not a pattern sweep. A few (contested spec: schM4 reorder, attQ per-document vs per-composition id-scope) need adjudication, not code. -## Part 2: the caps are band-aids because matching unrolls occurrence values, not text +## Part 2: the remaining cap is UPA determinism, not instance matching Two engines, two postures. @@ -93,35 +93,30 @@ pathological nested group references; when hit it **silently skips the UPA check (`return nil`, ContentModelDeterminism.swift:85): a hidden false negative and a stopper-3 silent gap, but rare. -**Instance-time matcher (`ContentMatcher`)**, the one that runs on keystrokes, is -the real problem. It builds the NFA by **unrolling** occurrences into states -(`for _ in 0..