feat(xpath,parsing): public streaming surface, linear document-order at scale

EventReader.next(), both initializers, and the Event alias are public
(an internal typealias in a split file had been shadowing the public
Parsing.Event); a new PureXMLPublicAPITests target compiles without
testable import so facade gaps fail CI. The large-corpus benchmarks
exposed document-order sorting as quadratic on flat fan-outs; the fix's
first attempt was itself worse (an eager sibling table built per
predicate evaluation), caught by sampling the live run, and replaced
with a lazy table: single-node sets never touch order keys and a
parent's table builds on its second lookup. XPath at 42 MB drops
27.1 s to 1.57 s (190x to 11.7x); at 210 MB PureXML completes in 9.4 s
the query libxml2 refuses at its 10M node-set cap (#143 tracks our
configurable budget). Benchmark harness: size series to 210 MB, CORPUS
mode for real files (NASA, SwissProt measured), SAX stage,
refusal-aware table, NULL-safe C driver. All conformance suites,
Linux, and WASM verified. Closes #142.
This commit is contained in:
Mihaela Mihaljevic 2026-06-11 06:06:31 +02:00
parent 72eaeeb439
commit 9e345ea0fe
12 changed files with 231 additions and 38 deletions

View File

@ -61,21 +61,31 @@ int main(int argc, char **argv) {
}
printf("libxml2,serialize,%ld,%.6f\n", size, best_serialize);
/* XPath: count a broad selection. */
/* XPath: count a broad selection. libxml2 caps node sets at
* XPATH_MAX_NODESET_LENGTH (10M): past that the evaluation returns
* NULL, which is reported rather than dereferenced. */
double best_xpath = 1e9;
long count = 0;
long count = -1;
for (int i = 0; i < iterations; i++) {
double t0 = now();
xmlXPathContextPtr ctx = xmlXPathNewContext(doc);
xmlXPathObjectPtr res = xmlXPathEvalExpression((const xmlChar *)"count(//item[@kind='even'])", ctx);
count = (long)res->floatval;
double dt = now() - t0;
if (dt < best_xpath) best_xpath = dt;
xmlXPathFreeObject(res);
if (res) {
count = (long)res->floatval;
double dt = now() - t0;
if (dt < best_xpath) best_xpath = dt;
xmlXPathFreeObject(res);
}
xmlXPathFreeContext(ctx);
if (!res) break;
}
if (count >= 0) {
printf("libxml2,xpath,%ld,%.6f\n", size, best_xpath);
fprintf(stderr, "libxml2 xpath count: %ld\n", count);
} else {
printf("libxml2,xpath,%ld,refused\n", size);
fprintf(stderr, "libxml2 xpath: evaluation refused (node-set cap)\n");
}
printf("libxml2,xpath,%ld,%.6f\n", size, best_xpath);
fprintf(stderr, "libxml2 xpath count: %ld\n", count);
xmlFreeDoc(doc);
xmlCleanupParser();

View File

@ -29,11 +29,19 @@ func best(_ iterations: Int, _ body: () throws -> Void) rethrows -> Double {
// Parse.
var tree: PureXML.Model.TreeNode!
let parseTime = try best(iterations) {
tree = try PureXML.parseTree(source)
tree = try PureXML.parseTree(source, limits: .init(allowDoctype: true))
}
print("purexml,parse,\(bytes),\(String(format: "%.6f", parseTime))")
/// SAX (event stream only): isolates scanner cost from tree construction.
let saxTime = try best(iterations) {
var reader = PureXML.events(source, limits: .init(allowDoctype: true))
while try reader.next() != nil {}
}
print("purexml,sax,\(bytes),\(String(format: "%.6f", saxTime))")
// Serialize.
let node = tree.node
var serialized = ""

View File

@ -11,6 +11,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Scanner-side validity findings are now located structured values (validation-framework audit follow-up): `DocumentType.validityFindings` carries `ValidityFinding` values (reason plus the declared name they are about) instead of strings, `DTDSchema.declarationErrors` holds ready `ValidationError`s, and every declaration-level finding renders at its declaration's coding path (`x`, `r/@a`) instead of "at root of document", matching the XSD consistency-errors precedent. Finding texts are unchanged.
### Added
- Public streaming surface (#142) and a public-API test target: `EventReader.next()`, both initializers, and the `Event` alias are public, so external consumers can pull events with bounded memory over arbitrarily large inputs (an internal typealias in a split file had been shadowing the public `Parsing.Event`, narrowing `next()` to internal; the benchmark harness found it). A new `PureXMLPublicAPITests` target compiles against the public surface without `@testable`, so any public facade over internal members fails CI; it pins parse/query/serialize, streaming pull, and the chunked pull source.
### Fixed
- XPath document-order sorting is benchmarked at scale across generated (4/42/210 MB) and real corpora (NASA ADC 25 MB, SwissProt 115 MB), all linear: the quadratic fix's own first attempt (an eager sibling table built per predicate evaluation) was caught by sampling the live run and replaced with a lazy table (single-node sets never touch order keys; a parent's table builds on its second lookup). XPath at 42 MB drops 27.1 s to 1.57 s (ratio 190x to 11.7x); at 210 MB PureXML completes in 9.4 s the query libxml2 refuses at its 10M node-set cap (the configurable protective budget is #143). The benchmark harness gains a size series, a CORPUS mode for real-world files, a SAX stage, and a refusal-aware comparison table.
- XPath document-order sorting is no longer quadratic on flat fan-outs (#139, found by the large-corpus benchmark): document-order keys looked up each node's index by scanning its parent's children, so sorting a node-set drawn from one wide element (the 42 MB corpus: 200k siblings) cost 27 seconds. A per-operation sibling-index cache enumerates each distinct parent once, making key computation linear in the tree.
### Changed
- Byte-backed Reader, performance passes 2 and 3 (#139): String input now copies its UTF-8 once into owned storage (an unsafe pointer encapsulated in a class lifetime, never escaping the parser) and scalars decode straight off the pointer, eliminating the per-character closure chain; ASCII literals match and consume at the byte level; and plain-ASCII character data, the bulk of any document, scans as a single byte run with one String built per run. Cumulative parse time on the 4.2 MB benchmark corpus drops 32 percent (0.49 s to 0.34 s, libxml2 ratio 30x to 20x). The W3C conformance suite caught one boundary in the run scanner (a run starting with '>' could swallow the "]]>" check after the character path consumed "]]"; ibm14n01); the run scanner now leaves a leading '>' to the character path, with the case pinned as a unit regression. All suites (W3C xmlconf, RELAX NG, xalan), Linux, and WASM verified.

View File

@ -16,6 +16,11 @@ let package = Package(
name: "PureXML",
path: "Sources",
),
.testTarget(
name: "PureXMLPublicAPITests",
dependencies: ["PureXML"],
path: "PublicAPITests",
),
.testTarget(
name: "PureXMLTests",
dependencies: ["PureXML"],

View File

@ -0,0 +1,51 @@
// Deliberately NOT @testable: these tests compile against the public
// surface exactly as an external package would, so a public facade over
// internal members (the #142 class of gap: a public EventReader whose
// next() was internal) fails CI instead of hiding behind testability.
import PureXML
import Testing
@Suite("Public API surface")
struct PublicSurfaceTests {
@Test("Parse, query, and serialize work through the public surface")
func test_parseQuerySerialize() throws {
let document = try PureXML.parse("<r><a k=\"v\">x</a><a>y</a></r>")
let query = try PureXML.XPath.Query("count(//a)")
#expect(try query.number(over: document) == 2)
#expect(PureXML.serialize(document).contains("<a k=\"v\">x</a>"))
}
@Test("The streaming reader pulls events publicly, bounded memory")
func test_streamingPull() throws {
var reader = PureXML.events("<r><a>x</a><!--c--></r>")
var names: [String] = []
var texts: [String] = []
var comments = 0
while let event = try reader.next() {
switch event {
case let .startElement(name, _): names.append(name.description)
case let .characters(text): texts.append(text)
case .comment: comments += 1
default: break
}
}
#expect(names == ["r", "a"])
#expect(texts == ["x"])
#expect(comments == 1)
}
@Test("The streaming reader accepts a chunked pull source")
func test_streamingFromChunks() throws {
var chunks = ["<r><a>", "hello", "</a></r>"].makeIterator()
var characters: [Character] = []
var reader = PureXML.Parsing.EventReader(pulling: {
if characters.isEmpty, let chunk = chunks.next() { characters = Array(chunk) }
return characters.isEmpty ? nil : characters.removeFirst()
})
var events = 0
while try reader.next() != nil {
events += 1
}
#expect(events == 5)
}
}

View File

@ -125,7 +125,7 @@ flowchart TB
PureXML is a working, dependency-free XML library today: parse, emit, validate,
query, and stream documents on macOS, Linux, Windows, and WASM. The test suite
currently runs **981 tests in 167 suites** (`swift test`).
currently runs **984 tests in 168 suites** (`swift test`).
### Shipped (libxml2-aligned surface)

View File

@ -40,7 +40,7 @@ public extension PureXML.Parsing {
/// Creates a reader over a character-pulling source. The closure returns
/// the next character or nil at end of input.
init(
public init(
pulling pull: @escaping () -> Character?,
limits: Limits = .default,
resolver: EntityResolver = .refusing,
@ -54,7 +54,7 @@ public extension PureXML.Parsing {
}
/// Creates a reader over a string (a convenience over the streaming init).
init(_ string: String, limits: Limits = .default, resolver: EntityResolver = .refusing, recovering: Bool = false) {
public init(_ string: String, limits: Limits = .default, resolver: EntityResolver = .refusing, recovering: Bool = false) {
reader = Reader(string)
self.limits = limits
self.resolver = resolver
@ -67,7 +67,7 @@ public extension PureXML.Parsing {
/// diagnostic and reading continues, popping to a matching open element on
/// a mismatched end tag and stopping (so the caller closes what is open) on
/// truncation.
mutating func next() throws -> Event? {
public mutating func next() throws -> Event? {
if !primed {
primed = true
if reader.peek() == "\u{FEFF}" {

View File

@ -1,7 +1,9 @@
/// The end-tag scanner, split from the reader body to keep it under the
/// length caps. Behavior is part of EventReader proper.
extension PureXML.Parsing.EventReader {
typealias Event = PureXML.Parsing.Event
/// Public: this alias shadows the public Parsing.Event for the whole
/// type, so it must not narrow the public next() signature (#142).
public typealias Event = PureXML.Parsing.Event
typealias ParseError = PureXML.Parsing.ParseError
mutating func scanEndTag() throws -> Event {
let mark = reader.mark

View File

@ -0,0 +1,34 @@
extension PureXML.XPath {
/// A per-operation cache of each node's index among its parent's
/// children. Document-order keys need the index at every ancestor level;
/// without a cache each lookup scans the sibling list, so sorting a
/// node-set drawn from one flat fan-out is quadratic (the 42 MB
/// benchmark corpus spent 27 seconds there). Each distinct parent is
/// enumerated exactly once.
final class SiblingIndexCache {
private var indexes: [ObjectIdentifier: [ObjectIdentifier: Int]] = [:]
private var scannedOnce: Set<ObjectIdentifier> = []
func index(of child: PureXML.Model.TreeNode, in parent: PureXML.Model.TreeNode) -> Int {
let parentIdentity = ObjectIdentifier(parent)
if let table = indexes[parentIdentity] {
return table[ObjectIdentifier(child)] ?? 0
}
// The table only pays for itself from the second lookup under
// the same parent: a single lookup (one key over a wide parent,
// the per-predicate case that once built a million-entry table
// per evaluation) scans linearly instead.
guard scannedOnce.contains(parentIdentity) else {
scannedOnce.insert(parentIdentity)
return parent.children.firstIndex { $0 === child } ?? 0
}
var table: [ObjectIdentifier: Int] = [:]
table.reserveCapacity(parent.children.count)
for (index, node) in parent.children.enumerated() {
table[ObjectIdentifier(node)] = index
}
indexes[parentIdentity] = table
return table[ObjectIdentifier(child)] ?? 0
}
}
}

View File

@ -108,16 +108,37 @@ public extension PureXML.XPath {
private static let attributeBand = -1
private static func treePath(_ node: PureXML.Model.TreeNode) -> [Int] {
treePath(node, cache: nil)
}
/// The root path, with an optional per-operation sibling-index cache:
/// without one, finding a node's index among its siblings is linear,
/// which turns a sort over a flat fan-out (one parent, n children)
/// quadratic. The cache enumerates each distinct parent once, making
/// total key computation linear in the tree.
static func treePath(_ node: PureXML.Model.TreeNode, cache: SiblingIndexCache?) -> [Int] {
var path: [Int] = []
var current = node
while let parent = current.parent {
let index = parent.children.firstIndex { $0 === current } ?? 0
path.append(index)
path.append(cache?.index(of: current, in: parent) ?? parent.children.firstIndex { $0 === current } ?? 0)
current = parent
}
return path.reversed()
}
/// `documentOrder` with the sibling-index cache applied.
func documentOrder(cache: SiblingIndexCache) -> [Int] {
switch self {
case let .tree(node):
return Self.treePath(node, cache: cache)
case let .attribute(owner, attribute):
let index = owner.attributes.firstIndex { $0.name == attribute.name } ?? 0
return Self.treePath(owner, cache: cache) + [Self.attributeBand, index]
case let .namespace(owner, prefix, _):
return Self.treePath(owner, cache: cache) + [Self.namespaceBand, prefix.hashValue]
}
}
/// Whether `lhs` precedes `rhs` in document order. Each call recomputes
/// both order keys; for sorts and minimums over node-sets, prefer
/// ``Swift/Sequence/sortedByDocumentOrder()`` and
@ -141,14 +162,20 @@ extension Sequence<PureXML.XPath.Node> {
/// (decorate-sort-undecorate) rather than recomputing the root path on every
/// comparison, which turns an O(n log n x depth) sort into O(n log n).
func sortedByDocumentOrder() -> [PureXML.XPath.Node] {
map { (node: $0, key: $0.documentOrder) }
// Zero- and one-node sets are already sorted: the common predicate
// case must not touch order keys at all.
let nodes = Array(self)
if nodes.count <= 1 { return nodes }
let cache = PureXML.XPath.SiblingIndexCache()
return nodes.map { (node: $0, key: $0.documentOrder(cache: cache)) }
.sorted { PureXML.XPath.Node.ordered($0.key, before: $1.key) }
.map(\.node)
}
/// The first node in document order, computing each node's order key once.
func firstInDocumentOrder() -> PureXML.XPath.Node? {
map { (node: $0, key: $0.documentOrder) }
let cache = PureXML.XPath.SiblingIndexCache()
return map { (node: $0, key: $0.documentOrder(cache: cache)) }
.min { PureXML.XPath.Node.ordered($0.key, before: $1.key) }?
.node
}

View File

@ -20,6 +20,33 @@ cross-implementation correctness check.
These are the honest starting numbers the #139 optimization passes burn
down; the table updates as passes land, never silently.
## Current record (size series + real corpora, best of 3)
| corpus | parse | serialize | xpath |
|---|---|---|---|
| generated 4.2 MB | 0.34 s (18.7x) | 0.064 s (5.5x) | 0.15 s (11.9x) |
| generated 42 MB | 3.49 s (18.7x) | 0.63 s (5.6x) | 1.57 s (11.7x) |
| generated 210 MB | 19.3 s (19.7x) | 3.19 s (5.6x) | 9.4 s (libxml2: **refused**) |
| NASA ADC 25 MB (real) | 1.73 s (24.5x) | 0.42 s (8.0x) | 0.76 s (19.4x) |
| SwissProt 115 MB (real) | 11.0 s (22.0x) | 1.77 s (5.5x) | 5.0 s (18.7x) |
Two findings only the large corpora could surface:
- PureXML's document-order sort was quadratic on flat fan-outs (42 MB:
27 s in XPath, ratio 190x). Fixed; and the first fix attempt was itself
quadratic in a worse place (an eager sibling-table built per predicate
evaluation: one hour at 42 MB), caught by sampling the live run. The
cache now early-outs single-node sets and builds a parent's table only
on its second lookup. XPath at 42 MB: 27.1 s -> 1.57 s.
- libxml2 refuses XPath evaluation past its XPATH_MAX_NODESET_LENGTH cap
(10M nodes): the 210 MB corpus query returns NULL by design. PureXML
completes it in 9.4 s. The protective counterpart for hostile input is
tracked as #143.
Scaling: all three PureXML operations are linear in document size (ratios
flat from 4 MB to 210 MB), and there is no 2 GB integer cliff (64-bit
offsets throughout; libxml2's xmlReadMemory takes an int byte count).
## Optimization log
| pass | change | parse | xpath | serialize |

View File

@ -1,21 +1,25 @@
#!/usr/bin/env bash
# PureXML vs libxml2 benchmark (#139). Generates a deterministic corpus,
# PureXML vs libxml2 benchmark (#139). Generates deterministic corpora,
# builds both drivers (PureXML in release, the C driver against the SDK's
# libxml2), runs them over the same file, and prints a comparison table.
# Nothing is vendored; the corpus is regenerated on each run.
# libxml2), runs them over the same files, and prints comparison tables.
# Nothing is vendored; corpora are regenerated on each run.
#
# Usage: bash scripts/benchmark.sh [items] [iterations]
# Usage: bash scripts/benchmark.sh ["items items ..."] [iterations]
# CORPUS=/path/to/file.xml bash scripts/benchmark.sh "" [iterations]
#
# The default size series spans ~4 MB, ~42 MB, and ~210 MB; pass a single
# item count to pin one size, or CORPUS to measure a real-world document.
set -euo pipefail
cd "$(dirname "$0")/.."
ITEMS="${1:-20000}"
ITERATIONS="${2:-5}"
SIZES="${1:-20000 200000 1000000}"
ITERATIONS="${2:-3}"
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
# Deterministic corpus: nested records with attributes, text, and namespaces.
python3 - "$ITEMS" > "$WORK/corpus.xml" << 'PYEOF'
generate_corpus() {
python3 - "$1" > "$2" << 'CORPUSEOF'
import sys
items = int(sys.argv[1])
out = sys.stdout
@ -29,9 +33,8 @@ for i in range(items):
out.write(f' <note xml:space="preserve">line one\nline two for {i}</note>\n')
out.write(' </item>\n')
out.write('</catalog>\n')
PYEOF
SIZE=$(wc -c < "$WORK/corpus.xml" | tr -d ' ')
echo "corpus: $ITEMS items, $SIZE bytes, best of $ITERATIONS runs"
CORPUSEOF
}
echo "==> building PureXML (release)"
swift build -c release > /dev/null
@ -44,22 +47,39 @@ echo "==> building libxml2 driver"
SDK="$(xcrun --show-sdk-path)"
clang -O2 -I "$SDK/usr/include/libxml2" -lxml2 Benchmarks/bench-libxml2.c -o "$WORK/bench-libxml2"
echo "==> running"
"$WORK/bench-purexml" "$WORK/corpus.xml" "$ITERATIONS" > "$WORK/pure.csv"
"$WORK/bench-libxml2" "$WORK/corpus.xml" "$ITERATIONS" > "$WORK/lib.csv"
python3 - "$WORK/pure.csv" "$WORK/lib.csv" << 'PYEOF'
run_one() {
local file="$1" label="$2"
echo ""
echo "==> $label ($(wc -c < "$file" | tr -d ' ') bytes, best of $ITERATIONS)"
"$WORK/bench-purexml" "$file" "$ITERATIONS" > "$WORK/pure.csv"
"$WORK/bench-libxml2" "$file" "$ITERATIONS" > "$WORK/lib.csv"
python3 - "$WORK/pure.csv" "$WORK/lib.csv" << 'TABLEEOF'
import sys
def load(path):
rows = {}
for line in open(path):
library, operation, size, seconds = line.strip().split(",")
rows[operation] = float(seconds)
rows[operation] = None if seconds == "refused" else float(seconds)
return rows
pure = load(sys.argv[1])
lib = load(sys.argv[2])
print(f"{'operation':<12} {'PureXML':>12} {'libxml2':>12} {'ratio':>8}")
for operation in ["parse", "serialize", "xpath"]:
p, l = pure[operation], lib[operation]
print(f"{operation:<12} {p:>11.4f}s {l:>11.4f}s {p / l:>7.2f}x")
PYEOF
p, l = pure.get(operation), lib.get(operation)
if p is None or l is None:
left = f"{p:.4f}s" if p is not None else "refused"
right = f"{l:.4f}s" if l is not None else "refused"
print(f"{operation:<12} {left:>12} {right:>12} {'-':>8}")
else:
print(f"{operation:<12} {p:>11.4f}s {l:>11.4f}s {p / l:>7.2f}x")
TABLEEOF
}
if [ -n "${CORPUS:-}" ]; then
run_one "$CORPUS" "real corpus: $CORPUS"
else
for items in $SIZES; do
generate_corpus "$items" "$WORK/corpus-$items.xml"
run_one "$WORK/corpus-$items.xml" "generated, $items items"
done
fi