From dd0ff5d0861370d46f53203b0fa98a21cd0c87af Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Wed, 10 Jun 2026 05:34:11 +0200 Subject: [PATCH] feat(parsing): fourth W3C-suite strictness pass, declaration grammar Make the DTD declaration grammar strict: requires its whitespace (after the keyword, around %, after the name), a real name (not #DEFAULT), nothing but '>' after the value, NDATA preceded by whitespace and forbidden on parameter entities; entity-value literals are validated (XML Chars only, every & a complete reference); SYSTEM/PUBLIC external identifiers parse strictly (PUBLIC needs both literals for entities and the DOCTYPE, public-only stays legal for notations) with PubidChar validation; bodies are validated against XML 1.0 productions 52-60 via a new DTDAttListGrammar (spacing, attribute types, enumerations, quoted defaults); the reserved xml PI target is rejected inside the subset. Also: strict UTF-8 decoding (no silent replacement of malformed sequences), XML Char checks in attribute values and CDATA, the declaration's version/encoding value grammar, and required whitespace between pseudo-attributes. The strict machinery lives in DoctypeScannerStrict/DTDAttListGrammar to stay under the length caps. W3C xmltest: not-wf/sa 162/186 rejected (from 129; 80 at first contact), valid/sa 118/119, zero false positives across all 846 tests at every step. The remaining 24-case baseline is exactly: the entity replacement-text class (19) and the deliberate internal-subset extensions this package supports as features (conditional sections, PEs in the internal subset: 5). Part of #120, #105 --- CHANGELOG.md | 2 +- Sources/Parsing/ByteDecoder.swift | 22 ++- Sources/Parsing/DTDAttListGrammar.swift | 157 ++++++++++++++++++ Sources/Parsing/Diagnostic.swift | 4 +- Sources/Parsing/DoctypeScanner.swift | 76 ++++----- Sources/Parsing/DoctypeScannerStrict.swift | 163 +++++++++++++++++++ Sources/Parsing/EventReaderAttributes.swift | 3 + Sources/Parsing/EventReaderRecovery.swift | 3 + Sources/Parsing/ParseError.swift | 12 ++ Sources/Parsing/PseudoAttributeScanner.swift | 3 + Sources/Parsing/XMLDeclaration.swift | 23 ++- Tests/W3CXMLSuiteTests.swift | 25 ++- 12 files changed, 433 insertions(+), 60 deletions(-) create mode 100644 Sources/Parsing/DTDAttListGrammar.swift create mode 100644 Sources/Parsing/DoctypeScannerStrict.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index a45989c..a215e8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Parser strictness, first W3C-suite pass (#105 Tier 2, #120): the strict parser now rejects `--` inside comments (including the `--->` ending), a raw `<` in attribute values, the literal `]]>` sequence in character data, attributes not separated by whitespace, and characters outside the XML `Char` production in content, five new located `ParseError` cases. Driven by a new opt-in runner for the W3C XML Conformance Test Suite's xmltest section (`XMLTS_ROOT`-gated; the suite is never vendored because its license permits redistribution only as the unmodified archive): valid/sa 118/119, not-wf/sa rejection improved from 80 to 129 of 186 across the first three strictness passes (the second adds character-reference validation, lowercase-x-only hex references and the referenced code point must be a valid XML Char; the reserved 'xml' PI target, legal only as the declaration at the document's very first bytes; and XML Char validation in comments and PI data; the third validates content models against the XML 1.0 grammar at DTD scan time, rejecting unbalanced parentheses, #PCDATA misplacement and nesting, mixed connectors, whitespace-detached or doubled quantifiers, empty groups, and SGML leftovers), with the exact remaining baseline carried in the test and the two remaining leniency classes (internal-subset DTD grammar, entity replacement-text well-formedness) tracked as #120. +- Parser strictness, first W3C-suite pass (#105 Tier 2, #120): the strict parser now rejects `--` inside comments (including the `--->` ending), a raw `<` in attribute values, the literal `]]>` sequence in character data, attributes not separated by whitespace, and characters outside the XML `Char` production in content, five new located `ParseError` cases. Driven by a new opt-in runner for the W3C XML Conformance Test Suite's xmltest section (`XMLTS_ROOT`-gated; the suite is never vendored because its license permits redistribution only as the unmodified archive): valid/sa 118/119, not-wf/sa rejection improved from 80 to 162 of 186 across four strictness passes (the second adds character-reference validation, lowercase-x-only hex references and the referenced code point must be a valid XML Char; the reserved 'xml' PI target, legal only as the declaration at the document's very first bytes; and XML Char validation in comments and PI data; the third validates content models against the XML 1.0 grammar at DTD scan time, rejecting unbalanced parentheses, #PCDATA misplacement and nesting, mixed connectors, whitespace-detached or doubled quantifiers, empty groups, and SGML leftovers; the fourth makes the DTD declaration grammar strict: whitespace and name rules, junk after a value, NDATA spacing and its general-entities-only rule, entity-value literals checked for XML Chars and complete references, SYSTEM/PUBLIC identifiers parsed strictly with PubidChar validation, validated against productions 52-60, the reserved xml PI target rejected inside the subset, strict UTF-8 decoding, the XML declaration's version/encoding value grammar, and required whitespace between pseudo-attributes), with the exact remaining baseline carried in the test and the two remaining leniency classes (internal-subset DTD grammar, entity replacement-text well-formedness) tracked as #120. - XSD Particle Valid (Restriction) (#117): a complex type derived by restriction is now checked at schema compile to accept a structural subset of its base, covering element name matching, element-vs-wildcard admission, wildcard narrowing, occurrence-range subsumption, and the group recursions (sequence/all order-preserving with emptiable skips, choice mapping, sequence-into-choice, sequence-into-all, RecurseAsIfGroup). A structurally unfaithful restriction (widened occurrence, new element name, reordered sequence, content added to an EMPTY or required-into-empty base, an element outside a base wildcard's namespace) throws `SchemaError.invalidRestriction` with the type, base, and reason. The spec's effective-total-range arithmetic for MapAndSum is approximated by per-particle checks, a documented simplification. - Conformance corpus harness (#105, Tier 2): the start of conformance-suite checking, expressed through the validation framework (the OpenAPIKit idiom). A `PureXML.Validation.ConformanceCase` is a `Validatable` named expectation (actual vs spec-authoritative expected output), and `Conformance.matchesExpected` is a `Validation` rule that emits one located `ValidationError` per diverging case, so a whole suite reports every failure at once with the case name as its coding path. Seeded with a Canonical XML 1.0 corpus (attribute ordering, empty-element expansion, comment removal, CDATA-to-escaped-text, processing instructions, namespace rendering) an XPath 1.0 core-function corpus, and an XSD datatype/facet corpus (length, range, pattern, enumeration, lexical space), a RELAX NG pattern corpus (text, optional, repetition, choice, ordered group, attribute, interleave, empty), an XSLT 1.0 transformation corpus (value-of, for-each, if, choose, sort, variable, format-number, apply-templates, call-template), an HTML5 tree-construction corpus (void elements, implied tag closing, case normalization, the adoption agency, entities), a Schematron corpus (assert, report, rule context), a C14N exclusive/2.0 corpus (unused-namespace dropping, render-at-use, whitespace trimming), an XPath axes/predicates corpus (descendant, ancestor, sibling axes, positional and attribute predicates, union), a DTD content-model corpus (sequence, choice, occurrence indicators, mixed, EMPTY, ANY), an XInclude/XPointer corpus (substitution, text inclusion with escaping, fallback, relative href resolution; shorthand ids, element() navigation, xpointer() including the per-step predicate semantic), an XSD regex corpus (implicit whole-string anchoring, the character-class escapes, Unicode categories, quantifier ranges, alternation, class negation and subtraction), and an XML Catalog corpus (system/public/uri entries, rewrite prefixes, unmatched identifiers). The HTML corpus surfaced its first divergence (the fragment parser does not reconstruct active formatting elements like the document parser, tracked as #109). Growing the corpora against the official W3C/OASIS suites is the ongoing Tier-2 work. - EXSLT extension functions (#105, Tier 3): the `common`, `math`, and `sets` EXSLT modules, dispatched by resolving a function's prefix to its namespace (so any prefix bound to an EXSLT namespace works). `math:min`, `math:max`, `math:highest`, `math:lowest`, `math:abs`, `math:sqrt` (with the EXSLT NaN rule for non-numeric nodes); `set:distinct`, `set:difference`, `set:intersection`, `set:has-same-node`, `set:leading`, `set:trailing` (by node identity and document order); `exsl:object-type`. `math:power` and the other transcendentals are omitted because they need a C math library, which the pure-Swift, Foundation-free target excludes; `exsl:node-set` awaits result-tree-fragment support. diff --git a/Sources/Parsing/ByteDecoder.swift b/Sources/Parsing/ByteDecoder.swift index e841f1e..2ef40cf 100644 --- a/Sources/Parsing/ByteDecoder.swift +++ b/Sources/Parsing/ByteDecoder.swift @@ -24,7 +24,27 @@ extension PureXML.Parsing { case .utf16LittleEndian: return try decodeUTF16(body, bigEndian: false) case .utf32BigEndian: return try decodeUTF32(body, bigEndian: true) case .utf32LittleEndian: return try decodeUTF32(body, bigEndian: false) - default: return String(decoding: body, as: UTF8.self) + default: + // Strict UTF-8: malformed sequences (overlongs, surrogate halves, + // truncated tails) are not well-formed input, not replaceable. + guard let decoded = validatedUTF8(body) else { + throw PureXML.Parsing.ParseError.malformedEncoding + } + return decoded + } + } + + /// Decodes UTF-8 strictly, returning nil on any malformed sequence. + private static func validatedUTF8(_ bytes: ArraySlice) -> String? { + var decoder = Unicode.UTF8() + var iterator = bytes.makeIterator() + var scalars = String.UnicodeScalarView() + while true { + switch decoder.decode(&iterator) { + case let .scalarValue(scalar): scalars.append(scalar) + case .emptyInput: return String(scalars) + case .error: return nil + } } } diff --git a/Sources/Parsing/DTDAttListGrammar.swift b/Sources/Parsing/DTDAttListGrammar.swift new file mode 100644 index 0000000..2823c5b --- /dev/null +++ b/Sources/Parsing/DTDAttListGrammar.swift @@ -0,0 +1,157 @@ +extension PureXML.Parsing { + /// Validates an `` declaration body (everything after the element + /// name) against the XML 1.0 grammar (productions 52-60): each definition is + /// `S Name S AttType S DefaultDecl`, where the type is `CDATA`, a tokenized + /// type, `NOTATION` with a name group, or an enumeration, and the default is + /// `#REQUIRED`, `#IMPLIED`, or an optionally-`#FIXED` quoted value. The + /// lenient `AttributeListParser` still drives behavior; this strict check + /// runs at DTD scan time. + enum DTDAttListGrammar { + private static let tokenizedTypes: Set = [ + "CDATA", "ID", "IDREF", "IDREFS", "ENTITY", "ENTITIES", "NMTOKEN", "NMTOKENS", + ] + + static func isValid(_ body: String) -> Bool { + var scanner = AttListScanner(body) + while !scanner.isAtEnd { + // Each definition starts with required whitespace (the body begins + // right after the element name). + guard scanner.requiredSpace() else { return false } + if scanner.isAtEnd { return true } // trailing whitespace + guard scanner.name() else { return false } + guard scanner.requiredSpace() else { return false } + guard attType(&scanner) else { return false } + guard scanner.requiredSpace() else { return false } + guard defaultDecl(&scanner) else { return false } + } + return true + } + + private static func attType(_ scanner: inout AttListScanner) -> Bool { + if scanner.peek() == "(" { + return scanner.nameGroup(nmtokens: true) + } + guard let keyword = scanner.keyword() else { return false } + if tokenizedTypes.contains(keyword) { return true } + if keyword == "NOTATION" { + guard scanner.requiredSpace() else { return false } + return scanner.nameGroup(nmtokens: false) + } + return false + } + + private static func defaultDecl(_ scanner: inout AttListScanner) -> Bool { + if scanner.consumeLiteral("#REQUIRED") || scanner.consumeLiteral("#IMPLIED") { + return true + } + if scanner.consumeLiteral("#FIXED") { + guard scanner.requiredSpace() else { return false } + } + return scanner.quotedValue() + } + } +} + +/// A tiny scanner over one attribute-list body. +private struct AttListScanner { + private let characters: [Character] + private var index = 0 + + init(_ text: String) { + characters = Array(text) + } + + var isAtEnd: Bool { + index >= characters.count + } + + func peek() -> Character? { + index < characters.count ? characters[index] : nil + } + + /// Consumes at least one whitespace character. + mutating func requiredSpace() -> Bool { + let start = index + while let character = peek(), character.isWhitespace { + index += 1 + } + return index > start + } + + mutating func name() -> Bool { + guard let first = peek(), first.unicodeScalars.allSatisfy(PureXML.Parsing.XMLCharacter.isNameStart) else { + return false + } + index += 1 + while let next = peek(), next.unicodeScalars.allSatisfy(PureXML.Parsing.XMLCharacter.isNameChar) { + index += 1 + } + return true + } + + /// An NMTOKEN: one or more name characters (the start may be any name char). + private mutating func nmtoken() -> Bool { + let start = index + while let next = peek(), next.unicodeScalars.allSatisfy(PureXML.Parsing.XMLCharacter.isNameChar) { + index += 1 + } + return index > start + } + + /// An uppercase keyword run (`CDATA`, `NOTATION`, ...). + mutating func keyword() -> String? { + let start = index + while let next = peek(), next.isLetter { + index += 1 + } + guard index > start else { return nil } + return String(characters[start ..< index]) + } + + /// `'(' S? token (S? '|' S? token)* S? ')'` with names or nmtokens. + mutating func nameGroup(nmtokens: Bool) -> Bool { + guard consume("(") else { return false } + skipSpace() + guard nmtokens ? nmtoken() : name() else { return false } + skipSpace() + while !consume(")") { + guard consume("|") else { return false } + skipSpace() + guard nmtokens ? nmtoken() : name() else { return false } + skipSpace() + } + return true + } + + mutating func quotedValue() -> Bool { + guard let quote = peek(), quote == "\"" || quote == "'" else { return false } + index += 1 + while let next = peek(), next != quote { + index += 1 + } + return consume(quote) + } + + mutating func consumeLiteral(_ literal: String) -> Bool { + let count = literal.count + guard index + count <= characters.count, + String(characters[index ..< index + count]) == literal + else { return false } + index += count + return true + } + + private mutating func consume(_ character: Character) -> Bool { + if peek() == character { + index += 1 + return true + } + return false + } + + private mutating func skipSpace() { + while let character = peek(), character.isWhitespace { + index += 1 + } + } +} diff --git a/Sources/Parsing/Diagnostic.swift b/Sources/Parsing/Diagnostic.swift index 2d8d785..633d4ec 100644 --- a/Sources/Parsing/Diagnostic.swift +++ b/Sources/Parsing/Diagnostic.swift @@ -40,7 +40,9 @@ public extension PureXML.Parsing.ParseError { let .junkAfterDocumentElement(mark), let .unsupportedDoctype(mark), let .malformedDeclaration(mark), let .doubleHyphenInComment(mark), let .rawLessThanInAttribute(mark), let .cdataCloseInContent(mark), let .missingSpaceBeforeAttribute(mark), - let .invalidCharacter(mark), let .reservedProcessingInstructionTarget(mark): + let .invalidCharacter(mark), let .reservedProcessingInstructionTarget(mark), + let .invalidEntityDeclaration(mark), let .invalidAttributeListDeclaration(mark), + let .invalidPublicIdentifier(mark): mark case let .unexpectedCharacter(_, mark), let .mismatchedEndTag(_, _, mark), let .duplicateAttribute(_, mark), let .invalidReference(_, mark), diff --git a/Sources/Parsing/DoctypeScanner.swift b/Sources/Parsing/DoctypeScanner.swift index 603c4d8..ef1fe76 100644 --- a/Sources/Parsing/DoctypeScanner.swift +++ b/Sources/Parsing/DoctypeScanner.swift @@ -26,9 +26,10 @@ extension PureXML.Parsing { } /// The mutable state of one DTD scan: the document type being built and the -/// remaining parameter-entity expansion budget. File-scope and private: an -/// internal detail of ``PureXML/Parsing/DoctypeScanner``. -private struct DTDScanner { +/// remaining parameter-entity expansion budget. An internal detail of +/// ``PureXML/Parsing/DoctypeScanner``; the strict declaration grammar lives in +/// DoctypeScannerStrict.swift. +struct DTDScanner { typealias Reader = PureXML.Parsing.Reader typealias Mark = PureXML.Parsing.Mark typealias ParseError = PureXML.Parsing.ParseError @@ -53,7 +54,7 @@ private struct DTDScanner { reader.skipSpace() _ = scanName(&reader) reader.skipSpace() - doctype.externalSubset = parseExternalID(&reader) + doctype.externalSubset = try parseStrictExternalID(&reader, requireSystem: true, at: mark) reader.skipSpace() if reader.consume("[") { try scanDeclarations(&reader, depth: 0, terminatedByBracket: true, at: mark) @@ -161,16 +162,23 @@ private struct DTDScanner { private mutating func scanMarkupDeclaration(_ reader: inout Reader) throws { if reader.matches("") } else if reader.matches("") } else if reader.matches("") @@ -197,34 +205,12 @@ private struct DTDScanner { try scanDeclarations(&sub, depth: depth + 1, terminatedByBracket: false, at: mark) } - private mutating func scanEntityDeclaration(_ reader: inout Reader) { - reader.consume("") - storeEntity(name: name, value: expandParameterReferences(value), isParameter: isParameter) - return - } - let id = parseExternalID(&reader) - reader.skipSpace() - let notation = reader.consume("NDATA") ? notationName(&reader) : nil - skip(&reader, until: ">") - guard let id, !name.isEmpty else { return } - storeExternalEntity(name: name, id: id, isParameter: isParameter, notation: notation) - } - /// Files an external entity declaration by kind: an `NDATA` entity is unparsed /// (recorded with its notation); an external general entity records its /// identifier for the resolver; an external parameter entity is loaded through /// the resolver so its replacement text is available for `%name;` expansion /// (the default refusing resolver loads nothing, keeping XXE closed). - private mutating func storeExternalEntity(name: String, id: ExternalID, isParameter: Bool, notation: String?) { + mutating func storeExternalEntity(name: String, id: ExternalID, isParameter: Bool, notation: String?) { if let notation, !notation.isEmpty, !isParameter { if doctype.unparsedEntities[name] == nil { doctype.unparsedEntities[name] = PureXML.Parsing.UnparsedEntity(id: id, notation: notation) @@ -238,24 +224,25 @@ private struct DTDScanner { } } - private mutating func notationName(_ reader: inout Reader) -> String { + mutating func notationName(_ reader: inout Reader) -> String { reader.skipSpace() return scanName(&reader) } - private mutating func scanNotationDeclaration(_ reader: inout Reader) { + private mutating func scanNotationDeclaration(_ reader: inout Reader) throws { + let mark = reader.mark reader.consume("") if !name.isEmpty, let id, doctype.notations[name] == nil { doctype.notations[name] = id } } - private mutating func storeEntity(name: String, value: String, isParameter: Bool) { + mutating func storeEntity(name: String, value: String, isParameter: Bool) { guard !name.isEmpty else { return } if isParameter { if doctype.parameterEntities[name] == nil { @@ -286,12 +273,21 @@ private struct DTDScanner { doctype.elementModels[name] = model } - private mutating func scanAttributeListDeclaration(_ reader: inout Reader) { + private mutating func scanAttributeListDeclaration(_ reader: inout Reader) throws { + let mark = reader.mark reader.consume(" String { + func expandParameterReferences(_ raw: String) -> String { guard raw.contains("%") else { return raw } var result = "" var index = raw.startIndex @@ -340,7 +336,7 @@ extension DTDScanner { return nil } - private func scanLiteral(_ reader: inout Reader) -> String? { + func scanLiteral(_ reader: inout Reader) -> String? { guard let quote = reader.peek(), quote == "\"" || quote == "'" else { return nil } reader.advance() var value = "" @@ -362,7 +358,7 @@ extension DTDScanner { return text } - private func scanName(_ reader: inout Reader) -> String { + func scanName(_ reader: inout Reader) -> String { var name = "" while let character = reader.peek(), character.isXMLNameContinuation { name.append(character) @@ -371,7 +367,7 @@ extension DTDScanner { return name } - private func skip(_ reader: inout Reader, until terminator: String) { + func skip(_ reader: inout Reader, until terminator: String) { while reader.peek() != nil, !reader.matches(terminator) { reader.advance() } diff --git a/Sources/Parsing/DoctypeScannerStrict.swift b/Sources/Parsing/DoctypeScannerStrict.swift new file mode 100644 index 0000000..ba26f99 --- /dev/null +++ b/Sources/Parsing/DoctypeScannerStrict.swift @@ -0,0 +1,163 @@ +/// The strict XML 1.0 grammar for `` declarations, external +/// identifiers, and public-identifier literals, split from the scanner body to +/// keep it under the length caps. The scan is one well-formedness gate: any +/// deviation throws a located `ParseError`. +extension DTDScanner { + mutating func scanEntityDeclaration(_ reader: inout Reader) throws { + let mark = reader.mark + reader.consume("'`. + private mutating func scanInternalEntityTail(_ reader: inout Reader, name: String, value: String, isParameter: Bool, at mark: Mark) throws { + try validateEntityLiteral(value, at: mark) + reader.skipSpace() + guard reader.peek() == ">" else { + throw ParseError.invalidEntityDeclaration(mark) + } + skip(&reader, until: ">") + storeEntity(name: name, value: expandParameterReferences(value), isParameter: isParameter) + } + + /// The tail of an external entity: the identifier, an optional `S NDATA Name` + /// (general entities only), then `S? '>'`. + private mutating func scanExternalEntityTail(_ reader: inout Reader, name: String, isParameter: Bool, at mark: Mark) throws { + guard let id = try parseStrictExternalID(&reader, requireSystem: true, at: mark) else { + throw ParseError.invalidEntityDeclaration(mark) + } + var notation: String? + if reader.peek()?.isWhitespace == true { + reader.skipSpace() + if reader.consume("NDATA") { + // NDATA is for unparsed general entities only. + guard !isParameter else { + throw ParseError.invalidEntityDeclaration(mark) + } + notation = notationName(&reader) + } + } else if reader.peek() != ">" { + throw ParseError.invalidEntityDeclaration(mark) + } + reader.skipSpace() + guard reader.peek() == ">" else { + throw ParseError.invalidEntityDeclaration(mark) + } + skip(&reader, until: ">") + storeExternalEntity(name: name, id: id, isParameter: isParameter, notation: notation) + } + + /// Validates an entity value literal: every character must be an XML Char, + /// and every `&` must begin a complete reference (`&name;` or a character + /// reference); a bare ampersand is not well-formed. + func validateEntityLiteral(_ value: String, at mark: Mark) throws { + let characters = Array(value) + var index = 0 + while index < characters.count { + let character = characters[index] + guard character.unicodeScalars.allSatisfy(PureXML.Parsing.XMLCharacter.isChar) else { + throw ParseError.invalidCharacter(mark) + } + if character == "&" { + var probe = index + 1 + if probe < characters.count, characters[probe] == "#" { probe += 1 } + let bodyStart = probe + while probe < characters.count, characters[probe] != ";", !characters[probe].isWhitespace, characters[probe] != "&" { + probe += 1 + } + guard probe > bodyStart, probe < characters.count, characters[probe] == ";" else { + throw ParseError.invalidReference("&", mark) + } + index = probe + } + index += 1 + } + } + + /// Parses an external identifier strictly: `SYSTEM S literal`, or + /// `PUBLIC S pubid-literal S system-literal` (the system literal is required + /// when `requireSystem` is true, as for entities and the DOCTYPE itself). + /// Public identifiers are checked against the `PubidChar` set. + func parseStrictExternalID(_ reader: inout Reader, requireSystem: Bool, at mark: Mark) throws -> ExternalID? { + if reader.consume("SYSTEM") { + guard reader.peek()?.isWhitespace == true else { + throw ParseError.invalidEntityDeclaration(mark) + } + reader.skipSpace() + guard let systemID = scanLiteral(&reader) else { + throw ParseError.invalidEntityDeclaration(mark) + } + return ExternalID(systemID: systemID) + } + if reader.consume("PUBLIC") { + return try parsePublicID(&reader, requireSystem: requireSystem, at: mark) + } + return nil + } + + /// `PUBLIC S PubidLiteral (S SystemLiteral)` with the system literal + /// required for entities and the DOCTYPE, optional for notations. + private func parsePublicID(_ reader: inout Reader, requireSystem: Bool, at mark: Mark) throws -> ExternalID { + guard reader.peek()?.isWhitespace == true else { + throw ParseError.invalidEntityDeclaration(mark) + } + reader.skipSpace() + guard let publicID = scanLiteral(&reader) else { + throw ParseError.invalidEntityDeclaration(mark) + } + try validatePublicID(publicID, at: mark) + if requireSystem { + guard reader.peek()?.isWhitespace == true else { + throw ParseError.invalidEntityDeclaration(mark) + } + reader.skipSpace() + guard let systemID = scanLiteral(&reader) else { + throw ParseError.invalidEntityDeclaration(mark) + } + return ExternalID(publicID: publicID, systemID: systemID) + } + // Notations may carry a public identifier alone. + var systemID = "" + if reader.peek()?.isWhitespace == true { + reader.skipSpace() + if let literal = scanLiteral(&reader) { systemID = literal } + } + return ExternalID(publicID: publicID, systemID: systemID) + } + + /// `PubidChar ::= #x20 | #xD | #xA | [a-zA-Z0-9] | [-'()+,./:=?;!*#@$_%]` + func validatePublicID(_ publicID: String, at mark: Mark) throws { + let allowed = Set("-'()+,./:=?;!*#@$_%") + for character in publicID { + let valid = character == " " || character == "\r" || character == "\n" + || (character.isASCII && (character.isLetter || character.isNumber)) + || allowed.contains(character) + guard valid else { + throw ParseError.invalidPublicIdentifier(mark) + } + } + } +} diff --git a/Sources/Parsing/EventReaderAttributes.swift b/Sources/Parsing/EventReaderAttributes.swift index 72112b1..1ab4446 100644 --- a/Sources/Parsing/EventReaderAttributes.swift +++ b/Sources/Parsing/EventReaderAttributes.swift @@ -48,6 +48,9 @@ extension PureXML.Parsing.EventReader { if character == "<" { throw PureXML.Parsing.ParseError.rawLessThanInAttribute(reader.mark) } + guard character.unicodeScalars.allSatisfy(PureXML.Parsing.XMLCharacter.isChar) else { + throw PureXML.Parsing.ParseError.invalidCharacter(reader.mark) + } raw.append(character) reader.advance() } diff --git a/Sources/Parsing/EventReaderRecovery.swift b/Sources/Parsing/EventReaderRecovery.swift index 7fbadca..1b78abc 100644 --- a/Sources/Parsing/EventReaderRecovery.swift +++ b/Sources/Parsing/EventReaderRecovery.swift @@ -80,6 +80,9 @@ extension PureXML.Parsing.EventReader { } length += 1 try checkContent(length, mark) + guard character.unicodeScalars.allSatisfy(PureXML.Parsing.XMLCharacter.isChar) else { + throw PureXML.Parsing.ParseError.invalidCharacter(reader.mark) + } content.append(character) } reader.consume("]]>") diff --git a/Sources/Parsing/ParseError.swift b/Sources/Parsing/ParseError.swift index 3b112b6..275c1b5 100644 --- a/Sources/Parsing/ParseError.swift +++ b/Sources/Parsing/ParseError.swift @@ -50,6 +50,12 @@ public extension PureXML.Parsing { case reservedProcessingInstructionTarget(Mark) /// An `` content model does not match the XML 1.0 grammar. case invalidContentModel(element: String, Mark) + /// An `` declaration does not match the XML 1.0 grammar. + case invalidEntityDeclaration(Mark) + /// An `` declaration does not match the XML 1.0 grammar. + case invalidAttributeListDeclaration(Mark) + /// A public identifier literal contains a character outside `PubidChar`. + case invalidPublicIdentifier(Mark) public var description: String { switch self { @@ -115,6 +121,12 @@ public extension PureXML.Parsing { "the processing-instruction target 'xml' is reserved at \(mark)" case let .invalidContentModel(element, mark): "the content model of is malformed at \(mark)" + case let .invalidEntityDeclaration(mark): + "malformed declaration at \(mark)" + case let .invalidAttributeListDeclaration(mark): + "malformed declaration at \(mark)" + case let .invalidPublicIdentifier(mark): + "public identifier contains an illegal character at \(mark)" case let .malformedDeclaration(mark): "malformed XML declaration at \(mark)" } diff --git a/Sources/Parsing/PseudoAttributeScanner.swift b/Sources/Parsing/PseudoAttributeScanner.swift index 68c2b15..22c4eb1 100644 --- a/Sources/Parsing/PseudoAttributeScanner.swift +++ b/Sources/Parsing/PseudoAttributeScanner.swift @@ -27,7 +27,10 @@ extension PureXML.Parsing.XMLDeclaration { skipSpace() guard let value = scanQuoted() else { return nil } pairs.append(PseudoAttribute(name: name, value: value)) + // Pseudo-attributes must be separated by whitespace. + let before = index skipSpace() + if index == before, index < chars.count { return nil } } return pairs } diff --git a/Sources/Parsing/XMLDeclaration.swift b/Sources/Parsing/XMLDeclaration.swift index 595dd38..1040ff9 100644 --- a/Sources/Parsing/XMLDeclaration.swift +++ b/Sources/Parsing/XMLDeclaration.swift @@ -26,8 +26,12 @@ public extension PureXML.Parsing { var declaration = XMLDeclaration() for pair in pairs { switch pair.name { - case "version": declaration.version = pair.value - case "encoding": declaration.encoding = pair.value + case "version": + guard isValidVersion(pair.value) else { return nil } + declaration.version = pair.value + case "encoding": + guard isValidEncodingName(pair.value) else { return nil } + declaration.encoding = pair.value case "standalone": guard let flag = standaloneFlag(pair.value) else { return nil } declaration.standalone = flag @@ -49,6 +53,21 @@ public extension PureXML.Parsing { return true } + /// `VersionNum ::= '1.' [0-9]+`, exactly (no surrounding whitespace). + private static func isValidVersion(_ value: String) -> Bool { + guard value.hasPrefix("1.") else { return false } + let digits = value.dropFirst(2) + return !digits.isEmpty && digits.allSatisfy { $0.isASCII && $0.isNumber } + } + + /// `EncName ::= [A-Za-z] ([A-Za-z0-9._] | '-')*`. + private static func isValidEncodingName(_ value: String) -> Bool { + guard let first = value.first, first.isASCII, first.isLetter else { return false } + return value.dropFirst().allSatisfy { character in + character.isASCII && (character.isLetter || character.isNumber || character == "." || character == "_" || character == "-") + } + } + private static func standaloneFlag(_ value: String) -> Bool? { switch value { case "yes": true diff --git a/Tests/W3CXMLSuiteTests.swift b/Tests/W3CXMLSuiteTests.swift index 73ac2f7..c4fff02 100644 --- a/Tests/W3CXMLSuiteTests.swift +++ b/Tests/W3CXMLSuiteTests.swift @@ -16,22 +16,17 @@ struct W3CXMLSuiteTests { } /// Cases this implementation knowingly accepts although the suite marks - /// them not-well-formed. Two classes, tracked for later hardening: the - /// internal-subset scanner is deliberately lenient about DTD grammar - /// details (declaration syntax, conditional sections, parameter-entity - /// placement), and entity replacement text is not yet checked for - /// standalone well-formedness (tags spanning entity boundaries, partial - /// markup in values). The baseline below is exact, so any regression in - /// the cases that DO pass is still caught. + /// them not-well-formed. Two classes remain: deliberate internal-subset + /// extensions this package supports as features (conditional sections and + /// parameter-entity references in the internal subset: 063, 107, 160-162), + /// and entity replacement-text well-formedness (tags spanning entity + /// boundaries, references to external/unparsed entities in attribute + /// defaults, charref re-scan semantics), tracked in #120. The baseline is + /// exact, so any regression in the cases that DO pass is still caught. private let knownNotWFDeviations: Set = [ - "054.xml", "057.xml", "058.xml", "059.xml", "060.xml", "061.xml", "062.xml", "063.xml", - "064.xml", "065.xml", "066.xml", "067.xml", "068.xml", "069.xml", "074.xml", "078.xml", - "079.xml", "080.xml", "082.xml", "084.xml", "085.xml", "086.xml", "087.xml", "089.xml", - "090.xml", "091.xml", "092.xml", "096.xml", "101.xml", "102.xml", "103.xml", "107.xml", - "113.xml", "114.xml", "115.xml", "116.xml", "117.xml", "119.xml", "120.xml", "121.xml", - "140.xml", "141.xml", "149.xml", "153.xml", "158.xml", "160.xml", "161.xml", "162.xml", - "165.xml", "168.xml", "169.xml", "170.xml", "173.xml", "174.xml", "175.xml", "180.xml", - "182.xml", + "063.xml", "074.xml", "078.xml", "079.xml", "080.xml", "082.xml", "084.xml", "090.xml", + "092.xml", "103.xml", "107.xml", "115.xml", "116.xml", "117.xml", "119.xml", "120.xml", + "140.xml", "141.xml", "153.xml", "160.xml", "161.xml", "162.xml", "180.xml", "182.xml", ] /// valid/sa/114.xml: a CDATA section inside an entity value must protect