feat(regex): Unicode category escapes and character-class subtraction

Close the two XSD-regex gaps from the audit:

- \p{...} and \P{...} now resolve Unicode general categories (one-letter
  groups like \p{L} and two-letter codes like \p{Lu}/\p{Nd}) through
  Swift's own scalar generalCategory, so no character tables are vendored.
  A curated set of well-known \p{Is<Block>} blocks is supported; an
  unknown category or block name is a compile error rather than a guess.
- Character-class subtraction [a-z-[aeiou]] removes the subtrahend from a
  class, and nests ([a-z-[b-d-[c]]]) and composes with categories.

Categories and subtraction work both as standalone atoms and inside
character classes, so XSD pattern facets that use them now validate.

Closes #89
This commit is contained in:
Mihaela Mihaljevic 2026-06-08 15:15:46 +02:00
parent dd6300538c
commit f5c04058dc
5 changed files with 194 additions and 4 deletions

View File

@ -148,10 +148,12 @@ public extension PureXML.Regex {
/// A compiled regular expression in the XML Schema flavor: literals, `.`, the
/// character-class escapes (`\d \D \w \W \s \S \i \I \c \C`), single-character
/// escapes, character classes with ranges and negation, grouping, alternation
/// (`|`), and the quantifiers `?`, `*`, `+`, `{n}`, `{n,}`, `{n,m}`. Matching
/// is whole-string (anchored), as XSD `pattern` facets require. Built on a
/// (`|`), and the quantifiers `?`, `*`, `+`, `{n}`, `{n,}`, `{n,m}`. Character
/// classes also support Unicode category and block escapes (`\p{L}`, `\P{Nd}`,
/// `\p{IsBasicLatin}`) and subtraction (`[a-z-[aeiou]]`). Matching is
/// whole-string (anchored), as XSD `pattern` facets require. Built on a
/// Thompson NFA, so it runs in time linear in the input with no backtracking
/// blow-up. Unicode category escapes (`\p{...}`) are not yet supported.
/// blow-up.
struct Pattern: Sendable {
private let nfa: NFA

View File

@ -0,0 +1,76 @@
extension PureXML.Regex {
/// A `\p{...}` / `\P{...}` Unicode category or block test. A category maps to
/// the scalar's Unicode general category through Swift's own `generalCategory`,
/// so no character tables are vendored; an `Is<Block>` name tests a contiguous
/// Unicode block. An unknown name is rejected at compile time.
struct CategoryPredicate: Equatable, Sendable {
let name: String
let negated: Bool
init?(name: String, negated: Bool) {
guard CategoryMatcher.isKnown(name) else { return nil }
self.name = name
self.negated = negated
}
func contains(_ scalar: Unicode.Scalar) -> Bool {
let hit = CategoryMatcher.matches(name, scalar)
return negated ? !hit : hit
}
}
/// Resolves `\p{...}` names: a one-letter general-category group (`L`, `N`, ...),
/// a two-letter category (`Lu`, `Nd`, ...), or an `Is<Block>` block name.
enum CategoryMatcher {
static func isKnown(_ name: String) -> Bool {
if name.hasPrefix("Is") { return blocks[String(name.dropFirst(2))] != nil }
return groups.contains(name) || codeValues.contains(name)
}
static func matches(_ name: String, _ scalar: Unicode.Scalar) -> Bool {
if name.hasPrefix("Is") {
guard let range = blocks[String(name.dropFirst(2))] else { return false }
return range.contains(scalar.value)
}
let code = codes[scalar.properties.generalCategory] ?? "Cn"
return name.count == 1 ? code.hasPrefix(name) : code == name
}
private static let groups: Set<String> = ["L", "M", "N", "P", "S", "Z", "C"]
private static let codeValues = Set(codes.values)
private static let codes: [Unicode.GeneralCategory: String] = [
.uppercaseLetter: "Lu", .lowercaseLetter: "Ll", .titlecaseLetter: "Lt",
.modifierLetter: "Lm", .otherLetter: "Lo",
.nonspacingMark: "Mn", .spacingMark: "Mc", .enclosingMark: "Me",
.decimalNumber: "Nd", .letterNumber: "Nl", .otherNumber: "No",
.connectorPunctuation: "Pc", .dashPunctuation: "Pd", .openPunctuation: "Ps",
.closePunctuation: "Pe", .initialPunctuation: "Pi", .finalPunctuation: "Pf",
.otherPunctuation: "Po",
.mathSymbol: "Sm", .currencySymbol: "Sc", .modifierSymbol: "Sk", .otherSymbol: "So",
.spaceSeparator: "Zs", .lineSeparator: "Zl", .paragraphSeparator: "Zp",
.control: "Cc", .format: "Cf", .surrogate: "Cs", .privateUse: "Co", .unassigned: "Cn",
]
/// A curated set of well-known Unicode blocks. An `Is<Block>` name outside
/// this set is rejected rather than guessed, so a pattern never matches
/// against a fabricated range.
private static let blocks: [String: ClosedRange<UInt32>] = [
"BasicLatin": 0x0000 ... 0x007F,
"Latin-1Supplement": 0x0080 ... 0x00FF,
"LatinExtended-A": 0x0100 ... 0x017F,
"LatinExtended-B": 0x0180 ... 0x024F,
"IPAExtensions": 0x0250 ... 0x02AF,
"Greek": 0x0370 ... 0x03FF,
"GreekandCoptic": 0x0370 ... 0x03FF,
"Cyrillic": 0x0400 ... 0x04FF,
"Hebrew": 0x0590 ... 0x05FF,
"Arabic": 0x0600 ... 0x06FF,
"GeneralPunctuation": 0x2000 ... 0x206F,
"CurrencySymbols": 0x20A0 ... 0x20CF,
"Hiragana": 0x3040 ... 0x309F,
"Katakana": 0x30A0 ... 0x30FF,
"CJKUnifiedIdeographs": 0x4E00 ... 0x9FFF,
]
}
}

View File

@ -4,6 +4,7 @@ private enum ClassMember {
case scalar(Unicode.Scalar)
case named(PureXML.Regex.NamedClass)
case negatedNamed(PureXML.Regex.NamedClass)
case category(PureXML.Regex.CategoryPredicate)
}
extension PureXML.Regex {
@ -112,14 +113,36 @@ extension PureXML.Regex {
if let member = Self.classEscape(character) {
return classFrom(member)
}
if character == "p" || character == "P" {
return try CharClass(categories: [parseCategory(negated: character == "P")])
}
throw RegexError.badEscape(String(character))
}
/// Parses the `{name}` of a `\p`/`\P` escape into a category predicate,
/// rejecting an unknown category or block name.
private mutating func parseCategory(negated: Bool) throws -> CategoryPredicate {
guard peek() == "{" else { throw RegexError.badEscape(negated ? "P" : "p") }
advance()
var name = ""
while let character = peek(), character != "}" {
name.append(character)
advance()
}
guard peek() == "}" else { throw RegexError.badClass }
advance()
guard let predicate = CategoryPredicate(name: name, negated: negated) else {
throw RegexError.unsupported("\\p{\(name)}")
}
return predicate
}
private func classFrom(_ member: ClassMember) -> CharClass {
switch member {
case let .scalar(value): .single(value)
case let .named(named): CharClass(named: [named])
case let .negatedNamed(named): CharClass(negatedNamed: [named])
case let .category(predicate): CharClass(categories: [predicate])
}
}
@ -156,6 +179,11 @@ extension PureXML.Regex {
cls.negated = true
}
while let character = peek(), character != "]" {
if character == "-", peek(1) == "[" {
advance()
cls.subtraction = try [parseClass()]
break
}
try parseClassMember(into: &cls)
}
guard peek() == "]" else { throw RegexError.badClass }
@ -183,6 +211,7 @@ extension PureXML.Regex {
case let .scalar(value): cls.ranges.append(value ... value)
case let .named(named): cls.named.append(named)
case let .negatedNamed(named): cls.negatedNamed.append(named)
case let .category(predicate): cls.categories.append(predicate)
}
}
@ -194,6 +223,9 @@ extension PureXML.Regex {
advance()
if let single = Self.singleEscape(escaped) { return .scalar(single) }
if let member = Self.classEscape(escaped) { return member }
if escaped == "p" || escaped == "P" {
return try .category(parseCategory(negated: escaped == "P"))
}
throw RegexError.badEscape(String(escaped))
}

View File

@ -49,14 +49,21 @@ extension PureXML.Regex {
var ranges: [ClosedRange<Unicode.Scalar>] = []
var named: [NamedClass] = []
var negatedNamed: [NamedClass] = []
/// `\p{...}`/`\P{...}` Unicode category and block tests.
var categories: [CategoryPredicate] = []
/// A character-class subtraction (`[a-z-[aeiou]]`): characters this class
/// otherwise admits are removed when the subtrahend matches them. Nests.
var subtraction: [CharClass] = []
func matches(_ character: Character) -> Bool {
guard character.unicodeScalars.count == 1, let scalar = character.unicodeScalars.first else {
return false
}
let hit = ranges.contains { $0.contains(scalar) }
var hit = ranges.contains { $0.contains(scalar) }
|| named.contains { $0.contains(scalar) }
|| negatedNamed.contains { !$0.contains(scalar) }
|| categories.contains { $0.contains(scalar) }
if hit, subtraction.contains(where: { $0.matches(character) }) { hit = false }
return negated ? !hit : hit
}

View File

@ -0,0 +1,73 @@
@testable import PureXML
import Testing
@Suite("Regex Unicode categories and class subtraction")
struct RegexCategoryTests {
private func matches(_ pattern: String, _ string: String) throws -> Bool {
try PureXML.Regex.matches(pattern, string)
}
@Test("\\p{L} matches a letter group and rejects non-letters")
func test_letterGroup() throws {
#expect(try matches("\\p{L}+", "abcÅ"))
#expect(try !matches("\\p{L}+", "ab1"))
}
@Test("\\p{Lu} and \\p{Ll} distinguish case")
func test_caseCategories() throws {
#expect(try matches("\\p{Lu}\\p{Ll}+", "Hello"))
#expect(try !matches("\\p{Lu}\\p{Ll}+", "hello"))
}
@Test("\\p{Nd} matches decimal digits")
func test_decimalNumber() throws {
#expect(try matches("\\p{Nd}+", "2026"))
#expect(try !matches("\\p{Nd}+", "12a"))
}
@Test("\\P{...} negates a category")
func test_negatedCategory() throws {
#expect(try matches("\\P{L}+", "123 !"))
#expect(try !matches("\\P{L}+", "12a"))
}
@Test("A category inside a character class composes with ranges")
func test_categoryInClass() throws {
// A letter or an underscore.
#expect(try matches("[\\p{L}_]+", "a_b"))
#expect(try !matches("[\\p{L}_]+", "a-b"))
}
@Test("\\p{IsBasicLatin} tests a Unicode block")
func test_block() throws {
#expect(try matches("\\p{IsBasicLatin}+", "abc"))
#expect(try !matches("\\p{IsBasicLatin}+", "abç"))
}
@Test("An unknown category name is a compile error")
func test_unknownCategory() {
#expect(throws: PureXML.Regex.RegexError.self) {
_ = try PureXML.Regex.Pattern("\\p{Zz}")
}
}
@Test("Character-class subtraction removes the subtrahend")
func test_subtraction() throws {
#expect(try matches("[a-z-[aeiou]]+", "bcdfg"))
#expect(try !matches("[a-z-[aeiou]]+", "bce"))
}
@Test("Subtraction nests")
func test_nestedSubtraction() throws {
// a-z, minus (b-d minus c): so b and d are removed, c stays.
#expect(try matches("[a-z-[b-d-[c]]]+", "acez"))
#expect(try !matches("[a-z-[b-d-[c]]]+", "abz"))
}
@Test("Subtraction composes with a category subtrahend")
func test_subtractCategory() throws {
// Any letter that is not lowercase.
#expect(try matches("[\\p{L}-[\\p{Ll}]]+", "ABC"))
#expect(try !matches("[\\p{L}-[\\p{Ll}]]+", "ABc"))
}
}