feat(regex): Unicode category escapes and character-class subtraction
Close the two XSD-regex gaps from the audit:
- \p{...} and \P{...} now resolve Unicode general categories (one-letter
groups like \p{L} and two-letter codes like \p{Lu}/\p{Nd}) through
Swift's own scalar generalCategory, so no character tables are vendored.
A curated set of well-known \p{Is<Block>} blocks is supported; an
unknown category or block name is a compile error rather than a guess.
- Character-class subtraction [a-z-[aeiou]] removes the subtrahend from a
class, and nests ([a-z-[b-d-[c]]]) and composes with categories.
Categories and subtraction work both as standalone atoms and inside
character classes, so XSD pattern facets that use them now validate.
Closes #89
This commit is contained in:
parent
dd6300538c
commit
f5c04058dc
@ -148,10 +148,12 @@ public extension PureXML.Regex {
|
||||
/// A compiled regular expression in the XML Schema flavor: literals, `.`, the
|
||||
/// character-class escapes (`\d \D \w \W \s \S \i \I \c \C`), single-character
|
||||
/// escapes, character classes with ranges and negation, grouping, alternation
|
||||
/// (`|`), and the quantifiers `?`, `*`, `+`, `{n}`, `{n,}`, `{n,m}`. Matching
|
||||
/// is whole-string (anchored), as XSD `pattern` facets require. Built on a
|
||||
/// (`|`), and the quantifiers `?`, `*`, `+`, `{n}`, `{n,}`, `{n,m}`. Character
|
||||
/// classes also support Unicode category and block escapes (`\p{L}`, `\P{Nd}`,
|
||||
/// `\p{IsBasicLatin}`) and subtraction (`[a-z-[aeiou]]`). Matching is
|
||||
/// whole-string (anchored), as XSD `pattern` facets require. Built on a
|
||||
/// Thompson NFA, so it runs in time linear in the input with no backtracking
|
||||
/// blow-up. Unicode category escapes (`\p{...}`) are not yet supported.
|
||||
/// blow-up.
|
||||
struct Pattern: Sendable {
|
||||
private let nfa: NFA
|
||||
|
||||
|
||||
76
Sources/Regex/RegexCategory.swift
Normal file
76
Sources/Regex/RegexCategory.swift
Normal file
@ -0,0 +1,76 @@
|
||||
extension PureXML.Regex {
|
||||
/// A `\p{...}` / `\P{...}` Unicode category or block test. A category maps to
|
||||
/// the scalar's Unicode general category through Swift's own `generalCategory`,
|
||||
/// so no character tables are vendored; an `Is<Block>` name tests a contiguous
|
||||
/// Unicode block. An unknown name is rejected at compile time.
|
||||
struct CategoryPredicate: Equatable, Sendable {
|
||||
let name: String
|
||||
let negated: Bool
|
||||
|
||||
init?(name: String, negated: Bool) {
|
||||
guard CategoryMatcher.isKnown(name) else { return nil }
|
||||
self.name = name
|
||||
self.negated = negated
|
||||
}
|
||||
|
||||
func contains(_ scalar: Unicode.Scalar) -> Bool {
|
||||
let hit = CategoryMatcher.matches(name, scalar)
|
||||
return negated ? !hit : hit
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves `\p{...}` names: a one-letter general-category group (`L`, `N`, ...),
|
||||
/// a two-letter category (`Lu`, `Nd`, ...), or an `Is<Block>` block name.
|
||||
enum CategoryMatcher {
|
||||
static func isKnown(_ name: String) -> Bool {
|
||||
if name.hasPrefix("Is") { return blocks[String(name.dropFirst(2))] != nil }
|
||||
return groups.contains(name) || codeValues.contains(name)
|
||||
}
|
||||
|
||||
static func matches(_ name: String, _ scalar: Unicode.Scalar) -> Bool {
|
||||
if name.hasPrefix("Is") {
|
||||
guard let range = blocks[String(name.dropFirst(2))] else { return false }
|
||||
return range.contains(scalar.value)
|
||||
}
|
||||
let code = codes[scalar.properties.generalCategory] ?? "Cn"
|
||||
return name.count == 1 ? code.hasPrefix(name) : code == name
|
||||
}
|
||||
|
||||
private static let groups: Set<String> = ["L", "M", "N", "P", "S", "Z", "C"]
|
||||
private static let codeValues = Set(codes.values)
|
||||
|
||||
private static let codes: [Unicode.GeneralCategory: String] = [
|
||||
.uppercaseLetter: "Lu", .lowercaseLetter: "Ll", .titlecaseLetter: "Lt",
|
||||
.modifierLetter: "Lm", .otherLetter: "Lo",
|
||||
.nonspacingMark: "Mn", .spacingMark: "Mc", .enclosingMark: "Me",
|
||||
.decimalNumber: "Nd", .letterNumber: "Nl", .otherNumber: "No",
|
||||
.connectorPunctuation: "Pc", .dashPunctuation: "Pd", .openPunctuation: "Ps",
|
||||
.closePunctuation: "Pe", .initialPunctuation: "Pi", .finalPunctuation: "Pf",
|
||||
.otherPunctuation: "Po",
|
||||
.mathSymbol: "Sm", .currencySymbol: "Sc", .modifierSymbol: "Sk", .otherSymbol: "So",
|
||||
.spaceSeparator: "Zs", .lineSeparator: "Zl", .paragraphSeparator: "Zp",
|
||||
.control: "Cc", .format: "Cf", .surrogate: "Cs", .privateUse: "Co", .unassigned: "Cn",
|
||||
]
|
||||
|
||||
/// A curated set of well-known Unicode blocks. An `Is<Block>` name outside
|
||||
/// this set is rejected rather than guessed, so a pattern never matches
|
||||
/// against a fabricated range.
|
||||
private static let blocks: [String: ClosedRange<UInt32>] = [
|
||||
"BasicLatin": 0x0000 ... 0x007F,
|
||||
"Latin-1Supplement": 0x0080 ... 0x00FF,
|
||||
"LatinExtended-A": 0x0100 ... 0x017F,
|
||||
"LatinExtended-B": 0x0180 ... 0x024F,
|
||||
"IPAExtensions": 0x0250 ... 0x02AF,
|
||||
"Greek": 0x0370 ... 0x03FF,
|
||||
"GreekandCoptic": 0x0370 ... 0x03FF,
|
||||
"Cyrillic": 0x0400 ... 0x04FF,
|
||||
"Hebrew": 0x0590 ... 0x05FF,
|
||||
"Arabic": 0x0600 ... 0x06FF,
|
||||
"GeneralPunctuation": 0x2000 ... 0x206F,
|
||||
"CurrencySymbols": 0x20A0 ... 0x20CF,
|
||||
"Hiragana": 0x3040 ... 0x309F,
|
||||
"Katakana": 0x30A0 ... 0x30FF,
|
||||
"CJKUnifiedIdeographs": 0x4E00 ... 0x9FFF,
|
||||
]
|
||||
}
|
||||
}
|
||||
@ -4,6 +4,7 @@ private enum ClassMember {
|
||||
case scalar(Unicode.Scalar)
|
||||
case named(PureXML.Regex.NamedClass)
|
||||
case negatedNamed(PureXML.Regex.NamedClass)
|
||||
case category(PureXML.Regex.CategoryPredicate)
|
||||
}
|
||||
|
||||
extension PureXML.Regex {
|
||||
@ -112,14 +113,36 @@ extension PureXML.Regex {
|
||||
if let member = Self.classEscape(character) {
|
||||
return classFrom(member)
|
||||
}
|
||||
if character == "p" || character == "P" {
|
||||
return try CharClass(categories: [parseCategory(negated: character == "P")])
|
||||
}
|
||||
throw RegexError.badEscape(String(character))
|
||||
}
|
||||
|
||||
/// Parses the `{name}` of a `\p`/`\P` escape into a category predicate,
|
||||
/// rejecting an unknown category or block name.
|
||||
private mutating func parseCategory(negated: Bool) throws -> CategoryPredicate {
|
||||
guard peek() == "{" else { throw RegexError.badEscape(negated ? "P" : "p") }
|
||||
advance()
|
||||
var name = ""
|
||||
while let character = peek(), character != "}" {
|
||||
name.append(character)
|
||||
advance()
|
||||
}
|
||||
guard peek() == "}" else { throw RegexError.badClass }
|
||||
advance()
|
||||
guard let predicate = CategoryPredicate(name: name, negated: negated) else {
|
||||
throw RegexError.unsupported("\\p{\(name)}")
|
||||
}
|
||||
return predicate
|
||||
}
|
||||
|
||||
private func classFrom(_ member: ClassMember) -> CharClass {
|
||||
switch member {
|
||||
case let .scalar(value): .single(value)
|
||||
case let .named(named): CharClass(named: [named])
|
||||
case let .negatedNamed(named): CharClass(negatedNamed: [named])
|
||||
case let .category(predicate): CharClass(categories: [predicate])
|
||||
}
|
||||
}
|
||||
|
||||
@ -156,6 +179,11 @@ extension PureXML.Regex {
|
||||
cls.negated = true
|
||||
}
|
||||
while let character = peek(), character != "]" {
|
||||
if character == "-", peek(1) == "[" {
|
||||
advance()
|
||||
cls.subtraction = try [parseClass()]
|
||||
break
|
||||
}
|
||||
try parseClassMember(into: &cls)
|
||||
}
|
||||
guard peek() == "]" else { throw RegexError.badClass }
|
||||
@ -183,6 +211,7 @@ extension PureXML.Regex {
|
||||
case let .scalar(value): cls.ranges.append(value ... value)
|
||||
case let .named(named): cls.named.append(named)
|
||||
case let .negatedNamed(named): cls.negatedNamed.append(named)
|
||||
case let .category(predicate): cls.categories.append(predicate)
|
||||
}
|
||||
}
|
||||
|
||||
@ -194,6 +223,9 @@ extension PureXML.Regex {
|
||||
advance()
|
||||
if let single = Self.singleEscape(escaped) { return .scalar(single) }
|
||||
if let member = Self.classEscape(escaped) { return member }
|
||||
if escaped == "p" || escaped == "P" {
|
||||
return try .category(parseCategory(negated: escaped == "P"))
|
||||
}
|
||||
throw RegexError.badEscape(String(escaped))
|
||||
}
|
||||
|
||||
|
||||
@ -49,14 +49,21 @@ extension PureXML.Regex {
|
||||
var ranges: [ClosedRange<Unicode.Scalar>] = []
|
||||
var named: [NamedClass] = []
|
||||
var negatedNamed: [NamedClass] = []
|
||||
/// `\p{...}`/`\P{...}` Unicode category and block tests.
|
||||
var categories: [CategoryPredicate] = []
|
||||
/// A character-class subtraction (`[a-z-[aeiou]]`): characters this class
|
||||
/// otherwise admits are removed when the subtrahend matches them. Nests.
|
||||
var subtraction: [CharClass] = []
|
||||
|
||||
func matches(_ character: Character) -> Bool {
|
||||
guard character.unicodeScalars.count == 1, let scalar = character.unicodeScalars.first else {
|
||||
return false
|
||||
}
|
||||
let hit = ranges.contains { $0.contains(scalar) }
|
||||
var hit = ranges.contains { $0.contains(scalar) }
|
||||
|| named.contains { $0.contains(scalar) }
|
||||
|| negatedNamed.contains { !$0.contains(scalar) }
|
||||
|| categories.contains { $0.contains(scalar) }
|
||||
if hit, subtraction.contains(where: { $0.matches(character) }) { hit = false }
|
||||
return negated ? !hit : hit
|
||||
}
|
||||
|
||||
|
||||
73
Tests/RegexCategoryTests.swift
Normal file
73
Tests/RegexCategoryTests.swift
Normal file
@ -0,0 +1,73 @@
|
||||
@testable import PureXML
|
||||
import Testing
|
||||
|
||||
@Suite("Regex Unicode categories and class subtraction")
|
||||
struct RegexCategoryTests {
|
||||
private func matches(_ pattern: String, _ string: String) throws -> Bool {
|
||||
try PureXML.Regex.matches(pattern, string)
|
||||
}
|
||||
|
||||
@Test("\\p{L} matches a letter group and rejects non-letters")
|
||||
func test_letterGroup() throws {
|
||||
#expect(try matches("\\p{L}+", "abcÅ"))
|
||||
#expect(try !matches("\\p{L}+", "ab1"))
|
||||
}
|
||||
|
||||
@Test("\\p{Lu} and \\p{Ll} distinguish case")
|
||||
func test_caseCategories() throws {
|
||||
#expect(try matches("\\p{Lu}\\p{Ll}+", "Hello"))
|
||||
#expect(try !matches("\\p{Lu}\\p{Ll}+", "hello"))
|
||||
}
|
||||
|
||||
@Test("\\p{Nd} matches decimal digits")
|
||||
func test_decimalNumber() throws {
|
||||
#expect(try matches("\\p{Nd}+", "2026"))
|
||||
#expect(try !matches("\\p{Nd}+", "12a"))
|
||||
}
|
||||
|
||||
@Test("\\P{...} negates a category")
|
||||
func test_negatedCategory() throws {
|
||||
#expect(try matches("\\P{L}+", "123 !"))
|
||||
#expect(try !matches("\\P{L}+", "12a"))
|
||||
}
|
||||
|
||||
@Test("A category inside a character class composes with ranges")
|
||||
func test_categoryInClass() throws {
|
||||
// A letter or an underscore.
|
||||
#expect(try matches("[\\p{L}_]+", "a_b"))
|
||||
#expect(try !matches("[\\p{L}_]+", "a-b"))
|
||||
}
|
||||
|
||||
@Test("\\p{IsBasicLatin} tests a Unicode block")
|
||||
func test_block() throws {
|
||||
#expect(try matches("\\p{IsBasicLatin}+", "abc"))
|
||||
#expect(try !matches("\\p{IsBasicLatin}+", "abç"))
|
||||
}
|
||||
|
||||
@Test("An unknown category name is a compile error")
|
||||
func test_unknownCategory() {
|
||||
#expect(throws: PureXML.Regex.RegexError.self) {
|
||||
_ = try PureXML.Regex.Pattern("\\p{Zz}")
|
||||
}
|
||||
}
|
||||
|
||||
@Test("Character-class subtraction removes the subtrahend")
|
||||
func test_subtraction() throws {
|
||||
#expect(try matches("[a-z-[aeiou]]+", "bcdfg"))
|
||||
#expect(try !matches("[a-z-[aeiou]]+", "bce"))
|
||||
}
|
||||
|
||||
@Test("Subtraction nests")
|
||||
func test_nestedSubtraction() throws {
|
||||
// a-z, minus (b-d minus c): so b and d are removed, c stays.
|
||||
#expect(try matches("[a-z-[b-d-[c]]]+", "acez"))
|
||||
#expect(try !matches("[a-z-[b-d-[c]]]+", "abz"))
|
||||
}
|
||||
|
||||
@Test("Subtraction composes with a category subtrahend")
|
||||
func test_subtractCategory() throws {
|
||||
// Any letter that is not lowercase.
|
||||
#expect(try matches("[\\p{L}-[\\p{Ll}]]+", "ABC"))
|
||||
#expect(try !matches("[\\p{L}-[\\p{Ll}]]+", "ABc"))
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user