From f5c04058dca094886ad6b98d1d3da0f38bd4a53e Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Mon, 8 Jun 2026 15:15:46 +0200 Subject: [PATCH] feat(regex): Unicode category escapes and character-class subtraction Close the two XSD-regex gaps from the audit: - \p{...} and \P{...} now resolve Unicode general categories (one-letter groups like \p{L} and two-letter codes like \p{Lu}/\p{Nd}) through Swift's own scalar generalCategory, so no character tables are vendored. A curated set of well-known \p{Is} blocks is supported; an unknown category or block name is a compile error rather than a guess. - Character-class subtraction [a-z-[aeiou]] removes the subtrahend from a class, and nests ([a-z-[b-d-[c]]]) and composes with categories. Categories and subtraction work both as standalone atoms and inside character classes, so XSD pattern facets that use them now validate. Closes #89 --- Sources/Regex/RegexAutomaton.swift | 8 ++-- Sources/Regex/RegexCategory.swift | 76 ++++++++++++++++++++++++++++++ Sources/Regex/RegexParser.swift | 32 +++++++++++++ Sources/Regex/RegexSyntax.swift | 9 +++- Tests/RegexCategoryTests.swift | 73 ++++++++++++++++++++++++++++ 5 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 Sources/Regex/RegexCategory.swift create mode 100644 Tests/RegexCategoryTests.swift diff --git a/Sources/Regex/RegexAutomaton.swift b/Sources/Regex/RegexAutomaton.swift index 486f483..bbcdab9 100644 --- a/Sources/Regex/RegexAutomaton.swift +++ b/Sources/Regex/RegexAutomaton.swift @@ -148,10 +148,12 @@ public extension PureXML.Regex { /// A compiled regular expression in the XML Schema flavor: literals, `.`, the /// character-class escapes (`\d \D \w \W \s \S \i \I \c \C`), single-character /// escapes, character classes with ranges and negation, grouping, alternation - /// (`|`), and the quantifiers `?`, `*`, `+`, `{n}`, `{n,}`, `{n,m}`. Matching - /// is whole-string (anchored), as XSD `pattern` facets require. Built on a + /// (`|`), and the quantifiers `?`, `*`, `+`, `{n}`, `{n,}`, `{n,m}`. Character + /// classes also support Unicode category and block escapes (`\p{L}`, `\P{Nd}`, + /// `\p{IsBasicLatin}`) and subtraction (`[a-z-[aeiou]]`). Matching is + /// whole-string (anchored), as XSD `pattern` facets require. Built on a /// Thompson NFA, so it runs in time linear in the input with no backtracking - /// blow-up. Unicode category escapes (`\p{...}`) are not yet supported. + /// blow-up. struct Pattern: Sendable { private let nfa: NFA diff --git a/Sources/Regex/RegexCategory.swift b/Sources/Regex/RegexCategory.swift new file mode 100644 index 0000000..7acb9fc --- /dev/null +++ b/Sources/Regex/RegexCategory.swift @@ -0,0 +1,76 @@ +extension PureXML.Regex { + /// A `\p{...}` / `\P{...}` Unicode category or block test. A category maps to + /// the scalar's Unicode general category through Swift's own `generalCategory`, + /// so no character tables are vendored; an `Is` name tests a contiguous + /// Unicode block. An unknown name is rejected at compile time. + struct CategoryPredicate: Equatable, Sendable { + let name: String + let negated: Bool + + init?(name: String, negated: Bool) { + guard CategoryMatcher.isKnown(name) else { return nil } + self.name = name + self.negated = negated + } + + func contains(_ scalar: Unicode.Scalar) -> Bool { + let hit = CategoryMatcher.matches(name, scalar) + return negated ? !hit : hit + } + } + + /// Resolves `\p{...}` names: a one-letter general-category group (`L`, `N`, ...), + /// a two-letter category (`Lu`, `Nd`, ...), or an `Is` block name. + enum CategoryMatcher { + static func isKnown(_ name: String) -> Bool { + if name.hasPrefix("Is") { return blocks[String(name.dropFirst(2))] != nil } + return groups.contains(name) || codeValues.contains(name) + } + + static func matches(_ name: String, _ scalar: Unicode.Scalar) -> Bool { + if name.hasPrefix("Is") { + guard let range = blocks[String(name.dropFirst(2))] else { return false } + return range.contains(scalar.value) + } + let code = codes[scalar.properties.generalCategory] ?? "Cn" + return name.count == 1 ? code.hasPrefix(name) : code == name + } + + private static let groups: Set = ["L", "M", "N", "P", "S", "Z", "C"] + private static let codeValues = Set(codes.values) + + private static let codes: [Unicode.GeneralCategory: String] = [ + .uppercaseLetter: "Lu", .lowercaseLetter: "Ll", .titlecaseLetter: "Lt", + .modifierLetter: "Lm", .otherLetter: "Lo", + .nonspacingMark: "Mn", .spacingMark: "Mc", .enclosingMark: "Me", + .decimalNumber: "Nd", .letterNumber: "Nl", .otherNumber: "No", + .connectorPunctuation: "Pc", .dashPunctuation: "Pd", .openPunctuation: "Ps", + .closePunctuation: "Pe", .initialPunctuation: "Pi", .finalPunctuation: "Pf", + .otherPunctuation: "Po", + .mathSymbol: "Sm", .currencySymbol: "Sc", .modifierSymbol: "Sk", .otherSymbol: "So", + .spaceSeparator: "Zs", .lineSeparator: "Zl", .paragraphSeparator: "Zp", + .control: "Cc", .format: "Cf", .surrogate: "Cs", .privateUse: "Co", .unassigned: "Cn", + ] + + /// A curated set of well-known Unicode blocks. An `Is` name outside + /// this set is rejected rather than guessed, so a pattern never matches + /// against a fabricated range. + private static let blocks: [String: ClosedRange] = [ + "BasicLatin": 0x0000 ... 0x007F, + "Latin-1Supplement": 0x0080 ... 0x00FF, + "LatinExtended-A": 0x0100 ... 0x017F, + "LatinExtended-B": 0x0180 ... 0x024F, + "IPAExtensions": 0x0250 ... 0x02AF, + "Greek": 0x0370 ... 0x03FF, + "GreekandCoptic": 0x0370 ... 0x03FF, + "Cyrillic": 0x0400 ... 0x04FF, + "Hebrew": 0x0590 ... 0x05FF, + "Arabic": 0x0600 ... 0x06FF, + "GeneralPunctuation": 0x2000 ... 0x206F, + "CurrencySymbols": 0x20A0 ... 0x20CF, + "Hiragana": 0x3040 ... 0x309F, + "Katakana": 0x30A0 ... 0x30FF, + "CJKUnifiedIdeographs": 0x4E00 ... 0x9FFF, + ] + } +} diff --git a/Sources/Regex/RegexParser.swift b/Sources/Regex/RegexParser.swift index b4b6a3d..b824e72 100644 --- a/Sources/Regex/RegexParser.swift +++ b/Sources/Regex/RegexParser.swift @@ -4,6 +4,7 @@ private enum ClassMember { case scalar(Unicode.Scalar) case named(PureXML.Regex.NamedClass) case negatedNamed(PureXML.Regex.NamedClass) + case category(PureXML.Regex.CategoryPredicate) } extension PureXML.Regex { @@ -112,14 +113,36 @@ extension PureXML.Regex { if let member = Self.classEscape(character) { return classFrom(member) } + if character == "p" || character == "P" { + return try CharClass(categories: [parseCategory(negated: character == "P")]) + } throw RegexError.badEscape(String(character)) } + /// Parses the `{name}` of a `\p`/`\P` escape into a category predicate, + /// rejecting an unknown category or block name. + private mutating func parseCategory(negated: Bool) throws -> CategoryPredicate { + guard peek() == "{" else { throw RegexError.badEscape(negated ? "P" : "p") } + advance() + var name = "" + while let character = peek(), character != "}" { + name.append(character) + advance() + } + guard peek() == "}" else { throw RegexError.badClass } + advance() + guard let predicate = CategoryPredicate(name: name, negated: negated) else { + throw RegexError.unsupported("\\p{\(name)}") + } + return predicate + } + private func classFrom(_ member: ClassMember) -> CharClass { switch member { case let .scalar(value): .single(value) case let .named(named): CharClass(named: [named]) case let .negatedNamed(named): CharClass(negatedNamed: [named]) + case let .category(predicate): CharClass(categories: [predicate]) } } @@ -156,6 +179,11 @@ extension PureXML.Regex { cls.negated = true } while let character = peek(), character != "]" { + if character == "-", peek(1) == "[" { + advance() + cls.subtraction = try [parseClass()] + break + } try parseClassMember(into: &cls) } guard peek() == "]" else { throw RegexError.badClass } @@ -183,6 +211,7 @@ extension PureXML.Regex { case let .scalar(value): cls.ranges.append(value ... value) case let .named(named): cls.named.append(named) case let .negatedNamed(named): cls.negatedNamed.append(named) + case let .category(predicate): cls.categories.append(predicate) } } @@ -194,6 +223,9 @@ extension PureXML.Regex { advance() if let single = Self.singleEscape(escaped) { return .scalar(single) } if let member = Self.classEscape(escaped) { return member } + if escaped == "p" || escaped == "P" { + return try .category(parseCategory(negated: escaped == "P")) + } throw RegexError.badEscape(String(escaped)) } diff --git a/Sources/Regex/RegexSyntax.swift b/Sources/Regex/RegexSyntax.swift index 9198f67..2baa0c5 100644 --- a/Sources/Regex/RegexSyntax.swift +++ b/Sources/Regex/RegexSyntax.swift @@ -49,14 +49,21 @@ extension PureXML.Regex { var ranges: [ClosedRange] = [] var named: [NamedClass] = [] var negatedNamed: [NamedClass] = [] + /// `\p{...}`/`\P{...}` Unicode category and block tests. + var categories: [CategoryPredicate] = [] + /// A character-class subtraction (`[a-z-[aeiou]]`): characters this class + /// otherwise admits are removed when the subtrahend matches them. Nests. + var subtraction: [CharClass] = [] func matches(_ character: Character) -> Bool { guard character.unicodeScalars.count == 1, let scalar = character.unicodeScalars.first else { return false } - let hit = ranges.contains { $0.contains(scalar) } + var hit = ranges.contains { $0.contains(scalar) } || named.contains { $0.contains(scalar) } || negatedNamed.contains { !$0.contains(scalar) } + || categories.contains { $0.contains(scalar) } + if hit, subtraction.contains(where: { $0.matches(character) }) { hit = false } return negated ? !hit : hit } diff --git a/Tests/RegexCategoryTests.swift b/Tests/RegexCategoryTests.swift new file mode 100644 index 0000000..ac4d607 --- /dev/null +++ b/Tests/RegexCategoryTests.swift @@ -0,0 +1,73 @@ +@testable import PureXML +import Testing + +@Suite("Regex Unicode categories and class subtraction") +struct RegexCategoryTests { + private func matches(_ pattern: String, _ string: String) throws -> Bool { + try PureXML.Regex.matches(pattern, string) + } + + @Test("\\p{L} matches a letter group and rejects non-letters") + func test_letterGroup() throws { + #expect(try matches("\\p{L}+", "abcÅ")) + #expect(try !matches("\\p{L}+", "ab1")) + } + + @Test("\\p{Lu} and \\p{Ll} distinguish case") + func test_caseCategories() throws { + #expect(try matches("\\p{Lu}\\p{Ll}+", "Hello")) + #expect(try !matches("\\p{Lu}\\p{Ll}+", "hello")) + } + + @Test("\\p{Nd} matches decimal digits") + func test_decimalNumber() throws { + #expect(try matches("\\p{Nd}+", "2026")) + #expect(try !matches("\\p{Nd}+", "12a")) + } + + @Test("\\P{...} negates a category") + func test_negatedCategory() throws { + #expect(try matches("\\P{L}+", "123 !")) + #expect(try !matches("\\P{L}+", "12a")) + } + + @Test("A category inside a character class composes with ranges") + func test_categoryInClass() throws { + // A letter or an underscore. + #expect(try matches("[\\p{L}_]+", "a_b")) + #expect(try !matches("[\\p{L}_]+", "a-b")) + } + + @Test("\\p{IsBasicLatin} tests a Unicode block") + func test_block() throws { + #expect(try matches("\\p{IsBasicLatin}+", "abc")) + #expect(try !matches("\\p{IsBasicLatin}+", "abç")) + } + + @Test("An unknown category name is a compile error") + func test_unknownCategory() { + #expect(throws: PureXML.Regex.RegexError.self) { + _ = try PureXML.Regex.Pattern("\\p{Zz}") + } + } + + @Test("Character-class subtraction removes the subtrahend") + func test_subtraction() throws { + #expect(try matches("[a-z-[aeiou]]+", "bcdfg")) + #expect(try !matches("[a-z-[aeiou]]+", "bce")) + } + + @Test("Subtraction nests") + func test_nestedSubtraction() throws { + // a-z, minus (b-d minus c): so b and d are removed, c stays. + #expect(try matches("[a-z-[b-d-[c]]]+", "acez")) + #expect(try !matches("[a-z-[b-d-[c]]]+", "abz")) + } + + @Test("Subtraction composes with a category subtrahend") + func test_subtractCategory() throws { + // Any letter that is not lowercase. + #expect(try matches("[\\p{L}-[\\p{Ll}]]+", "ABC")) + #expect(try !matches("[\\p{L}-[\\p{Ll}]]+", "ABc")) + } +}