diff --git a/CHANGELOG.md b/CHANGELOG.md index 91898c5..5750c4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- The regex engine now recognizes the full XSD `\p{Is}` Unicode block set (#147). Only about fifteen blocks were tabulated, so a pattern over any other block (`\p{IsArmenian}`, `\p{IsTamil}`, `\p{IsHangulJamo}`, and so on) failed to compile and rejected valid instances. The table now carries the Unicode 3.1 blocks the XSD regular-expression appendix fixes (surrogate blocks omitted, since surrogates are not scalar values); each range is verified bidirectionally by the XSTS corpus (valid in-block instances pass, invalid out-of-block instances still fail). XSTS: valid instances rejected 502 to 395. - Length facets on `xs:QName` no longer reject valid values (#147). XSD 1.0 Datatypes 4.3.1 does not define a unit of length for QName (only string, anyURI, binary, and list), so `length`/`minLength`/`maxLength` on QName are implementation-defined; like Xerces and the XSTS NIST set we treat them as non-constraining rather than counting characters. Net XSTS: valid instances rejected 547 to 502 (36+ NIST false rejections fixed); 4 Microsoft cases that read QName length as a character count remain in invalid instances accepted (257 to 261) and are documented as a deliberate spec-ambiguity interpretation in docs/xsts-deviations.md. - Ordering facets on `xs:duration` are now enforced (#147). `minInclusive`, `maxInclusive`, `minExclusive`, and `maxExclusive` on duration were ignored because duration has no total order in the generic comparator. A `DurationValue` now implements the XSD 1.0 Datatypes 3.2.6.2 partial order (compare by adding to the four reference dateTimes); a value incomparable to the bound (such as `P1M` against `P30D`) is outside the range and rejected. XSTS: invalid instances accepted 355 to 257. - Pattern and enumeration facets on a union type are now enforced (#147). A union admits a value valid for any member, but `pattern` and `enumeration` (the only constraining facets XSD allows on a union) had been ignored, so a member-valid value outside the enumeration or failing the pattern was wrongly accepted. Enumeration compares in the value space of the admitting member (so `01` equals an enumerated `1` for an integer member), matching the atomic path. XSTS #147: invalid instances accepted 555 to 355. diff --git a/Sources/Regex/RegexCategory.swift b/Sources/Regex/RegexCategory.swift index 7acb9fc..46b0e58 100644 --- a/Sources/Regex/RegexCategory.swift +++ b/Sources/Regex/RegexCategory.swift @@ -52,25 +52,99 @@ extension PureXML.Regex { .control: "Cc", .format: "Cf", .surrogate: "Cs", .privateUse: "Co", .unassigned: "Cn", ] - /// A curated set of well-known Unicode blocks. An `Is` name outside - /// this set is rejected rather than guessed, so a pattern never matches - /// against a fabricated range. + /// The Unicode block names XSD 1.0 admits in `\p{Is}` (the + /// Unicode 3.1 blocks the regular-expression appendix fixes), each mapped + /// to its code-point range. An `Is` name outside this set is + /// rejected rather than guessed, so a pattern never matches against a + /// fabricated range. Surrogate blocks are omitted: surrogates are not + /// scalar values. `Specials` uses the main `FFF0..FFFD` range (the spec's + /// separate `FEFF` half is the byte-order mark, not matched here). private static let blocks: [String: ClosedRange] = [ "BasicLatin": 0x0000 ... 0x007F, "Latin-1Supplement": 0x0080 ... 0x00FF, "LatinExtended-A": 0x0100 ... 0x017F, "LatinExtended-B": 0x0180 ... 0x024F, "IPAExtensions": 0x0250 ... 0x02AF, + "SpacingModifierLetters": 0x02B0 ... 0x02FF, + "CombiningDiacriticalMarks": 0x0300 ... 0x036F, "Greek": 0x0370 ... 0x03FF, "GreekandCoptic": 0x0370 ... 0x03FF, "Cyrillic": 0x0400 ... 0x04FF, + "Armenian": 0x0530 ... 0x058F, "Hebrew": 0x0590 ... 0x05FF, "Arabic": 0x0600 ... 0x06FF, + "Syriac": 0x0700 ... 0x074F, + "Thaana": 0x0780 ... 0x07BF, + "Devanagari": 0x0900 ... 0x097F, + "Bengali": 0x0980 ... 0x09FF, + "Gurmukhi": 0x0A00 ... 0x0A7F, + "Gujarati": 0x0A80 ... 0x0AFF, + "Oriya": 0x0B00 ... 0x0B7F, + "Tamil": 0x0B80 ... 0x0BFF, + "Telugu": 0x0C00 ... 0x0C7F, + "Kannada": 0x0C80 ... 0x0CFF, + "Malayalam": 0x0D00 ... 0x0D7F, + "Sinhala": 0x0D80 ... 0x0DFF, + "Thai": 0x0E00 ... 0x0E7F, + "Lao": 0x0E80 ... 0x0EFF, + "Tibetan": 0x0F00 ... 0x0FFF, + "Myanmar": 0x1000 ... 0x109F, + "Georgian": 0x10A0 ... 0x10FF, + "HangulJamo": 0x1100 ... 0x11FF, + "Ethiopic": 0x1200 ... 0x137F, + "Cherokee": 0x13A0 ... 0x13FF, + "UnifiedCanadianAboriginalSyllabics": 0x1400 ... 0x167F, + "Ogham": 0x1680 ... 0x169F, + "Runic": 0x16A0 ... 0x16FF, + "Khmer": 0x1780 ... 0x17FF, + "Mongolian": 0x1800 ... 0x18AF, + "LatinExtendedAdditional": 0x1E00 ... 0x1EFF, + "GreekExtended": 0x1F00 ... 0x1FFF, "GeneralPunctuation": 0x2000 ... 0x206F, + "SuperscriptsandSubscripts": 0x2070 ... 0x209F, "CurrencySymbols": 0x20A0 ... 0x20CF, + "CombiningMarksforSymbols": 0x20D0 ... 0x20FF, + "LetterlikeSymbols": 0x2100 ... 0x214F, + "NumberForms": 0x2150 ... 0x218F, + "Arrows": 0x2190 ... 0x21FF, + "MathematicalOperators": 0x2200 ... 0x22FF, + "MiscellaneousTechnical": 0x2300 ... 0x23FF, + "ControlPictures": 0x2400 ... 0x243F, + "OpticalCharacterRecognition": 0x2440 ... 0x245F, + "EnclosedAlphanumerics": 0x2460 ... 0x24FF, + "BoxDrawing": 0x2500 ... 0x257F, + "BlockElements": 0x2580 ... 0x259F, + "GeometricShapes": 0x25A0 ... 0x25FF, + "MiscellaneousSymbols": 0x2600 ... 0x26FF, + "Dingbats": 0x2700 ... 0x27BF, + "BraillePatterns": 0x2800 ... 0x28FF, + "CJKRadicalsSupplement": 0x2E80 ... 0x2EFF, + "KangxiRadicals": 0x2F00 ... 0x2FDF, + "IdeographicDescriptionCharacters": 0x2FF0 ... 0x2FFF, + "CJKSymbolsandPunctuation": 0x3000 ... 0x303F, "Hiragana": 0x3040 ... 0x309F, "Katakana": 0x30A0 ... 0x30FF, + "Bopomofo": 0x3100 ... 0x312F, + "HangulCompatibilityJamo": 0x3130 ... 0x318F, + "Kanbun": 0x3190 ... 0x319F, + "BopomofoExtended": 0x31A0 ... 0x31BF, + "EnclosedCJKLettersandMonths": 0x3200 ... 0x32FF, + "CJKCompatibility": 0x3300 ... 0x33FF, + "CJKUnifiedIdeographsExtensionA": 0x3400 ... 0x4DB5, "CJKUnifiedIdeographs": 0x4E00 ... 0x9FFF, + "YiSyllables": 0xA000 ... 0xA48F, + "YiRadicals": 0xA490 ... 0xA4CF, + "HangulSyllables": 0xAC00 ... 0xD7A3, + "PrivateUse": 0xE000 ... 0xF8FF, + "CJKCompatibilityIdeographs": 0xF900 ... 0xFAFF, + "AlphabeticPresentationForms": 0xFB00 ... 0xFB4F, + "ArabicPresentationForms-A": 0xFB50 ... 0xFDFF, + "CombiningHalfMarks": 0xFE20 ... 0xFE2F, + "CJKCompatibilityForms": 0xFE30 ... 0xFE4F, + "SmallFormVariants": 0xFE50 ... 0xFE6F, + "ArabicPresentationForms-B": 0xFE70 ... 0xFEFF, + "HalfwidthandFullwidthForms": 0xFF00 ... 0xFFEF, + "Specials": 0xFFF0 ... 0xFFFD, ] } } diff --git a/Tests/RegexTests.swift b/Tests/RegexTests.swift index 04db049..e6065a8 100644 --- a/Tests/RegexTests.swift +++ b/Tests/RegexTests.swift @@ -112,4 +112,16 @@ struct RegexTests { #expect(try !matches("a{2,4}", "a")) #expect(try !matches("a{2,4}", "aaaaa")) } + + @Test("Unicode block escapes match within their block and reject outside it") + func test_unicodeBlocks() throws { + // \u{0531} Armenian, \u{0B85} Tamil, \u{1100} Hangul Jamo. + #expect(try matches(#"\p{IsArmenian}+"#, "\u{0531}\u{0561}")) + #expect(try !matches(#"\p{IsArmenian}+"#, "abc")) + #expect(try matches(#"\p{IsTamil}+"#, "\u{0B85}")) + #expect(try matches(#"\p{IsHangulJamo}+"#, "\u{1100}")) + #expect(try !matches(#"\p{IsTamil}+"#, "\u{0531}")) // Armenian is not Tamil + // An unknown block name is still a compile error. + #expect(throws: PureXML.Regex.RegexError.self) { _ = try PureXML.Regex.Pattern(#"\p{IsNotARealBlock}"#) } + } } diff --git a/Tests/XSTSSuiteTests.swift b/Tests/XSTSSuiteTests.swift index 119942f..40ba4e3 100644 --- a/Tests/XSTSSuiteTests.swift +++ b/Tests/XSTSSuiteTests.swift @@ -30,7 +30,7 @@ struct XSTSSuiteTests { /// are written to /tmp/xsts-failures.txt for the burn-down. private let knownSchemaValidRejected = 75 private let knownSchemaInvalidAccepted = 2467 - private let knownInstanceValidRejected = 502 + private let knownInstanceValidRejected = 395 private let knownInstanceInvalidAccepted = 261 @Test("Every XSTS case behaves: compile, reject, validate, invalidate")