From ed1904534ce865dccc00f70cd48aca2330f974b5 Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Wed, 24 Jun 2026 16:46:28 +0200 Subject: [PATCH] feat(parsing): port libyaml tag scanner to Swift Stage 3 of the faithful libyaml scanner port, continued. Translates fetch_tag, scan_tag, scan_tag_handle, scan_tag_uri, and scan_uri_escapes: verbatim (!), shorthand (!handle!suffix), and local (!suffix) tags, producing the handle and suffix for the parser to resolve. URI %-escapes are decoded by collecting the UTF-8 octets and decoding them as a unit. Unit-tested for a secondary-handle shorthand tag, a local tag, and a verbatim tag. Only directives (%YAML/%TAG) remain unported in the scanner. 317 tests, no regression. --- Sources/Parsing/LibYAML.Scanner.Fetch.swift | 21 ++- Sources/Parsing/LibYAML.Scanner.Tag.swift | 155 ++++++++++++++++++++ Sources/Parsing/LibYAML.TokenType.swift | 1 + Tests/LibYAMLScannerFetchTests.swift | 33 +++++ 4 files changed, 199 insertions(+), 11 deletions(-) create mode 100644 Sources/Parsing/LibYAML.Scanner.Tag.swift diff --git a/Sources/Parsing/LibYAML.Scanner.Fetch.swift b/Sources/Parsing/LibYAML.Scanner.Fetch.swift index 3818b77..c8bbc30 100644 --- a/Sources/Parsing/LibYAML.Scanner.Fetch.swift +++ b/Sources/Parsing/LibYAML.Scanner.Fetch.swift @@ -87,9 +87,6 @@ extension PureYAML.Parsing.LibYAML.Scanner { /// Dispatch the block-structure indicators and plain scalars; unported /// constructs throw an explicit error. private mutating func dispatchIndicator(_ character: Unicode.Scalar) throws { - if let unported = unportedConstruct(character) { - throw notPorted(unported) - } if try fetchFlowToken(character) { return } @@ -130,6 +127,7 @@ extension PureYAML.Parsing.LibYAML.Scanner { case "\"": try fetchFlowScalar(single: false) case "&": try fetchAnchor(isAnchor: true) case "*": try fetchAnchor(isAnchor: false) + case "!": try fetchTag() case "|" where flowLevel == 0: try fetchBlockScalar(literal: true) case ">" where flowLevel == 0: try fetchBlockScalar(literal: false) default: return false @@ -137,6 +135,15 @@ extension PureYAML.Parsing.LibYAML.Scanner { return true } + /// Port of `yaml_parser_fetch_tag`. + private mutating func fetchTag() throws { + mark = reader.mark + try saveSimpleKey() + simpleKeyAllowed = false + let token = try scanTag() + indentEngine.enqueue(token) + } + /// Port of `yaml_parser_fetch_block_scalar`. private mutating func fetchBlockScalar(literal: Bool) throws { try removeSimpleKey() @@ -163,14 +170,6 @@ extension PureYAML.Parsing.LibYAML.Scanner { indentEngine.enqueue(token) } - /// The constructs whose fetchers are not yet ported, named for the error. - private func unportedConstruct(_ character: Unicode.Scalar) -> String? { - switch character { - case "!": "tag" - default: nil - } - } - /// Port of `yaml_parser_fetch_flow_collection_start`. private mutating func fetchFlowCollectionStart(_ type: PureYAML.Parsing.LibYAML.TokenType) { // saveSimpleKey cannot throw in flow: a required key needs block context. diff --git a/Sources/Parsing/LibYAML.Scanner.Tag.swift b/Sources/Parsing/LibYAML.Scanner.Tag.swift new file mode 100644 index 0000000..a3fd681 --- /dev/null +++ b/Sources/Parsing/LibYAML.Scanner.Tag.swift @@ -0,0 +1,155 @@ +extension PureYAML.Parsing.LibYAML.Scanner { + private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass + + /// Port of `yaml_parser_scan_tag`. Scans a tag in verbatim `!`, shorthand + /// `!handle!suffix`, or `!suffix` form, producing the handle and suffix; the + /// parser resolves them against the tag directives. + mutating func scanTag() throws -> PureYAML.Parsing.LibYAML.Token { + let startMark = reader.mark + var handle = "" + var suffix = "" + if reader.peek(1) == "<" { + reader.skip() + reader.skip() + suffix = try scanTagURI(uriChar: true, directive: false, head: "", startMark: startMark) + guard reader.peek() == ">" else { + throw tagError("did not find the expected '>'", startMark) + } + reader.skip() + } else { + handle = try scanTagHandle(directive: false, startMark: startMark) + if handle.first == "!", handle.count > 1, handle.last == "!" { + suffix = try scanTagURI(uriChar: false, directive: false, head: "", startMark: startMark) + } else { + suffix = try scanTagURI(uriChar: false, directive: false, head: handle, startMark: startMark) + handle = "!" + if suffix.isEmpty { + swap(&handle, &suffix) + } + } + } + guard Classify.isBlankZ(reader.peek()) || (flowLevel > 0 && reader.peek() == ",") else { + throw tagError("did not find expected whitespace or line break", startMark) + } + return PureYAML.Parsing.LibYAML.Token(type: .tag(handle: handle, suffix: suffix), startMark: startMark, endMark: reader.mark) + } + + /// Port of `yaml_parser_scan_tag_handle`. A handle is `!`, optionally followed + /// by alphanumeric characters and a closing `!`. + mutating func scanTagHandle(directive: Bool, startMark: PureYAML.Parsing.Mark) throws -> String { + guard reader.peek() == "!" else { + throw tagError("did not find expected '!'", startMark) + } + var handle = "!" + reader.skip() + while let character = reader.peek(), Classify.isAlpha(character) { + handle.unicodeScalars.append(character) + reader.skip() + } + if reader.peek() == "!" { + handle.unicodeScalars.append("!") + reader.skip() + } else if directive, handle != "!" { + throw tagError("did not find expected '!'", startMark) + } + return handle + } + + /// Port of `yaml_parser_scan_tag_uri`. Consumes the URI character set (plus the + /// flow indicators inside a verbatim tag), decoding `%`-escapes. + mutating func scanTagURI(uriChar: Bool, directive: Bool, head: String, startMark: PureYAML.Parsing.Mark) throws -> String { + var string = head.count > 1 ? String(head.dropFirst()) : "" + var length = head.isEmpty ? 0 : head.count + while let character = reader.peek(), isURICharacter(character, uriChar: uriChar) { + if character == "%" { + try scanURIEscapes(directive: directive, startMark: startMark, into: &string) + } else { + string.unicodeScalars.append(character) + reader.skip() + } + length += 1 + } + guard length > 0 else { + throw tagError("did not find expected tag URI", startMark) + } + return string + } + + /// Port of `yaml_parser_scan_uri_escapes`. Decodes one UTF-8 character from a + /// run of `%XX` octets. + private mutating func scanURIEscapes(directive _: Bool, startMark: PureYAML.Parsing.Mark, into string: inout String) throws { + var bytes: [UInt8] = [] + var width = 0 + repeat { + guard reader.peek() == "%", let high = reader.peek(1), Classify.isHex(high), let low = reader.peek(2), Classify.isHex(low) else { + throw tagError("did not find URI escaped octet", startMark) + } + let octet = UInt8((hexDigit(high) << 4) + hexDigit(low)) + if width == 0 { + width = utf8Width(of: octet) + guard width > 0 else { + throw tagError("found an incorrect leading UTF-8 octet", startMark) + } + } else if octet & 0xC0 != 0x80 { + throw tagError("found an incorrect trailing UTF-8 octet", startMark) + } + bytes.append(octet) + reader.skip() + reader.skip() + reader.skip() + width -= 1 + } while width > 0 + guard let scalar = Unicode.Scalar(decodeUTF8(bytes)) else { + throw tagError("found invalid UTF-8 in a URI escape", startMark) + } + string.unicodeScalars.append(scalar) + } + + /// Assemble a code point from the validated 1-to-4 UTF-8 octets of one escape. + private func decodeUTF8(_ bytes: [UInt8]) -> UInt32 { + switch bytes.count { + case 1: + UInt32(bytes[0]) + case 2: + (UInt32(bytes[0] & 0x1F) << 6) | UInt32(bytes[1] & 0x3F) + case 3: + (UInt32(bytes[0] & 0x0F) << 12) | (UInt32(bytes[1] & 0x3F) << 6) | UInt32(bytes[2] & 0x3F) + default: + (UInt32(bytes[0] & 0x07) << 18) | (UInt32(bytes[1] & 0x3F) << 12) | (UInt32(bytes[2] & 0x3F) << 6) | UInt32(bytes[3] & 0x3F) + } + } + + private func utf8Width(of octet: UInt8) -> Int { + if octet & 0x80 == 0x00 { return 1 } + if octet & 0xE0 == 0xC0 { return 2 } + if octet & 0xF0 == 0xE0 { return 3 } + if octet & 0xF8 == 0xF0 { return 4 } + return 0 + } + + private func isURICharacter(_ scalar: Unicode.Scalar, uriChar: Bool) -> Bool { + if Classify.isAlpha(scalar) { + return true + } + switch scalar { + case ";", "/", "?", ":", "@", "&", "=", "+", "$", ".", "%", "!", "~", "*", "'", "(", ")": + return true + case ",", "[", "]": + return uriChar + default: + return false + } + } + + private func hexDigit(_ scalar: Unicode.Scalar) -> Int { + switch scalar { + case "0" ... "9": Int(scalar.value - Unicode.Scalar("0").value) + case "A" ... "F": Int(scalar.value - Unicode.Scalar("A").value) + 10 + default: Int(scalar.value - Unicode.Scalar("a").value) + 10 + } + } + + private func tagError(_ problem: String, _ mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError { + PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a tag", problem: problem, mark: mark) + } +} diff --git a/Sources/Parsing/LibYAML.TokenType.swift b/Sources/Parsing/LibYAML.TokenType.swift index 5c28cf1..32ed4ac 100644 --- a/Sources/Parsing/LibYAML.TokenType.swift +++ b/Sources/Parsing/LibYAML.TokenType.swift @@ -20,6 +20,7 @@ extension PureYAML.Parsing.LibYAML { case value case anchor(name: String) case alias(name: String) + case tag(handle: String, suffix: String) case scalar(value: String, style: PureYAML.Parsing.ScalarStyle) } } diff --git a/Tests/LibYAMLScannerFetchTests.swift b/Tests/LibYAMLScannerFetchTests.swift index f2f4678..b6fce0e 100644 --- a/Tests/LibYAMLScannerFetchTests.swift +++ b/Tests/LibYAMLScannerFetchTests.swift @@ -204,4 +204,37 @@ struct LibYAMLScannerFetchTests { .streamEnd, ]) } + + @Test("a secondary-handle shorthand tag") + func test_shorthandTag() throws { + let tokens = try tokenize("!!str x\n") + #expect(tokens == [ + .streamStart, + .tag(handle: "!!", suffix: "str"), + .scalar(value: "x", style: .plain), + .streamEnd, + ]) + } + + @Test("a local tag uses the primary handle") + func test_localTag() throws { + let tokens = try tokenize("!local x\n") + #expect(tokens == [ + .streamStart, + .tag(handle: "!", suffix: "local"), + .scalar(value: "x", style: .plain), + .streamEnd, + ]) + } + + @Test("a verbatim tag has an empty handle") + func test_verbatimTag() throws { + let tokens = try tokenize("! x\n") + #expect(tokens == [ + .streamStart, + .tag(handle: "", suffix: "tag:yaml.org,2002:str"), + .scalar(value: "x", style: .plain), + .streamEnd, + ]) + } }