feat(parsing): port libyaml tag scanner to Swift

Stage 3 of the faithful libyaml scanner port, continued. Translates fetch_tag, scan_tag, scan_tag_handle, scan_tag_uri, and scan_uri_escapes: verbatim (!<uri>), shorthand (!handle!suffix), and local (!suffix) tags, producing the handle and suffix for the parser to resolve. URI %-escapes are decoded by collecting the UTF-8 octets and decoding them as a unit. Unit-tested for a secondary-handle shorthand tag, a local tag, and a verbatim tag. Only directives (%YAML/%TAG) remain unported in the scanner. 317 tests, no regression.
This commit is contained in:
Mihaela Mihaljevic 2026-06-24 16:46:28 +02:00
parent 7918eefe14
commit ed1904534c
4 changed files with 199 additions and 11 deletions

View File

@ -87,9 +87,6 @@ extension PureYAML.Parsing.LibYAML.Scanner {
/// Dispatch the block-structure indicators and plain scalars; unported
/// constructs throw an explicit error.
private mutating func dispatchIndicator(_ character: Unicode.Scalar) throws {
if let unported = unportedConstruct(character) {
throw notPorted(unported)
}
if try fetchFlowToken(character) {
return
}
@ -130,6 +127,7 @@ extension PureYAML.Parsing.LibYAML.Scanner {
case "\"": try fetchFlowScalar(single: false)
case "&": try fetchAnchor(isAnchor: true)
case "*": try fetchAnchor(isAnchor: false)
case "!": try fetchTag()
case "|" where flowLevel == 0: try fetchBlockScalar(literal: true)
case ">" where flowLevel == 0: try fetchBlockScalar(literal: false)
default: return false
@ -137,6 +135,15 @@ extension PureYAML.Parsing.LibYAML.Scanner {
return true
}
/// Port of `yaml_parser_fetch_tag`.
private mutating func fetchTag() throws {
mark = reader.mark
try saveSimpleKey()
simpleKeyAllowed = false
let token = try scanTag()
indentEngine.enqueue(token)
}
/// Port of `yaml_parser_fetch_block_scalar`.
private mutating func fetchBlockScalar(literal: Bool) throws {
try removeSimpleKey()
@ -163,14 +170,6 @@ extension PureYAML.Parsing.LibYAML.Scanner {
indentEngine.enqueue(token)
}
/// The constructs whose fetchers are not yet ported, named for the error.
private func unportedConstruct(_ character: Unicode.Scalar) -> String? {
switch character {
case "!": "tag"
default: nil
}
}
/// Port of `yaml_parser_fetch_flow_collection_start`.
private mutating func fetchFlowCollectionStart(_ type: PureYAML.Parsing.LibYAML.TokenType) {
// saveSimpleKey cannot throw in flow: a required key needs block context.

View File

@ -0,0 +1,155 @@
extension PureYAML.Parsing.LibYAML.Scanner {
private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
/// Port of `yaml_parser_scan_tag`. Scans a tag in verbatim `!<uri>`, shorthand
/// `!handle!suffix`, or `!suffix` form, producing the handle and suffix; the
/// parser resolves them against the tag directives.
mutating func scanTag() throws -> PureYAML.Parsing.LibYAML.Token {
let startMark = reader.mark
var handle = ""
var suffix = ""
if reader.peek(1) == "<" {
reader.skip()
reader.skip()
suffix = try scanTagURI(uriChar: true, directive: false, head: "", startMark: startMark)
guard reader.peek() == ">" else {
throw tagError("did not find the expected '>'", startMark)
}
reader.skip()
} else {
handle = try scanTagHandle(directive: false, startMark: startMark)
if handle.first == "!", handle.count > 1, handle.last == "!" {
suffix = try scanTagURI(uriChar: false, directive: false, head: "", startMark: startMark)
} else {
suffix = try scanTagURI(uriChar: false, directive: false, head: handle, startMark: startMark)
handle = "!"
if suffix.isEmpty {
swap(&handle, &suffix)
}
}
}
guard Classify.isBlankZ(reader.peek()) || (flowLevel > 0 && reader.peek() == ",") else {
throw tagError("did not find expected whitespace or line break", startMark)
}
return PureYAML.Parsing.LibYAML.Token(type: .tag(handle: handle, suffix: suffix), startMark: startMark, endMark: reader.mark)
}
/// Port of `yaml_parser_scan_tag_handle`. A handle is `!`, optionally followed
/// by alphanumeric characters and a closing `!`.
mutating func scanTagHandle(directive: Bool, startMark: PureYAML.Parsing.Mark) throws -> String {
guard reader.peek() == "!" else {
throw tagError("did not find expected '!'", startMark)
}
var handle = "!"
reader.skip()
while let character = reader.peek(), Classify.isAlpha(character) {
handle.unicodeScalars.append(character)
reader.skip()
}
if reader.peek() == "!" {
handle.unicodeScalars.append("!")
reader.skip()
} else if directive, handle != "!" {
throw tagError("did not find expected '!'", startMark)
}
return handle
}
/// Port of `yaml_parser_scan_tag_uri`. Consumes the URI character set (plus the
/// flow indicators inside a verbatim tag), decoding `%`-escapes.
mutating func scanTagURI(uriChar: Bool, directive: Bool, head: String, startMark: PureYAML.Parsing.Mark) throws -> String {
var string = head.count > 1 ? String(head.dropFirst()) : ""
var length = head.isEmpty ? 0 : head.count
while let character = reader.peek(), isURICharacter(character, uriChar: uriChar) {
if character == "%" {
try scanURIEscapes(directive: directive, startMark: startMark, into: &string)
} else {
string.unicodeScalars.append(character)
reader.skip()
}
length += 1
}
guard length > 0 else {
throw tagError("did not find expected tag URI", startMark)
}
return string
}
/// Port of `yaml_parser_scan_uri_escapes`. Decodes one UTF-8 character from a
/// run of `%XX` octets.
private mutating func scanURIEscapes(directive _: Bool, startMark: PureYAML.Parsing.Mark, into string: inout String) throws {
var bytes: [UInt8] = []
var width = 0
repeat {
guard reader.peek() == "%", let high = reader.peek(1), Classify.isHex(high), let low = reader.peek(2), Classify.isHex(low) else {
throw tagError("did not find URI escaped octet", startMark)
}
let octet = UInt8((hexDigit(high) << 4) + hexDigit(low))
if width == 0 {
width = utf8Width(of: octet)
guard width > 0 else {
throw tagError("found an incorrect leading UTF-8 octet", startMark)
}
} else if octet & 0xC0 != 0x80 {
throw tagError("found an incorrect trailing UTF-8 octet", startMark)
}
bytes.append(octet)
reader.skip()
reader.skip()
reader.skip()
width -= 1
} while width > 0
guard let scalar = Unicode.Scalar(decodeUTF8(bytes)) else {
throw tagError("found invalid UTF-8 in a URI escape", startMark)
}
string.unicodeScalars.append(scalar)
}
/// Assemble a code point from the validated 1-to-4 UTF-8 octets of one escape.
private func decodeUTF8(_ bytes: [UInt8]) -> UInt32 {
switch bytes.count {
case 1:
UInt32(bytes[0])
case 2:
(UInt32(bytes[0] & 0x1F) << 6) | UInt32(bytes[1] & 0x3F)
case 3:
(UInt32(bytes[0] & 0x0F) << 12) | (UInt32(bytes[1] & 0x3F) << 6) | UInt32(bytes[2] & 0x3F)
default:
(UInt32(bytes[0] & 0x07) << 18) | (UInt32(bytes[1] & 0x3F) << 12) | (UInt32(bytes[2] & 0x3F) << 6) | UInt32(bytes[3] & 0x3F)
}
}
private func utf8Width(of octet: UInt8) -> Int {
if octet & 0x80 == 0x00 { return 1 }
if octet & 0xE0 == 0xC0 { return 2 }
if octet & 0xF0 == 0xE0 { return 3 }
if octet & 0xF8 == 0xF0 { return 4 }
return 0
}
private func isURICharacter(_ scalar: Unicode.Scalar, uriChar: Bool) -> Bool {
if Classify.isAlpha(scalar) {
return true
}
switch scalar {
case ";", "/", "?", ":", "@", "&", "=", "+", "$", ".", "%", "!", "~", "*", "'", "(", ")":
return true
case ",", "[", "]":
return uriChar
default:
return false
}
}
private func hexDigit(_ scalar: Unicode.Scalar) -> Int {
switch scalar {
case "0" ... "9": Int(scalar.value - Unicode.Scalar("0").value)
case "A" ... "F": Int(scalar.value - Unicode.Scalar("A").value) + 10
default: Int(scalar.value - Unicode.Scalar("a").value) + 10
}
}
private func tagError(_ problem: String, _ mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError {
PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a tag", problem: problem, mark: mark)
}
}

View File

@ -20,6 +20,7 @@ extension PureYAML.Parsing.LibYAML {
case value
case anchor(name: String)
case alias(name: String)
case tag(handle: String, suffix: String)
case scalar(value: String, style: PureYAML.Parsing.ScalarStyle)
}
}

View File

@ -204,4 +204,37 @@ struct LibYAMLScannerFetchTests {
.streamEnd,
])
}
@Test("a secondary-handle shorthand tag")
func test_shorthandTag() throws {
let tokens = try tokenize("!!str x\n")
#expect(tokens == [
.streamStart,
.tag(handle: "!!", suffix: "str"),
.scalar(value: "x", style: .plain),
.streamEnd,
])
}
@Test("a local tag uses the primary handle")
func test_localTag() throws {
let tokens = try tokenize("!local x\n")
#expect(tokens == [
.streamStart,
.tag(handle: "!", suffix: "local"),
.scalar(value: "x", style: .plain),
.streamEnd,
])
}
@Test("a verbatim tag has an empty handle")
func test_verbatimTag() throws {
let tokens = try tokenize("!<tag:yaml.org,2002:str> x\n")
#expect(tokens == [
.streamStart,
.tag(handle: "", suffix: "tag:yaml.org,2002:str"),
.scalar(value: "x", style: .plain),
.streamEnd,
])
}
}