feat(parsing): port libyaml tag scanner to Swift
Stage 3 of the faithful libyaml scanner port, continued. Translates fetch_tag, scan_tag, scan_tag_handle, scan_tag_uri, and scan_uri_escapes: verbatim (!<uri>), shorthand (!handle!suffix), and local (!suffix) tags, producing the handle and suffix for the parser to resolve. URI %-escapes are decoded by collecting the UTF-8 octets and decoding them as a unit. Unit-tested for a secondary-handle shorthand tag, a local tag, and a verbatim tag. Only directives (%YAML/%TAG) remain unported in the scanner. 317 tests, no regression.
This commit is contained in:
parent
7918eefe14
commit
ed1904534c
@ -87,9 +87,6 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
/// Dispatch the block-structure indicators and plain scalars; unported
|
||||
/// constructs throw an explicit error.
|
||||
private mutating func dispatchIndicator(_ character: Unicode.Scalar) throws {
|
||||
if let unported = unportedConstruct(character) {
|
||||
throw notPorted(unported)
|
||||
}
|
||||
if try fetchFlowToken(character) {
|
||||
return
|
||||
}
|
||||
@ -130,6 +127,7 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
case "\"": try fetchFlowScalar(single: false)
|
||||
case "&": try fetchAnchor(isAnchor: true)
|
||||
case "*": try fetchAnchor(isAnchor: false)
|
||||
case "!": try fetchTag()
|
||||
case "|" where flowLevel == 0: try fetchBlockScalar(literal: true)
|
||||
case ">" where flowLevel == 0: try fetchBlockScalar(literal: false)
|
||||
default: return false
|
||||
@ -137,6 +135,15 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
return true
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_fetch_tag`.
|
||||
private mutating func fetchTag() throws {
|
||||
mark = reader.mark
|
||||
try saveSimpleKey()
|
||||
simpleKeyAllowed = false
|
||||
let token = try scanTag()
|
||||
indentEngine.enqueue(token)
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_fetch_block_scalar`.
|
||||
private mutating func fetchBlockScalar(literal: Bool) throws {
|
||||
try removeSimpleKey()
|
||||
@ -163,14 +170,6 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
indentEngine.enqueue(token)
|
||||
}
|
||||
|
||||
/// The constructs whose fetchers are not yet ported, named for the error.
|
||||
private func unportedConstruct(_ character: Unicode.Scalar) -> String? {
|
||||
switch character {
|
||||
case "!": "tag"
|
||||
default: nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_fetch_flow_collection_start`.
|
||||
private mutating func fetchFlowCollectionStart(_ type: PureYAML.Parsing.LibYAML.TokenType) {
|
||||
// saveSimpleKey cannot throw in flow: a required key needs block context.
|
||||
|
||||
155
Sources/Parsing/LibYAML.Scanner.Tag.swift
Normal file
155
Sources/Parsing/LibYAML.Scanner.Tag.swift
Normal file
@ -0,0 +1,155 @@
|
||||
extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
private typealias Classify = PureYAML.Parsing.LibYAML.CharacterClass
|
||||
|
||||
/// Port of `yaml_parser_scan_tag`. Scans a tag in verbatim `!<uri>`, shorthand
|
||||
/// `!handle!suffix`, or `!suffix` form, producing the handle and suffix; the
|
||||
/// parser resolves them against the tag directives.
|
||||
mutating func scanTag() throws -> PureYAML.Parsing.LibYAML.Token {
|
||||
let startMark = reader.mark
|
||||
var handle = ""
|
||||
var suffix = ""
|
||||
if reader.peek(1) == "<" {
|
||||
reader.skip()
|
||||
reader.skip()
|
||||
suffix = try scanTagURI(uriChar: true, directive: false, head: "", startMark: startMark)
|
||||
guard reader.peek() == ">" else {
|
||||
throw tagError("did not find the expected '>'", startMark)
|
||||
}
|
||||
reader.skip()
|
||||
} else {
|
||||
handle = try scanTagHandle(directive: false, startMark: startMark)
|
||||
if handle.first == "!", handle.count > 1, handle.last == "!" {
|
||||
suffix = try scanTagURI(uriChar: false, directive: false, head: "", startMark: startMark)
|
||||
} else {
|
||||
suffix = try scanTagURI(uriChar: false, directive: false, head: handle, startMark: startMark)
|
||||
handle = "!"
|
||||
if suffix.isEmpty {
|
||||
swap(&handle, &suffix)
|
||||
}
|
||||
}
|
||||
}
|
||||
guard Classify.isBlankZ(reader.peek()) || (flowLevel > 0 && reader.peek() == ",") else {
|
||||
throw tagError("did not find expected whitespace or line break", startMark)
|
||||
}
|
||||
return PureYAML.Parsing.LibYAML.Token(type: .tag(handle: handle, suffix: suffix), startMark: startMark, endMark: reader.mark)
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_scan_tag_handle`. A handle is `!`, optionally followed
|
||||
/// by alphanumeric characters and a closing `!`.
|
||||
mutating func scanTagHandle(directive: Bool, startMark: PureYAML.Parsing.Mark) throws -> String {
|
||||
guard reader.peek() == "!" else {
|
||||
throw tagError("did not find expected '!'", startMark)
|
||||
}
|
||||
var handle = "!"
|
||||
reader.skip()
|
||||
while let character = reader.peek(), Classify.isAlpha(character) {
|
||||
handle.unicodeScalars.append(character)
|
||||
reader.skip()
|
||||
}
|
||||
if reader.peek() == "!" {
|
||||
handle.unicodeScalars.append("!")
|
||||
reader.skip()
|
||||
} else if directive, handle != "!" {
|
||||
throw tagError("did not find expected '!'", startMark)
|
||||
}
|
||||
return handle
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_scan_tag_uri`. Consumes the URI character set (plus the
|
||||
/// flow indicators inside a verbatim tag), decoding `%`-escapes.
|
||||
mutating func scanTagURI(uriChar: Bool, directive: Bool, head: String, startMark: PureYAML.Parsing.Mark) throws -> String {
|
||||
var string = head.count > 1 ? String(head.dropFirst()) : ""
|
||||
var length = head.isEmpty ? 0 : head.count
|
||||
while let character = reader.peek(), isURICharacter(character, uriChar: uriChar) {
|
||||
if character == "%" {
|
||||
try scanURIEscapes(directive: directive, startMark: startMark, into: &string)
|
||||
} else {
|
||||
string.unicodeScalars.append(character)
|
||||
reader.skip()
|
||||
}
|
||||
length += 1
|
||||
}
|
||||
guard length > 0 else {
|
||||
throw tagError("did not find expected tag URI", startMark)
|
||||
}
|
||||
return string
|
||||
}
|
||||
|
||||
/// Port of `yaml_parser_scan_uri_escapes`. Decodes one UTF-8 character from a
|
||||
/// run of `%XX` octets.
|
||||
private mutating func scanURIEscapes(directive _: Bool, startMark: PureYAML.Parsing.Mark, into string: inout String) throws {
|
||||
var bytes: [UInt8] = []
|
||||
var width = 0
|
||||
repeat {
|
||||
guard reader.peek() == "%", let high = reader.peek(1), Classify.isHex(high), let low = reader.peek(2), Classify.isHex(low) else {
|
||||
throw tagError("did not find URI escaped octet", startMark)
|
||||
}
|
||||
let octet = UInt8((hexDigit(high) << 4) + hexDigit(low))
|
||||
if width == 0 {
|
||||
width = utf8Width(of: octet)
|
||||
guard width > 0 else {
|
||||
throw tagError("found an incorrect leading UTF-8 octet", startMark)
|
||||
}
|
||||
} else if octet & 0xC0 != 0x80 {
|
||||
throw tagError("found an incorrect trailing UTF-8 octet", startMark)
|
||||
}
|
||||
bytes.append(octet)
|
||||
reader.skip()
|
||||
reader.skip()
|
||||
reader.skip()
|
||||
width -= 1
|
||||
} while width > 0
|
||||
guard let scalar = Unicode.Scalar(decodeUTF8(bytes)) else {
|
||||
throw tagError("found invalid UTF-8 in a URI escape", startMark)
|
||||
}
|
||||
string.unicodeScalars.append(scalar)
|
||||
}
|
||||
|
||||
/// Assemble a code point from the validated 1-to-4 UTF-8 octets of one escape.
|
||||
private func decodeUTF8(_ bytes: [UInt8]) -> UInt32 {
|
||||
switch bytes.count {
|
||||
case 1:
|
||||
UInt32(bytes[0])
|
||||
case 2:
|
||||
(UInt32(bytes[0] & 0x1F) << 6) | UInt32(bytes[1] & 0x3F)
|
||||
case 3:
|
||||
(UInt32(bytes[0] & 0x0F) << 12) | (UInt32(bytes[1] & 0x3F) << 6) | UInt32(bytes[2] & 0x3F)
|
||||
default:
|
||||
(UInt32(bytes[0] & 0x07) << 18) | (UInt32(bytes[1] & 0x3F) << 12) | (UInt32(bytes[2] & 0x3F) << 6) | UInt32(bytes[3] & 0x3F)
|
||||
}
|
||||
}
|
||||
|
||||
private func utf8Width(of octet: UInt8) -> Int {
|
||||
if octet & 0x80 == 0x00 { return 1 }
|
||||
if octet & 0xE0 == 0xC0 { return 2 }
|
||||
if octet & 0xF0 == 0xE0 { return 3 }
|
||||
if octet & 0xF8 == 0xF0 { return 4 }
|
||||
return 0
|
||||
}
|
||||
|
||||
private func isURICharacter(_ scalar: Unicode.Scalar, uriChar: Bool) -> Bool {
|
||||
if Classify.isAlpha(scalar) {
|
||||
return true
|
||||
}
|
||||
switch scalar {
|
||||
case ";", "/", "?", ":", "@", "&", "=", "+", "$", ".", "%", "!", "~", "*", "'", "(", ")":
|
||||
return true
|
||||
case ",", "[", "]":
|
||||
return uriChar
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
private func hexDigit(_ scalar: Unicode.Scalar) -> Int {
|
||||
switch scalar {
|
||||
case "0" ... "9": Int(scalar.value - Unicode.Scalar("0").value)
|
||||
case "A" ... "F": Int(scalar.value - Unicode.Scalar("A").value) + 10
|
||||
default: Int(scalar.value - Unicode.Scalar("a").value) + 10
|
||||
}
|
||||
}
|
||||
|
||||
private func tagError(_ problem: String, _ mark: PureYAML.Parsing.Mark) -> PureYAML.Parsing.LibYAML.ScannerError {
|
||||
PureYAML.Parsing.LibYAML.ScannerError(context: "while scanning a tag", problem: problem, mark: mark)
|
||||
}
|
||||
}
|
||||
@ -20,6 +20,7 @@ extension PureYAML.Parsing.LibYAML {
|
||||
case value
|
||||
case anchor(name: String)
|
||||
case alias(name: String)
|
||||
case tag(handle: String, suffix: String)
|
||||
case scalar(value: String, style: PureYAML.Parsing.ScalarStyle)
|
||||
}
|
||||
}
|
||||
|
||||
@ -204,4 +204,37 @@ struct LibYAMLScannerFetchTests {
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
|
||||
@Test("a secondary-handle shorthand tag")
|
||||
func test_shorthandTag() throws {
|
||||
let tokens = try tokenize("!!str x\n")
|
||||
#expect(tokens == [
|
||||
.streamStart,
|
||||
.tag(handle: "!!", suffix: "str"),
|
||||
.scalar(value: "x", style: .plain),
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
|
||||
@Test("a local tag uses the primary handle")
|
||||
func test_localTag() throws {
|
||||
let tokens = try tokenize("!local x\n")
|
||||
#expect(tokens == [
|
||||
.streamStart,
|
||||
.tag(handle: "!", suffix: "local"),
|
||||
.scalar(value: "x", style: .plain),
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
|
||||
@Test("a verbatim tag has an empty handle")
|
||||
func test_verbatimTag() throws {
|
||||
let tokens = try tokenize("!<tag:yaml.org,2002:str> x\n")
|
||||
#expect(tokens == [
|
||||
.streamStart,
|
||||
.tag(handle: "", suffix: "tag:yaml.org,2002:str"),
|
||||
.scalar(value: "x", style: .plain),
|
||||
.streamEnd,
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user