diff --git a/CHANGELOG.md b/CHANGELOG.md index b5ec8cd..4baf415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- A string no longer changes on the way through YAML or JSON. `PureYAML.dump` wrote control characters, DEL, the C1 controls, U+2028, U+2029, U+FEFF, U+FFFE and + U+FFFF as they are, where the reader refused them (`invalid control character`), folded U+0085, U+2028 and U+2029 to a space without a word, or dropped a leading + U+FEFF as a byte order mark. They are now written as the escape that stands for each (`\x01`, `\N`, `\L`, `\P`, ``) and are never written plain or in a + literal block. `PureYAML.JSON.Writer` escapes the same characters (`\u007f`, `
`, and so on), which RFC 8259 allows and a YAML reader needs. +- A JSON text that holds one of those characters as it is (RFC 8259 allows it) was read as another string, or refused. `PureYAML.JSON.parse(_:)` and `PureYAML.JSON.parseLocated(_:)` read JSON and keep + every character of its strings, and `PureYAML.JSON.Convert.convert(_:from:to:pretty:)` takes the format of the source (`from:`, default nil: the source is read as YAML, as + before) and reads a JSON source with it. + +### Added + +- `PureYAML.JSON.parse(_:)`, `PureYAML.JSON.parseLocated(_:)` (every position is where it is written) and the `from:` parameter of `PureYAML.JSON.Convert.convert`. + ## [0.3.0] - 2026-11-14 ### Changed diff --git a/Sources/PureYAMLJSON/JSON.Convert.swift b/Sources/PureYAMLJSON/JSON.Convert.swift index e30a85e..e288063 100644 --- a/Sources/PureYAMLJSON/JSON.Convert.swift +++ b/Sources/PureYAMLJSON/JSON.Convert.swift @@ -16,11 +16,12 @@ public extension PureYAML.JSON { /// value may use a non-string key or a non-finite number). "To YAML" still /// rejects duplicate keys, which are legal in JSON but not in YAML. enum Convert { - /// Convert source text to the target format, validated end to end. - public static func convert(_ source: String, to target: Format, pretty: Bool = true) throws -> String { - let value = try parse(source) + /// Convert source text to the target format, validated end to end. When the format of the source is known it is given as `from`, and a source that is JSON + /// is read as JSON (``PureYAML/JSON/parse(_:)``), which keeps every character of its strings; without it the source is read by the YAML reader as it is. + public static func convert(_ source: String, from sourceFormat: Format? = nil, to target: Format, pretty: Bool = true) throws -> String { + let value = try parse(source, as: sourceFormat) let output = try emit(value, as: target, pretty: pretty) - guard try parse(output) == value else { + guard try parse(output, as: target) == value else { throw Failure.roundTripMismatch(target) } return output @@ -53,9 +54,9 @@ public extension PureYAML.JSON { } } - private static func parse(_ text: String) throws -> PureYAML.Model.Value { + private static func parse(_ text: String, as format: Format?) throws -> PureYAML.Model.Value { do { - return try PureYAML.parse(text) + return try format == .json ? PureYAML.JSON.parse(text) : PureYAML.parse(text) } catch { throw Failure.sourceInvalid(String(describing: error)) } diff --git a/Sources/PureYAMLJSON/JSON.Parse.swift b/Sources/PureYAMLJSON/JSON.Parse.swift new file mode 100644 index 0000000..cd6b9a9 --- /dev/null +++ b/Sources/PureYAMLJSON/JSON.Parse.swift @@ -0,0 +1,21 @@ +import PureYAML + +public extension PureYAML.JSON { + /// Reads text that is JSON into the shared value tree. + /// + /// JSON is a subset of YAML 1.2 apart from a few characters that RFC 8259 lets a string hold as they are and the YAML reader does not: it folds U+0085, U+2028 and + /// U+2029 as line breaks, it refuses U+007F, the C1 controls, U+FFFE and U+FFFF, and so a string that holds one comes back from ``PureYAML/parse(_:)`` as another + /// string or not at all. Here each such character inside a string is held back from the reader and given back in the value, so the string is the one the JSON + /// means. Everything else is read by the YAML reader as it is, and what is not JSON is refused by it as before. + static func parse(_ text: String) throws -> PureYAML.Model.Value { + let source = try Source(text) + return try source.restoring(PureYAML.parse(source.masked)) + } + + /// As ``parse(_:)``, with the span of every node. The spans are those of the text as it is: a held back character is one scalar in the text the reader sees, as it is + /// in `text`, so no position moves. + static func parseLocated(_ text: String) throws -> PureYAML.Model.LocatedValue { + let source = try Source(text) + return try source.restoring(PureYAML.parseLocated(source.masked)) + } +} diff --git a/Sources/PureYAMLJSON/JSON.Source.swift b/Sources/PureYAMLJSON/JSON.Source.swift new file mode 100644 index 0000000..d5f3550 --- /dev/null +++ b/Sources/PureYAMLJSON/JSON.Source.swift @@ -0,0 +1,105 @@ +import PureYAML + +extension PureYAML.JSON { + /// JSON text with the characters the YAML reader cannot be given held back. + /// + /// Each U+007F to U+009F, U+2028, U+2029, U+FFFE and U+FFFF that is inside a string is replaced, one scalar for one scalar, by a private-use character that is not + /// in the text, so that the reader sees a plain character and every position of the text is the position it had. The replacement is undone in the value that + /// comes out of the reader. A private-use character that the text itself holds is never used as a replacement, so what comes back is never mistaken for it. + struct Source { + let masked: String + /// What each replacement stands for. + private let originals: [Unicode.Scalar: Unicode.Scalar] + + init(_ text: String) throws { + let present = Set(text.unicodeScalars.filter { Self.privateUse.contains($0.value) }) + var free = Self.privateUse.lazy.compactMap(Unicode.Scalar.init).filter { !present.contains($0) }.makeIterator() + var replacement: [Unicode.Scalar: Unicode.Scalar] = [:] + var originals: [Unicode.Scalar: Unicode.Scalar] = [:] + var output = String.UnicodeScalarView() + var inString = false + var escaped = false + for scalar in text.unicodeScalars { + guard inString else { + inString = scalar == "\"" + output.append(scalar) + continue + } + if escaped { + escaped = false + } else if scalar == "\\" { + escaped = true + } else if scalar == "\"" { + inString = false + } else if Self.isHeldBack(scalar) { + if replacement[scalar] == nil { + guard let next = free.next() else { + throw PureYAML.JSON.Convert.Failure.sourceInvalid("the text holds too many private-use characters to read it as JSON") + } + replacement[scalar] = next + originals[next] = scalar + } + output.append(replacement[scalar] ?? scalar) + continue + } + output.append(scalar) + } + masked = String(output) + self.originals = originals + } + + private static let privateUse: ClosedRange = 0xE000 ... 0xF8FF + + /// The characters RFC 8259 lets a string hold as they are and the YAML reader folds or refuses. + static func isHeldBack(_ scalar: Unicode.Scalar) -> Bool { + (0x7F ... 0x9F).contains(scalar.value) || [0x2028, 0x2029, 0xFFFE, 0xFFFF].contains(scalar.value) + } + + func restoring(_ text: String) -> String { + guard !originals.isEmpty else { return text } + var output = String.UnicodeScalarView() + for scalar in text.unicodeScalars { + output.append(originals[scalar] ?? scalar) + } + return String(output) + } + + func restoring(_ value: PureYAML.Model.Value) -> PureYAML.Model.Value { + guard !originals.isEmpty else { return value } + switch value { + case .null, .bool, .int, .double: + return value + case let .string(text): + return .string(restoring(text)) + case let .sequence(items): + return .sequence(items.map(restoring)) + case let .mapping(mapping): + return .mapping(restoring(mapping)) + } + } + + func restoring(_ located: PureYAML.Model.LocatedValue) -> PureYAML.Model.LocatedValue { + guard !originals.isEmpty else { return located } + switch located { + case let .scalar(value, range): + return .scalar(restoring(value), range) + case let .sequence(items, range): + return .sequence(items.map(restoring), range) + case let .mapping(pairs, range): + return .mapping(pairs.map { .init(keyNode: restoring($0.keyNode), keyRange: $0.keyRange, value: restoring($0.value)) }, range) + } + } + + private func restoring(_ mapping: PureYAML.Model.Mapping) -> PureYAML.Model.Mapping { + .init(mapping.pairs.map { .init(keyNode: restoring($0.keyNode), value: restoring($0.value)) }) + } + + private func restoring(_ key: PureYAML.Model.Key) -> PureYAML.Model.Key { + switch key { + case let .string(text): .string(restoring(text)) + case let .sequence(items): .sequence(items.map(restoring)) + case let .mapping(mapping): .mapping(restoring(mapping)) + } + } + } +} diff --git a/Sources/PureYAMLJSON/JSON.Writer.swift b/Sources/PureYAMLJSON/JSON.Writer.swift index bf0ebe7..9500c23 100644 --- a/Sources/PureYAMLJSON/JSON.Writer.swift +++ b/Sources/PureYAMLJSON/JSON.Writer.swift @@ -92,7 +92,9 @@ public extension PureYAML.JSON { /// Quote and escape a string per RFC 8259: `"` and `\` are escaped, the /// named short escapes are used where they apply, and any remaining control - /// character below U+0020 becomes a `\u00XX` sequence. + /// character below U+0020 becomes a `\u00XX` sequence. So that a YAML reader gives the same string back from the text (JSON may hold them as they are, + /// and YAML does not: it folds U+0085, U+2028 and U+2029 as line breaks, refuses DEL and the C1 controls, and drops a byte order mark that starts a stream), + /// U+007F to U+009F, U+2028, U+2029, U+FEFF, U+FFFE and U+FFFF are escaped too. private static func escaped(_ text: String) -> String { var out = "\"" for scalar in text.unicodeScalars { @@ -105,7 +107,7 @@ public extension PureYAML.JSON { case "\r": out += "\\r" case "\t": out += "\\t" default: - if scalar.value < 0x20 { + if mustEscape(scalar) { out += "\\u" + hex4(scalar.value) } else { out.unicodeScalars.append(scalar) @@ -116,6 +118,10 @@ public extension PureYAML.JSON { return out } + private static func mustEscape(_ scalar: Unicode.Scalar) -> Bool { + scalar.value < 0x20 || (0x7F ... 0x9F).contains(scalar.value) || [0x2028, 0x2029, 0xFEFF, 0xFFFE, 0xFFFF].contains(scalar.value) + } + private static func hex4(_ value: UInt32) -> String { let digits = "0123456789abcdef" var result = "" diff --git a/Tests/PureYAMLJSONTests/JSONStringCharactersTests.swift b/Tests/PureYAMLJSONTests/JSONStringCharactersTests.swift new file mode 100644 index 0000000..46700b2 --- /dev/null +++ b/Tests/PureYAMLJSONTests/JSONStringCharactersTests.swift @@ -0,0 +1,124 @@ +import PureYAML +@testable import PureYAMLJSON +import Testing + +/// A string that JSON holds is the same string after JSON is read, written, or converted to YAML and back. The characters RFC 8259 lets a string hold as they are, +/// and the YAML reader folds, refuses or drops, are the ones that are checked: U+0085, U+2028 and U+2029, U+007F and the C1 controls, the controls below U+0020 written +/// as escapes, and a byte order mark. +@Suite("JSON string characters") +struct JSONStringCharactersTests { + static let characters: [String] = [ + "\u{0}", "\u{1}", "\u{1F}", "\u{7F}", "\u{80}", "\u{85}", "\u{9F}", "\u{A0}", "\u{2028}", "\u{2029}", "a\u{2028}b", "a\u{85}b", "\u{FEFF}", "\u{FEFF}x", "x\u{FEFF}", + "\u{FFFE}", "é", "日本語", "😀", " ", "", "a b", "true", "1.5", "- x", "tab\there", "line\nbreak", "cr\rlf", + ] + + /// The JSON text of a string, written from RFC 8259 and not with the code under test: every character that has to be escaped is, and nothing else is, so a raw + /// U+2028 or U+0085 stays raw, as JSON lets it. + static func json(_ string: String) -> String { + var out = "\"" + for scalar in string.unicodeScalars { + switch scalar { + case "\"": out += "\\\"" + case "\\": out += "\\\\" + case "\n": out += "\\n" + case "\r": out += "\\r" + case "\t": out += "\\t" + case let control where control.value < 0x20: out += "\\u00" + String(control.value >> 4, radix: 16) + String(control.value & 0xF, radix: 16) + default: out.unicodeScalars.append(scalar) + } + } + return out + "\"" + } + + @Test("JSON that holds a character as it is, is read as the string it means", arguments: characters) + func test_parseKeepsEveryCharacter(_ string: String) throws { + #expect(try PureYAML.JSON.parse(Self.json(string)) == .string(string)) + #expect(try PureYAML.JSON.parse("[\(Self.json(string))]") == .sequence([.string(string)])) + #expect(try PureYAML.JSON.parse("{\(Self.json(string)): \(Self.json(string))}") == .mapping(.init([.init(key: string, value: .string(string))]))) + } + + @Test("A string is written as JSON that a JSON reader and the YAML reader give back as the same string", arguments: characters) + func test_writerIsReadBack(_ string: String) throws { + let value = PureYAML.Model.Value.string(string) + let written = try PureYAML.JSON.Writer.string(from: value, options: .init(pretty: false, indent: 2)) + #expect(try PureYAML.JSON.parse(written) == value, "\(written.debugDescription)") + #expect(try PureYAML.parse(written) == value, "\(written.debugDescription)") + } + + @Test("Characters JSON may hold as they are are written escaped, so that no YAML reader can fold or drop them") + func test_writerEscapes() throws { + let value = PureYAML.Model.Value.string("\u{7F}\u{80}\u{85}\u{9F}\u{2028}\u{2029}\u{FEFF}") + let written = try PureYAML.JSON.Writer.string(from: value, options: .init(pretty: false, indent: 2)) + #expect(written == "\"\\u007f\\u0080\\u0085\\u009f\\u2028\\u2029\\ufeff\"") + } + + @Test("A string converts from JSON to YAML to JSON and back to YAML as the same string", arguments: characters) + func test_convertKeepsTheString(_ string: String) throws { + let source = Self.json(string) + let yaml = try PureYAML.JSON.Convert.convert(source, from: .json, to: .yaml) + #expect(try PureYAML.parse(yaml) == .string(string), "\(yaml.debugDescription)") + let json = try PureYAML.JSON.Convert.convert(yaml, from: .yaml, to: .json) + #expect(try PureYAML.JSON.parse(json) == .string(string), "\(json.debugDescription)") + #expect(try PureYAML.JSON.Convert.convert(json, from: .json, to: .yaml) == yaml) + } + + @Test("A key converts as the same key", arguments: characters) + func test_convertKeepsTheKey(_ key: String) throws { + let source = "{\(Self.json(key)): 1}" + let yaml = try PureYAML.JSON.Convert.convert(source, from: .json, to: .yaml) + #expect(try PureYAML.parse(yaml) == .mapping(.init([.init(key: key, value: .int(1))])), "\(yaml.debugDescription)") + } + + @Test("Only characters inside a string are held back: an escaped quote does not end the string, and text outside strings is left as it is") + func test_onlyInsideStrings() throws { + let inside = try PureYAML.JSON.Source("\"a\\\"\u{2028}\"") + #expect(inside.masked.unicodeScalars.count == 6 && !inside.masked.unicodeScalars.contains("\u{2028}")) + let afterEscapedBackslash = try PureYAML.JSON.Source("\"a\\\\\"\u{2028}") + #expect(afterEscapedBackslash.masked == "\"a\\\\\"\u{2028}", "the string ended at the second quote, so the character after it is outside") + let two = try PureYAML.JSON.Source("[\"\u{85}\", \"\u{85}\"]") + let scalars = Array(two.masked.unicodeScalars) + #expect(scalars[2] == scalars[7] && scalars[2] != "\u{85}", "one replacement for one character") + #expect(try PureYAML.JSON.Source("{\"k\": 1}").masked == "{\"k\": 1}") + } + + @Test("A private-use character that the text holds is never used as a replacement for another character") + func test_replacementIsNotInTheText() throws { + let text = "[\"\u{E000}\", \"\u{85}\"]" + #expect(try PureYAML.JSON.parse(text) == .sequence([.string("\u{E000}"), .string("\u{85}")])) + } + + /// The spans of a JSON text that holds characters the YAML reader cannot be given are the spans of the same text with each of them written as one ordinary character, + /// which is a statement about positions that does not use the code under test to say what they are. + @Test("A located parse of JSON that holds such characters has the spans of the text with ordinary characters in their place") + func test_locatedSpansDoNotMove() throws { + let held = "{\n \"a\": \"x\u{2028}y\u{85}z\",\n \"b\u{7F}\": [\"\u{9F}\", 1],\n \"c\": {\"d\": \"\u{FFFE}\"}\n}\n" + let plain = String(held.unicodeScalars.map { PureYAML.JSON.Source.isHeldBack($0) ? "q" : Character($0) }) + let located = try PureYAML.JSON.parseLocated(held) + let reference = try PureYAML.parseLocated(plain) + #expect(Self.ranges(of: located) == Self.ranges(of: reference)) + #expect(located.value == .mapping(.init([ + .init(key: "a", value: .string("x\u{2028}y\u{85}z")), + .init(key: "b\u{7F}", value: .sequence([.string("\u{9F}"), .int(1)])), + .init(key: "c", value: .mapping(.init([.init(key: "d", value: .string("\u{FFFE}"))]))), + ]))) + #expect(try located.value == PureYAML.JSON.parse(held)) + } + + /// Every span of a located tree, in order. + private static func ranges(of located: PureYAML.Model.LocatedValue) -> [PureYAML.Model.SourceRange] { + switch located { + case let .scalar(_, range): + [range] + case let .sequence(items, range): + [range] + items.flatMap(ranges) + case let .mapping(pairs, range): + [range] + pairs.flatMap { [$0.keyRange] + ranges(of: $0.value) } + } + } + + @Test("Without the format of the source, the source is read as YAML as it was before") + func test_unspecifiedSourceIsYAML() throws { + #expect(try PureYAML.JSON.Convert.convert("a: 1\n", to: .json, pretty: false) == "{\"a\":1}") + #expect(try PureYAML.JSON.Convert.convert("{\"a\":1}", to: .yaml) == "a: 1\n") + } +}