fix(json): keep every character of a JSON string through the trip

The JSON writer escapes the characters a YAML reader mangles. PureYAML.JSON.parse and parseLocated read JSON with those characters held back and given back, so no position moves, and Convert.convert takes the format of its source.

CHANGELOG.md

@@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- A string no longer changes on the way through YAML or JSON. `PureYAML.dump` wrote control characters, DEL, the C1 controls, U+2028, U+2029, U+FEFF, U+FFFE and
+  U+FFFF as they are, where the reader refused them (`invalid control character`), folded U+0085, U+2028 and U+2029 to a space without a word, or dropped a leading
+  U+FEFF as a byte order mark. They are now written as the escape that stands for each (`\x01`, `\N`, `\L`, `\P`, ``) and are never written plain or in a
+  literal block. `PureYAML.JSON.Writer` escapes the same characters (`\u007f`, `
`, and so on), which RFC 8259 allows and a YAML reader needs.
+- A JSON text that holds one of those characters as it is (RFC 8259 allows it) was read as another string, or refused. `PureYAML.JSON.parse(_:)` and `PureYAML.JSON.parseLocated(_:)` read JSON and keep
+  every character of its strings, and `PureYAML.JSON.Convert.convert(_:from:to:pretty:)` takes the format of the source (`from:`, default nil: the source is read as YAML, as
+  before) and reads a JSON source with it.
+
+### Added
+
+- `PureYAML.JSON.parse(_:)`, `PureYAML.JSON.parseLocated(_:)` (every position is where it is written) and the `from:` parameter of `PureYAML.JSON.Convert.convert`.
+
 ## [0.3.0] - 2026-11-14
 
 ### Changed

Sources/PureYAMLJSON/JSON.Convert.swift

@@ -16,11 +16,12 @@ public extension PureYAML.JSON {
     /// value may use a non-string key or a non-finite number). "To YAML" still
     /// rejects duplicate keys, which are legal in JSON but not in YAML.
     enum Convert {
-        /// Convert source text to the target format, validated end to end.
-        public static func convert(_ source: String, to target: Format, pretty: Bool = true) throws -> String {
-            let value = try parse(source)
+        /// Convert source text to the target format, validated end to end. When the format of the source is known it is given as `from`, and a source that is JSON
+        /// is read as JSON (``PureYAML/JSON/parse(_:)``), which keeps every character of its strings; without it the source is read by the YAML reader as it is.
+        public static func convert(_ source: String, from sourceFormat: Format? = nil, to target: Format, pretty: Bool = true) throws -> String {
+            let value = try parse(source, as: sourceFormat)
             let output = try emit(value, as: target, pretty: pretty)
-            guard try parse(output) == value else {
+            guard try parse(output, as: target) == value else {
                 throw Failure.roundTripMismatch(target)
             }
             return output
@@ -53,9 +54,9 @@ public extension PureYAML.JSON {
             }
         }
 
-        private static func parse(_ text: String) throws -> PureYAML.Model.Value {
+        private static func parse(_ text: String, as format: Format?) throws -> PureYAML.Model.Value {
             do {
-                return try PureYAML.parse(text)
+                return try format == .json ? PureYAML.JSON.parse(text) : PureYAML.parse(text)
             } catch {
                 throw Failure.sourceInvalid(String(describing: error))
             }

Sources/PureYAMLJSON/JSON.Parse.swift

@@ -0,0 +1,21 @@
+import PureYAML
+
+public extension PureYAML.JSON {
+    /// Reads text that is JSON into the shared value tree.
+    ///
+    /// JSON is a subset of YAML 1.2 apart from a few characters that RFC 8259 lets a string hold as they are and the YAML reader does not: it folds U+0085, U+2028 and
+    /// U+2029 as line breaks, it refuses U+007F, the C1 controls, U+FFFE and U+FFFF, and so a string that holds one comes back from ``PureYAML/parse(_:)`` as another
+    /// string or not at all. Here each such character inside a string is held back from the reader and given back in the value, so the string is the one the JSON
+    /// means. Everything else is read by the YAML reader as it is, and what is not JSON is refused by it as before.
+    static func parse(_ text: String) throws -> PureYAML.Model.Value {
+        let source = try Source(text)
+        return try source.restoring(PureYAML.parse(source.masked))
+    }
+
+    /// As ``parse(_:)``, with the span of every node. The spans are those of the text as it is: a held back character is one scalar in the text the reader sees, as it is
+    /// in `text`, so no position moves.
+    static func parseLocated(_ text: String) throws -> PureYAML.Model.LocatedValue {
+        let source = try Source(text)
+        return try source.restoring(PureYAML.parseLocated(source.masked))
+    }
+}

Sources/PureYAMLJSON/JSON.Source.swift

@@ -0,0 +1,105 @@
+import PureYAML
+
+extension PureYAML.JSON {
+    /// JSON text with the characters the YAML reader cannot be given held back.
+    ///
+    /// Each U+007F to U+009F, U+2028, U+2029, U+FFFE and U+FFFF that is inside a string is replaced, one scalar for one scalar, by a private-use character that is not
+    /// in the text, so that the reader sees a plain character and every position of the text is the position it had. The replacement is undone in the value that
+    /// comes out of the reader. A private-use character that the text itself holds is never used as a replacement, so what comes back is never mistaken for it.
+    struct Source {
+        let masked: String
+        /// What each replacement stands for.
+        private let originals: [Unicode.Scalar: Unicode.Scalar]
+
+        init(_ text: String) throws {
+            let present = Set(text.unicodeScalars.filter { Self.privateUse.contains($0.value) })
+            var free = Self.privateUse.lazy.compactMap(Unicode.Scalar.init).filter { !present.contains($0) }.makeIterator()
+            var replacement: [Unicode.Scalar: Unicode.Scalar] = [:]
+            var originals: [Unicode.Scalar: Unicode.Scalar] = [:]
+            var output = String.UnicodeScalarView()
+            var inString = false
+            var escaped = false
+            for scalar in text.unicodeScalars {
+                guard inString else {
+                    inString = scalar == "\""
+                    output.append(scalar)
+                    continue
+                }
+                if escaped {
+                    escaped = false
+                } else if scalar == "\\" {
+                    escaped = true
+                } else if scalar == "\"" {
+                    inString = false
+                } else if Self.isHeldBack(scalar) {
+                    if replacement[scalar] == nil {
+                        guard let next = free.next() else {
+                            throw PureYAML.JSON.Convert.Failure.sourceInvalid("the text holds too many private-use characters to read it as JSON")
+                        }
+                        replacement[scalar] = next
+                        originals[next] = scalar
+                    }
+                    output.append(replacement[scalar] ?? scalar)
+                    continue
+                }
+                output.append(scalar)
+            }
+            masked = String(output)
+            self.originals = originals
+        }
+
+        private static let privateUse: ClosedRange<UInt32> = 0xE000 ... 0xF8FF
+
+        /// The characters RFC 8259 lets a string hold as they are and the YAML reader folds or refuses.
+        static func isHeldBack(_ scalar: Unicode.Scalar) -> Bool {
+            (0x7F ... 0x9F).contains(scalar.value) || [0x2028, 0x2029, 0xFFFE, 0xFFFF].contains(scalar.value)
+        }
+
+        func restoring(_ text: String) -> String {
+            guard !originals.isEmpty else { return text }
+            var output = String.UnicodeScalarView()
+            for scalar in text.unicodeScalars {
+                output.append(originals[scalar] ?? scalar)
+            }
+            return String(output)
+        }
+
+        func restoring(_ value: PureYAML.Model.Value) -> PureYAML.Model.Value {
+            guard !originals.isEmpty else { return value }
+            switch value {
+            case .null, .bool, .int, .double:
+                return value
+            case let .string(text):
+                return .string(restoring(text))
+            case let .sequence(items):
+                return .sequence(items.map(restoring))
+            case let .mapping(mapping):
+                return .mapping(restoring(mapping))
+            }
+        }
+
+        func restoring(_ located: PureYAML.Model.LocatedValue) -> PureYAML.Model.LocatedValue {
+            guard !originals.isEmpty else { return located }
+            switch located {
+            case let .scalar(value, range):
+                return .scalar(restoring(value), range)
+            case let .sequence(items, range):
+                return .sequence(items.map(restoring), range)
+            case let .mapping(pairs, range):
+                return .mapping(pairs.map { .init(keyNode: restoring($0.keyNode), keyRange: $0.keyRange, value: restoring($0.value)) }, range)
+            }
+        }
+
+        private func restoring(_ mapping: PureYAML.Model.Mapping) -> PureYAML.Model.Mapping {
+            .init(mapping.pairs.map { .init(keyNode: restoring($0.keyNode), value: restoring($0.value)) })
+        }
+
+        private func restoring(_ key: PureYAML.Model.Key) -> PureYAML.Model.Key {
+            switch key {
+            case let .string(text): .string(restoring(text))
+            case let .sequence(items): .sequence(items.map(restoring))
+            case let .mapping(mapping): .mapping(restoring(mapping))
+            }
+        }
+    }
+}

Sources/PureYAMLJSON/JSON.Writer.swift

@@ -92,7 +92,9 @@ public extension PureYAML.JSON {
 
         /// Quote and escape a string per RFC 8259: `"` and `\` are escaped, the
         /// named short escapes are used where they apply, and any remaining control
-        /// character below U+0020 becomes a `\u00XX` sequence.
+        /// character below U+0020 becomes a `\u00XX` sequence. So that a YAML reader gives the same string back from the text (JSON may hold them as they are,
+        /// and YAML does not: it folds U+0085, U+2028 and U+2029 as line breaks, refuses DEL and the C1 controls, and drops a byte order mark that starts a stream),
+        /// U+007F to U+009F, U+2028, U+2029, U+FEFF, U+FFFE and U+FFFF are escaped too.
         private static func escaped(_ text: String) -> String {
             var out = "\""
             for scalar in text.unicodeScalars {
@@ -105,7 +107,7 @@ public extension PureYAML.JSON {
                 case "\r": out += "\\r"
                 case "\t": out += "\\t"
                 default:
-                    if scalar.value < 0x20 {
+                    if mustEscape(scalar) {
                         out += "\\u" + hex4(scalar.value)
                     } else {
                         out.unicodeScalars.append(scalar)
@@ -116,6 +118,10 @@ public extension PureYAML.JSON {
             return out
         }
 
+        private static func mustEscape(_ scalar: Unicode.Scalar) -> Bool {
+            scalar.value < 0x20 || (0x7F ... 0x9F).contains(scalar.value) || [0x2028, 0x2029, 0xFEFF, 0xFFFE, 0xFFFF].contains(scalar.value)
+        }
+
         private static func hex4(_ value: UInt32) -> String {
             let digits = "0123456789abcdef"
             var result = ""

Tests/PureYAMLJSONTests/JSONStringCharactersTests.swift

@@ -0,0 +1,124 @@
+import PureYAML
+@testable import PureYAMLJSON
+import Testing
+
+/// A string that JSON holds is the same string after JSON is read, written, or converted to YAML and back. The characters RFC 8259 lets a string hold as they are,
+/// and the YAML reader folds, refuses or drops, are the ones that are checked: U+0085, U+2028 and U+2029, U+007F and the C1 controls, the controls below U+0020 written
+/// as escapes, and a byte order mark.
+@Suite("JSON string characters")
+struct JSONStringCharactersTests {
+    static let characters: [String] = [
+        "\u{0}", "\u{1}", "\u{1F}", "\u{7F}", "\u{80}", "\u{85}", "\u{9F}", "\u{A0}", "\u{2028}", "\u{2029}", "a\u{2028}b", "a\u{85}b", "\u{FEFF}", "\u{FEFF}x", "x\u{FEFF}",
+        "\u{FFFE}", "é", "日本語", "😀", " ", "", "a b", "true", "1.5", "- x", "tab\there", "line\nbreak", "cr\rlf",
+    ]
+
+    /// The JSON text of a string, written from RFC 8259 and not with the code under test: every character that has to be escaped is, and nothing else is, so a raw
+    /// U+2028 or U+0085 stays raw, as JSON lets it.
+    static func json(_ string: String) -> String {
+        var out = "\""
+        for scalar in string.unicodeScalars {
+            switch scalar {
+            case "\"": out += "\\\""
+            case "\\": out += "\\\\"
+            case "\n": out += "\\n"
+            case "\r": out += "\\r"
+            case "\t": out += "\\t"
+            case let control where control.value < 0x20: out += "\\u00" + String(control.value >> 4, radix: 16) + String(control.value & 0xF, radix: 16)
+            default: out.unicodeScalars.append(scalar)
+            }
+        }
+        return out + "\""
+    }
+
+    @Test("JSON that holds a character as it is, is read as the string it means", arguments: characters)
+    func test_parseKeepsEveryCharacter(_ string: String) throws {
+        #expect(try PureYAML.JSON.parse(Self.json(string)) == .string(string))
+        #expect(try PureYAML.JSON.parse("[\(Self.json(string))]") == .sequence([.string(string)]))
+        #expect(try PureYAML.JSON.parse("{\(Self.json(string)): \(Self.json(string))}") == .mapping(.init([.init(key: string, value: .string(string))])))
+    }
+
+    @Test("A string is written as JSON that a JSON reader and the YAML reader give back as the same string", arguments: characters)
+    func test_writerIsReadBack(_ string: String) throws {
+        let value = PureYAML.Model.Value.string(string)
+        let written = try PureYAML.JSON.Writer.string(from: value, options: .init(pretty: false, indent: 2))
+        #expect(try PureYAML.JSON.parse(written) == value, "\(written.debugDescription)")
+        #expect(try PureYAML.parse(written) == value, "\(written.debugDescription)")
+    }
+
+    @Test("Characters JSON may hold as they are are written escaped, so that no YAML reader can fold or drop them")
+    func test_writerEscapes() throws {
+        let value = PureYAML.Model.Value.string("\u{7F}\u{80}\u{85}\u{9F}\u{2028}\u{2029}\u{FEFF}")
+        let written = try PureYAML.JSON.Writer.string(from: value, options: .init(pretty: false, indent: 2))
+        #expect(written == "\"\\u007f\\u0080\\u0085\\u009f\\u2028\\u2029\\ufeff\"")
+    }
+
+    @Test("A string converts from JSON to YAML to JSON and back to YAML as the same string", arguments: characters)
+    func test_convertKeepsTheString(_ string: String) throws {
+        let source = Self.json(string)
+        let yaml = try PureYAML.JSON.Convert.convert(source, from: .json, to: .yaml)
+        #expect(try PureYAML.parse(yaml) == .string(string), "\(yaml.debugDescription)")
+        let json = try PureYAML.JSON.Convert.convert(yaml, from: .yaml, to: .json)
+        #expect(try PureYAML.JSON.parse(json) == .string(string), "\(json.debugDescription)")
+        #expect(try PureYAML.JSON.Convert.convert(json, from: .json, to: .yaml) == yaml)
+    }
+
+    @Test("A key converts as the same key", arguments: characters)
+    func test_convertKeepsTheKey(_ key: String) throws {
+        let source = "{\(Self.json(key)): 1}"
+        let yaml = try PureYAML.JSON.Convert.convert(source, from: .json, to: .yaml)
+        #expect(try PureYAML.parse(yaml) == .mapping(.init([.init(key: key, value: .int(1))])), "\(yaml.debugDescription)")
+    }
+
+    @Test("Only characters inside a string are held back: an escaped quote does not end the string, and text outside strings is left as it is")
+    func test_onlyInsideStrings() throws {
+        let inside = try PureYAML.JSON.Source("\"a\\\"\u{2028}\"")
+        #expect(inside.masked.unicodeScalars.count == 6 && !inside.masked.unicodeScalars.contains("\u{2028}"))
+        let afterEscapedBackslash = try PureYAML.JSON.Source("\"a\\\\\"\u{2028}")
+        #expect(afterEscapedBackslash.masked == "\"a\\\\\"\u{2028}", "the string ended at the second quote, so the character after it is outside")
+        let two = try PureYAML.JSON.Source("[\"\u{85}\", \"\u{85}\"]")
+        let scalars = Array(two.masked.unicodeScalars)
+        #expect(scalars[2] == scalars[7] && scalars[2] != "\u{85}", "one replacement for one character")
+        #expect(try PureYAML.JSON.Source("{\"k\": 1}").masked == "{\"k\": 1}")
+    }
+
+    @Test("A private-use character that the text holds is never used as a replacement for another character")
+    func test_replacementIsNotInTheText() throws {
+        let text = "[\"\u{E000}\", \"\u{85}\"]"
+        #expect(try PureYAML.JSON.parse(text) == .sequence([.string("\u{E000}"), .string("\u{85}")]))
+    }
+
+    /// The spans of a JSON text that holds characters the YAML reader cannot be given are the spans of the same text with each of them written as one ordinary character,
+    /// which is a statement about positions that does not use the code under test to say what they are.
+    @Test("A located parse of JSON that holds such characters has the spans of the text with ordinary characters in their place")
+    func test_locatedSpansDoNotMove() throws {
+        let held = "{\n  \"a\": \"x\u{2028}y\u{85}z\",\n  \"b\u{7F}\": [\"\u{9F}\", 1],\n  \"c\": {\"d\": \"\u{FFFE}\"}\n}\n"
+        let plain = String(held.unicodeScalars.map { PureYAML.JSON.Source.isHeldBack($0) ? "q" : Character($0) })
+        let located = try PureYAML.JSON.parseLocated(held)
+        let reference = try PureYAML.parseLocated(plain)
+        #expect(Self.ranges(of: located) == Self.ranges(of: reference))
+        #expect(located.value == .mapping(.init([
+            .init(key: "a", value: .string("x\u{2028}y\u{85}z")),
+            .init(key: "b\u{7F}", value: .sequence([.string("\u{9F}"), .int(1)])),
+            .init(key: "c", value: .mapping(.init([.init(key: "d", value: .string("\u{FFFE}"))]))),
+        ])))
+        #expect(try located.value == PureYAML.JSON.parse(held))
+    }
+
+    /// Every span of a located tree, in order.
+    private static func ranges(of located: PureYAML.Model.LocatedValue) -> [PureYAML.Model.SourceRange] {
+        switch located {
+        case let .scalar(_, range):
+            [range]
+        case let .sequence(items, range):
+            [range] + items.flatMap(ranges)
+        case let .mapping(pairs, range):
+            [range] + pairs.flatMap { [$0.keyRange] + ranges(of: $0.value) }
+        }
+    }
+
+    @Test("Without the format of the source, the source is read as YAML as it was before")
+    func test_unspecifiedSourceIsYAML() throws {
+        #expect(try PureYAML.JSON.Convert.convert("a: 1\n", to: .json, pretty: false) == "{\"a\":1}")
+        #expect(try PureYAML.JSON.Convert.convert("{\"a\":1}", to: .yaml) == "a: 1\n")
+    }
+}