diff --git a/CHANGELOG.md b/CHANGELOG.md index 061ef8c..c4cf9a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -401,6 +401,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- A string the emitter wrote without quotes could be read back as something + else, so the string `yes` (and `no`, `on`, `off` in every case), `0x1F`, + `0b1`, `0o17`, `1_000`, `.inf` and `.nan` did not survive a dump and parse. + `PureYAML.Parsing.PlainScalar` is now the one definition of how a plain + scalar reads, and the parser and the emitter both take it from there; the + emitter quotes every string that does not read back as itself. Also quoted + now: `<<` (the merge key) and `...` (the document end marker), and a string + with a CR+LF break, which was written raw because Swift counts CR+LF as one + character. `PlainScalarRoundTripTests` holds 128 awkward strings at the + root, as a mapping key and value, and as a sequence item, in block and flow + style. `PureYAML.JSON.Convert` no longer refuses `{"s":"yes"}`. + - Stream dump no longer doubles the `---` document marker under `explicitDocumentStart` or `canonical`. The stream path now owns marker emission and renders each document body marker-free, so a two-document diff --git a/Sources/Emitting/Dumper.ScalarAnalysis.swift b/Sources/Emitting/Dumper.ScalarAnalysis.swift index 097b39a..94b971e 100644 --- a/Sources/Emitting/Dumper.ScalarAnalysis.swift +++ b/Sources/Emitting/Dumper.ScalarAnalysis.swift @@ -91,7 +91,8 @@ extension PureYAML.Emitting.Dumper { guard value.first?.isWhitespace == false, value.last?.isWhitespace == false else { return false } - guard !value.contains("\n"), !value.contains("\r"), !value.contains("\t") else { + // By unicode scalar: CR+LF is one Character in Swift, so `contains("\n")` is false for it and a string with a CRLF break would be written raw. + guard !value.unicodeScalars.contains(where: { $0 == "\n" || $0 == "\r" || $0 == "\t" }) else { return false } guard !isReservedPlainScalar(value), !isNumberLikePlainScalar(value) else { @@ -106,6 +107,10 @@ extension PureYAML.Emitting.Dumper { guard value.last != ":" else { return false } + // `<<` as a key is the merge key, and `...` at the start of a line ends the document. + guard value != "<<", value != "...", !value.hasPrefix("... ") else { + return false + } return !containsColonSpace(value) && !value.contains("#") } @@ -132,13 +137,9 @@ extension PureYAML.Emitting.Dumper { return false } + /// A word the parser reads as something other than a string (null, a boolean, a number): written plain, a string of it would come back changed. func isReservedPlainScalar(_ value: String) -> Bool { - switch value { - case "~", "null", "Null", "NULL", "true", "True", "TRUE", "false", "False", "FALSE": - true - default: - false - } + !PureYAML.Parsing.PlainScalar.readsAsString(value) } func isNumberLikePlainScalar(_ value: String) -> Bool { diff --git a/Sources/Parsing/Parsing.PlainScalar.swift b/Sources/Parsing/Parsing.PlainScalar.swift new file mode 100644 index 0000000..d44fc86 --- /dev/null +++ b/Sources/Parsing/Parsing.PlainScalar.swift @@ -0,0 +1,98 @@ +extension PureYAML.Parsing { + /// How a plain (unquoted) scalar reads: as null, a boolean, an integer, a floating-point number, or else a string. + /// + /// This is the one definition of that reading. The parser takes the value of every plain scalar from it, and the emitter asks it whether a string can be + /// written without quotes, since a string that reads as anything else would come back changed. Two lists of the same words kept apart drift apart: the + /// emitter once wrote the string `yes` plain while the parser read `yes` as a boolean. + enum PlainScalar { + /// The value `text` has as a plain scalar. `text` has no surrounding whitespace. + static func resolve(_ text: String) -> PureYAML.Model.Value { + switch text { + case "", "~", "null", "Null", "NULL": + return .null + default: + break + } + if let bool = bool(text) { + return .bool(bool) + } + if let int = integer(text), !text.contains(".") { + return .int(int) + } + if text.contains(".") || text.contains("e") || text.contains("E"), let double = double(text) { + return .double(double) + } + return .string(text) + } + + /// Whether `text` written plain reads back as the same string. + static func readsAsString(_ text: String) -> Bool { + if case .string = resolve(text) { + return true + } + return false + } + + static func bool(_ text: String) -> Bool? { + switch text { + case "true", "True", "TRUE", "yes", "Yes", "YES", "on", "On", "ON": + true + case "false", "False", "FALSE", "no", "No", "NO", "off", "Off", "OFF": + false + default: + nil + } + } + + static func integer(_ text: String) -> Int? { + let value = removingNumericSeparators(from: text) + let negative = value.hasPrefix("-") + let hasSign = negative || value.hasPrefix("+") + let unsigned = hasSign ? String(value.dropFirst()) : value + + guard !unsigned.isEmpty else { + return nil + } + guard unsigned != "0" else { + return 0 + } + + let radixPrefixes: [(prefix: String, radix: Int)] = [ + ("0x", 16), + ("0X", 16), + ("0b", 2), + ("0B", 2), + ("0o", 8), + ("0O", 8), + ("0", 8), + ] + + for radixPrefix in radixPrefixes where unsigned.hasPrefix(radixPrefix.prefix) { + let digits = String(unsigned.dropFirst(radixPrefix.prefix.count)) + guard !digits.isEmpty, let magnitude = Int(digits, radix: radixPrefix.radix) else { + return nil + } + return negative ? -magnitude : magnitude + } + + return Int(value) + } + + static func double(_ text: String) -> Double? { + switch text { + case ".inf", ".Inf", ".INF", "+.inf", "+.Inf", "+.INF": + .infinity + case "-.inf", "-.Inf", "-.INF": + -.infinity + case ".nan", ".NaN", ".NAN", "+.nan", "+.NaN", "+.NAN", "-.nan", "-.NaN", "-.NAN": + .nan + default: + Double(removingNumericSeparators(from: text)) + } + } + + private static func removingNumericSeparators(from text: String) -> String { + text.filter { $0 != "_" } + } + } +} diff --git a/Sources/Parsing/Scalar.swift b/Sources/Parsing/Scalar.swift index 2823967..a1f97d2 100644 --- a/Sources/Parsing/Scalar.swift +++ b/Sources/Parsing/Scalar.swift @@ -14,93 +14,25 @@ extension PureYAML.Parsing.Parser { line: Int, ) throws -> PureYAML.Model.Value { let value = trim(text) - switch value { - case "", "~", "null", "Null", "NULL": - return .null - default: - break - } - - if let bool = parseBool(value) { - return .bool(bool) - } if value.hasPrefix("\"") { return try .string(parseDoubleQuoted(value, line: line)) } if value.hasPrefix("'") { return try .string(parseSingleQuoted(value, line: line)) } - if let int = parseInteger(value), !value.contains(".") { - return .int(int) - } - if value.contains(".") || value.contains("e") || value.contains("E") { - if let double = parseDouble(value) { - return .double(double) - } - } - return .string(value) + return PureYAML.Parsing.PlainScalar.resolve(value) } func parseBool(_ text: String) -> Bool? { - switch text { - case "true", "True", "TRUE", "yes", "Yes", "YES", "on", "On", "ON": - true - case "false", "False", "FALSE", "no", "No", "NO", "off", "Off", "OFF": - false - default: - nil - } + PureYAML.Parsing.PlainScalar.bool(text) } func parseInteger(_ text: String) -> Int? { - let value = removingNumericSeparators(from: text) - let negative = value.hasPrefix("-") - let hasSign = negative || value.hasPrefix("+") - let unsigned = hasSign ? String(value.dropFirst()) : value - - guard !unsigned.isEmpty else { - return nil - } - guard unsigned != "0" else { - return 0 - } - - let radixPrefixes: [(prefix: String, radix: Int)] = [ - ("0x", 16), - ("0X", 16), - ("0b", 2), - ("0B", 2), - ("0o", 8), - ("0O", 8), - ("0", 8), - ] - - for radixPrefix in radixPrefixes where unsigned.hasPrefix(radixPrefix.prefix) { - let digits = String(unsigned.dropFirst(radixPrefix.prefix.count)) - guard !digits.isEmpty, let magnitude = Int(digits, radix: radixPrefix.radix) else { - return nil - } - return negative ? -magnitude : magnitude - } - - return Int(value) + PureYAML.Parsing.PlainScalar.integer(text) } func parseDouble(_ text: String) -> Double? { - switch text { - case ".inf", ".Inf", ".INF", "+.inf", "+.Inf", "+.INF": - .infinity - case "-.inf", "-.Inf", "-.INF": - -.infinity - case ".nan", ".NaN", ".NAN", "+.nan", "+.NaN", "+.NAN", "-.nan", "-.NaN", "-.NAN": - .nan - default: - Double(removingNumericSeparators(from: text)) - } - } - - func removingNumericSeparators(from text: String) -> String { - text.filter { $0 != "_" } + PureYAML.Parsing.PlainScalar.double(text) } func parseSingleQuoted( diff --git a/Tests/PlainScalarRoundTripTests.swift b/Tests/PlainScalarRoundTripTests.swift new file mode 100644 index 0000000..61cec7e --- /dev/null +++ b/Tests/PlainScalarRoundTripTests.swift @@ -0,0 +1,56 @@ +@testable import PureYAML +import Testing + +/// A string the emitter writes without quotes must be read back as that same string. The parser decides what a plain word means (a boolean, a number, null, +/// or a string), so the emitter quotes every string the parser would read as anything else. +@Suite("Plain scalar round trip") +struct PlainScalarRoundTripTests { + /// Words and shapes that a reader may take for something other than a string, and shapes that break plain syntax. + static let corpus: [String] = [ + "yes", "Yes", "YES", "no", "No", "NO", "on", "On", "ON", "off", "Off", "OFF", "y", "n", "Y", "N", "true", "True", "TRUE", "false", "False", "FALSE", + "null", "Null", "NULL", "~", "0", "-0", "+0", "1", "+1", "-1", "017", "0123", "0o17", "0O17", "0x1F", "0X1f", "0b101", "0B1", "-0x1F", "+0x1F", "1_000", + "1__0", "0_1", "_1", "1_", "1e3", "1E3", "1e+3", "1.5", "1.", ".5", "-.5", "+.5", ".inf", ".Inf", ".INF", "-.inf", "+.inf", ".nan", ".NaN", ".NAN", "inf", + "nan", "Infinity", "1e400", "0x", "0b", "0o", "9223372036854775807", "9223372036854775808", "-9223372036854775808", "12345678901234567890", + "2001-01-01", "12:30:45", "1:30", "<<", "=", "---", "...", "?", "- x", "-", "a: b", "a:", "a :", ":a", "#c", "a #c", "x#c", "!x", "&a", "*a", "|", ">", + "%x", "@x", "`x", "'x", "\"x", " x", "x ", "a\tb", "a\nb", "a\r\nb", "héllo", "✓", "a,b", "[a]", "{a}", "a]", "k: v", "x:y", "x: ", "", " ", "word", + "two words", "a-b", "a?b", "?x", "-x", "- ", "1 2", "1e", "e1", ".", "..", "+", "0.", "00", "0e0", "1_0.5", "١٢٣", "٣.٥", + ] + + private static let styles: [PureYAML.Emitting.CollectionStyle] = [.block, .flow] + + private func roundTrip(_ value: PureYAML.Model.Value, _ style: PureYAML.Emitting.CollectionStyle) throws -> PureYAML.Model.Value { + let yaml = PureYAML.dump(value, options: .init(scalarStyle: .plainWhenSafe, collectionStyle: style)) + return try PureYAML.parse(yaml) + } + + @Test("A string at the root reads back as itself", arguments: corpus) + func root(text: String) throws { + for style in Self.styles { + #expect(try roundTrip(.string(text), style) == .string(text), "\(style): \(text.debugDescription)") + } + } + + @Test("A string as a mapping value reads back as itself", arguments: corpus) + func mappingValue(text: String) throws { + let value = PureYAML.Model.Value.mapping(.init([.init(key: "k", value: .string(text))])) + for style in Self.styles { + #expect(try roundTrip(value, style) == value, "\(style): \(text.debugDescription)") + } + } + + @Test("A string as a mapping key reads back as itself", arguments: corpus) + func mappingKey(text: String) throws { + let value = PureYAML.Model.Value.mapping(.init([.init(key: text, value: .int(1))])) + for style in Self.styles { + #expect(try roundTrip(value, style) == value, "\(style): \(text.debugDescription)") + } + } + + @Test("A string in a sequence reads back as itself", arguments: corpus) + func sequenceItem(text: String) throws { + let value = PureYAML.Model.Value.sequence([.string(text), .string("after")]) + for style in Self.styles { + #expect(try roundTrip(value, style) == value, "\(style): \(text.debugDescription)") + } + } +}