diff --git a/CHANGELOG.md b/CHANGELOG.md index 51a3e67..8a62034 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- A tab in block scalar content past the block's indentation is now kept verbatim instead of being + re-expanded to spaces. A block scalar is indented with spaces only, so a line like `␣\ttext` under a + one-space indent strips the space and keeps the tab, yielding `\ttext`; the scanner measured the + leading whitespace as a tab-expanded width and rebuilt the surplus as spaces, turning the tab into + three spaces. The surplus is now reconstructed from the actual leading characters, so a tab stays a + tab while surplus spaces stay spaces. Raises yaml-test-suite value-level conformance to 180/194 + (92.8%), with the parse and should-error rates unchanged. + - Mid-line separation white space between tokens now includes tabs, not only spaces, completing the tab-as-separation handling. An indicator followed by a tab no longer leaks the tab into the next node: `?\tkey` is the key `key`, not `\tkey`, and a run of tabs between a `:` and its value diff --git a/Sources/Parsing/Scanner.Indentation.swift b/Sources/Parsing/Scanner.Indentation.swift index d92944d..53dd24a 100644 --- a/Sources/Parsing/Scanner.Indentation.swift +++ b/Sources/Parsing/Scanner.Indentation.swift @@ -81,29 +81,33 @@ extension PureYAML.Parsing.Scanner { guard let blockScalarIndentation = state.blockScalarIndentation, width > blockScalarIndentation else { return } - state.blockScalarContentPrefix = String(repeating: " ", count: width - blockScalarIndentation) + state.blockScalarContentPrefix = blockScalarSurplusPrefix(beyond: blockScalarIndentation, in: state) } func scanIndentationWidth(_ state: inout State) throws -> Int { var width = 0 var leadingSpaces = 0 var sawTab = false + var whitespace = "" while let character = state.reader.peek() { if character == " " { width += 1 if !sawTab { leadingSpaces += 1 } + whitespace.append(character) state.reader.advance() } else if character == "\t" { sawTab = true width = nextTabStop(after: width) + whitespace.append(character) state.reader.advance() } else { break } } state.lastLeadingSpaceCount = leadingSpaces + state.lastLeadingWhitespace = whitespace return width } @@ -111,6 +115,21 @@ extension PureYAML.Parsing.Scanner { ((width / 4) + 1) * 4 } + /// The leading-whitespace characters that lie beyond a block scalar's space-only `indent`, kept + /// verbatim so a tab in the content (` \ttext` under a 2-space indent) stays a tab rather than + /// being re-expanded to spaces. Walks the recorded whitespace, counting columns the way the width + /// scan does, and returns the remainder once the indentation columns are covered. + func blockScalarSurplusPrefix(beyond indent: Int, in state: State) -> String { + var column = 0 + var index = state.lastLeadingWhitespace.startIndex + while index < state.lastLeadingWhitespace.endIndex, column < indent { + let character = state.lastLeadingWhitespace[index] + column = character == "\t" ? nextTabStop(after: column) : column + 1 + index = state.lastLeadingWhitespace.index(after: index) + } + return String(state.lastLeadingWhitespace[index...]) + } + func shouldProcessIndentation(_ state: State) -> Bool { guard !state.reader.isAtEnd, !isLineBreak(state.reader.peek()) else { return false @@ -136,7 +155,7 @@ extension PureYAML.Parsing.Scanner { return false } guard width < blockScalarIndentation else { - state.blockScalarContentPrefix = String(repeating: " ", count: width - blockScalarIndentation) + state.blockScalarContentPrefix = blockScalarSurplusPrefix(beyond: blockScalarIndentation, in: state) return true } state.blockScalarIndentation = nil diff --git a/Sources/Parsing/Scanner.swift b/Sources/Parsing/Scanner.swift index 4b0d7bc..844d81a 100644 --- a/Sources/Parsing/Scanner.swift +++ b/Sources/Parsing/Scanner.swift @@ -69,10 +69,14 @@ extension PureYAML.Parsing.Scanner { /// content line). The spec forbids a leading empty line from being more indented than the /// content, so this is compared to the content indentation once that is known. var pendingBlockScalarMaxBlankIndent = 0 - // The count of leading space characters (before any tab) from the most recent indentation - // scan. Indentation is measured in spaces, so this, not the tab-expanded width, is what the - // leading-blank comparison uses: a tab is not indentation. + /// The count of leading space characters (before any tab) from the most recent indentation + /// scan. Indentation is measured in spaces, so this, not the tab-expanded width, is what the + /// leading-blank comparison uses: a tab is not indentation. var lastLeadingSpaceCount = 0 + // The literal leading-whitespace characters from the most recent indentation scan. Block + // scalar content keeps tabs beyond its (space-only) indentation verbatim, so the surplus is + // reconstructed from these characters, not as spaces from the tab-expanded width. + var lastLeadingWhitespace = "" var blockScalarIndentation: Int? var blockScalarContentPrefix = "" /// Whether a real token has been scanned on the current physical line. Reset when the scan diff --git a/Tests/BlockScalarTabContentTests.swift b/Tests/BlockScalarTabContentTests.swift new file mode 100644 index 0000000..b7acf66 --- /dev/null +++ b/Tests/BlockScalarTabContentTests.swift @@ -0,0 +1,23 @@ +@testable import PureYAML +import Testing + +/// A block scalar's indentation is measured in spaces only; a tab past that indentation is literal +/// content, kept verbatim rather than re-expanded to spaces. The surplus whitespace beyond the +/// content indentation is reconstructed from the actual leading characters, so a tab stays a tab. +@Suite("Block scalar tab content") +struct BlockScalarTabContentTests { + @Test("A tab after the content indentation is kept literally") + func test_tabAfterIndent() throws { + #expect(try PureYAML.parse("|\n literal\n \ttext\n") == .string("literal\n\ttext\n")) + } + + @Test("A deeper space indentation still keeps the content tab") + func test_tabAfterDeeperIndent() throws { + #expect(try PureYAML.parse("|\n literal\n \ttext\n") == .string("literal\n\ttext\n")) + } + + @Test("Surplus spaces beyond the indentation are still preserved as spaces") + func test_spaceSurplusUnchanged() throws { + #expect(try PureYAML.parse("|\n text\n more\n") == .string("text\n more\n")) + } +}