fix(parsing): a byte order mark opening the stream is not a column
Some checks failed
rules-freshness / rules (push) Failing after 19s
Some checks failed
rules-freshness / rules (push) Failing after 19s
The mark was skipped but counted, so the first line sat one column right of the rest and a block sequence or a leading comment did not parse. It is now consumed without advancing the column, as libyaml's reader does before the scanner sees a character.
This commit is contained in:
parent
d5b8da8d8b
commit
bdb2ed646e
@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- A stream that begins with a byte order mark (U+FEFF as text; the bytes EF BB BF) reads as the same document as without it. The mark was skipped but counted as a column, so the first line sat one column to the right of every other line: a block sequence (`- a`, `- b`) failed with `did not find expected <document start>`, and a comment on the first line with `found a comment without a preceding space`; a document of one mapping line happened to parse. The mark that opens the stream no longer advances the column, as in libyaml, whose reader takes it as the byte order of the input before the scanner sees a character. Positions on the first line therefore do not count the mark: a caller that turns a line and column into an offset in a string that starts with the mark adds one. A mark at the start of a later line keeps the behaviour libyaml has. Held by `ByteOrderMarkTests` over 15 single documents and 3 streams, through `parse`, `parseLocated`, `parseStream` and `parseStreamLocated`; without the change 25 of those checks fail.
|
||||
|
||||
## [0.4.0] - 2026-11-15
|
||||
|
||||
### Fixed
|
||||
|
||||
@ -91,6 +91,15 @@ extension PureYAML.Parsing.LibYAML {
|
||||
column += 1
|
||||
}
|
||||
|
||||
/// Consumes the byte order mark at the very start of the stream without advancing the column (the index moves, so positions in the text stay the
|
||||
/// text's). Does nothing anywhere else.
|
||||
mutating func skipStreamByteOrderMark() {
|
||||
guard index == 0, index < scalars.count, scalars[index] == "\u{FEFF}" else {
|
||||
return
|
||||
}
|
||||
index += 1
|
||||
}
|
||||
|
||||
/// `SKIP_LINE`: consume a line break and start a new line. A CR+LF pair is
|
||||
/// one break, matching libyaml.
|
||||
mutating func skipLine() {
|
||||
|
||||
@ -13,7 +13,14 @@ extension PureYAML.Parsing.LibYAML.Scanner {
|
||||
while true {
|
||||
tabbedSeparation = false
|
||||
if reader.column == 0, reader.peek() == "\u{FEFF}" {
|
||||
reader.skip()
|
||||
if reader.index == 0 {
|
||||
// The mark that opens the stream is not content and not a column: libyaml's reader takes it as the byte order of the input before the
|
||||
// scanner sees a character, so the first line starts at column 0 with or without it. Counted as a column, a mark pushed the first line
|
||||
// one column right of every other line, and a block sequence or a comment on that line no longer parsed.
|
||||
reader.skipStreamByteOrderMark()
|
||||
} else {
|
||||
reader.skip()
|
||||
}
|
||||
}
|
||||
while reader.peek() == " " || Classify.isTab(reader.peek()) {
|
||||
if Classify.isTab(reader.peek()) {
|
||||
|
||||
41
Tests/ByteOrderMarkTests.swift
Normal file
41
Tests/ByteOrderMarkTests.swift
Normal file
@ -0,0 +1,41 @@
|
||||
@testable import PureYAML
|
||||
import Testing
|
||||
|
||||
/// YAML 1.2.2, section 5.2: a stream may begin with a byte order mark (U+FEFF), which is not part of the content. Through each entry point, a document gives the same
|
||||
/// value, the same nodes and the same positions with the mark as without it; the mark does not count as a column of the first line, as in libyaml, whose reader takes it
|
||||
/// before the scanner sees a character. Before, it was skipped but counted, so a block sequence or a comment on the first line did not parse.
|
||||
@Suite("Byte order mark at the start of a stream")
|
||||
struct ByteOrderMarkTests {
|
||||
static let single = [
|
||||
"a: 1\n", "a: 1\nb:\n - x\n - y\n", "{a: 1}\n", "plain\n", "- a\n- b\n", "- a:\n b: 1\n c: 2\n", "# comment\na: 1\n", "---\na: 1\n", "a: |\n text\n more\n",
|
||||
"? key\n: value\n", "a: &x 1\nb: *x\n", "", "\n", "- - a\n - b\n- c\n", "a:\n- x\n- y\n",
|
||||
]
|
||||
static let streams = ["--- 1\n--- 2\n", "a: 1\n---\nb: 2\n", "# c\n--- a\n...\n--- b\n"]
|
||||
|
||||
@Test("The value read is the same with the mark as without it", arguments: single)
|
||||
func parse(_ body: String) throws {
|
||||
#expect(try PureYAML.parse("\u{FEFF}" + body) == PureYAML.parse(body))
|
||||
}
|
||||
|
||||
@Test("The located value has the same shape and the same positions, the mark not counted as a column", arguments: single)
|
||||
func located(_ body: String) throws {
|
||||
let plain = try PureYAML.parseLocated(body)
|
||||
let marked = try PureYAML.parseLocated("\u{FEFF}" + body)
|
||||
#expect(marked == plain)
|
||||
}
|
||||
|
||||
@Test("A stream of documents reads the same with the mark as without it", arguments: streams)
|
||||
func stream(_ body: String) throws {
|
||||
let plain = try PureYAML.parseStreamLocated(body)
|
||||
let marked = try PureYAML.parseStreamLocated("\u{FEFF}" + body)
|
||||
#expect(marked == plain)
|
||||
#expect(try PureYAML.parseStream("\u{FEFF}" + body).count == PureYAML.parseStream(body).count)
|
||||
}
|
||||
|
||||
@Test("A mark inside a quoted string is the string's own character, written bare or as the escape", arguments: ["a: \"\u{FEFF}1\"\n", "a: \"1\u{FEFF}\"\n"])
|
||||
func insideAString(_ body: String) throws {
|
||||
let escaped = body.replacingOccurrences(of: "\u{FEFF}", with: "\\uFEFF")
|
||||
#expect(try PureYAML.parse(body) == PureYAML.parse(escaped))
|
||||
#expect(try PureYAML.parse("\u{FEFF}" + body) == PureYAML.parse(escaped), "and a mark that opens the stream does not change that")
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user