fix: harden browser WASM benchmark

Make the live benchmark report Swift-measured parse timing, use deterministic browser CI, and deploy only after local and live Pages validation pass.

.github/workflows/wasm-site.yml

@@ -5,6 +5,16 @@ on:
     branches: [main]
     paths:
       - "Package.swift"
+      - "Package.resolved"
+      - "Sources/PureYAMLStreaming/**"
+      - "Sources/PureYAMLStreamingWasmSmoke/**"
+      - "WasmSite/**"
+      - "scripts/build-wasm-site.sh"
+      - ".github/workflows/wasm-site.yml"
+  pull_request:
+    paths:
+      - "Package.swift"
+      - "Package.resolved"
       - "Sources/PureYAMLStreaming/**"
       - "Sources/PureYAMLStreamingWasmSmoke/**"
       - "WasmSite/**"
@@ -14,11 +24,9 @@ on:
 
 permissions:
   contents: read
-  pages: write
-  id-token: write
 
 concurrency:
-  group: pureyaml-streaming-wasm-site
+  group: pureyaml-streaming-wasm-site-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -54,6 +62,19 @@ jobs:
           cp WasmSite/main.js site/main.js
           cp WasmSite/style.css site/style.css
           cp WasmSite/pureyaml-streaming-wasm-smoke.wasm.gz site/pureyaml-streaming-wasm-smoke.wasm.gz
+          cp -R WasmSite/vendor site/vendor
+          mkdir -p site/fixtures
+          python3 - <<'PY'
+          from pathlib import Path
+          fixture = Path("site/fixtures/generated-large.yaml")
+          payload = "x" * 1024
+          with fixture.open("w", encoding="utf-8") as handle:
+              for index in range(3000):
+                  handle.write("---\n")
+                  handle.write(f"name: ci-doc-{index}\n")
+                  handle.write(f"enabled: {'true' if index % 2 == 0 else 'false'}\n")
+                  handle.write(f"payload: {payload}\n")
+          PY
           ls -lh site
 
       - name: Install Playwright Chromium
@@ -76,7 +97,11 @@ jobs:
             const browser = await chromium.launch({ headless: true });
             const page = await browser.newPage();
             await page.goto('http://127.0.0.1:8766/?v=' + Date.now());
-            await page.selectOption('#sampleSelect', { index: 4 });
+            const optionCount = await page.locator('#sampleSelect option').count();
+            if (optionCount < 10) {
+              throw new Error(`Expected at least 10 public YAML samples, got ${optionCount}`);
+            }
+            await page.fill('#urlInput', 'http://127.0.0.1:8766/fixtures/generated-large.yaml');
             await page.click('#runButton');
             await page.waitForFunction(
               () => document.querySelector('#status')?.textContent === 'Passed',
@@ -84,9 +109,24 @@ jobs:
               { timeout: 60000 }
             );
             const output = await page.locator('#output').textContent();
+            const bytes = await page.locator('#bytesMetric').textContent();
+            const parse = await page.locator('#parseMetric').textContent();
             const throughput = await page.locator('#throughputMetric').textContent();
-            if (!output.includes('"ok" : true') && !output.includes('"ok":true')) {
-              throw new Error(output);
+            const documents = await page.locator('#documentsMetric').textContent();
+            const stdoutMatch = output.match(/stdout:\n([\s\S]*?)\n\nstderr:/);
+            const parsed = JSON.parse(stdoutMatch?.[1] || '{}');
+            if (parsed.ok !== true) throw new Error(output);
+            if (parsed.documentCount !== 3000) throw new Error(output);
+            if (parsed.firstName !== 'ci-doc-0') throw new Error(output);
+            if (parsed.lastName !== 'ci-doc-2999') throw new Error(output);
+            if (!Number.isFinite(parsed.parseMilliseconds) || parsed.parseMilliseconds <= 0) {
+              throw new Error(`Expected positive parseMilliseconds, got ${parsed.parseMilliseconds}`);
+            }
+            if (documents !== '3000') {
+              throw new Error(`Expected document metric 3000, got: ${documents}`);
+            }
+            if (bytes === '-' || parse === '-') {
+              throw new Error(`Expected benchmark metrics, got bytes=${bytes}, parse=${parse}`);
             }
             if (!throughput.includes('/s')) {
               throw new Error(`Expected throughput metric, got: ${throughput}`);
@@ -96,16 +136,22 @@ jobs:
           NODE
 
       - name: Configure Pages
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         uses: actions/configure-pages@v5
 
       - name: Upload Pages artifact
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         uses: actions/upload-pages-artifact@v3
         with:
           path: site
 
   deploy:
     needs: build
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
@@ -113,3 +159,47 @@ jobs:
       - name: Deploy to GitHub Pages
         id: deployment
         uses: actions/deploy-pages@v4
+
+      - name: Install Playwright Chromium
+        run: |
+          mkdir -p .build/playwright-live
+          cd .build/playwright-live
+          npm init -y
+          npm install playwright-chromium
+          npx playwright install --with-deps chromium
+
+      - name: Validate live Pages benchmark
+        env:
+          PAGE_URL: ${{ steps.deployment.outputs.page_url }}
+        run: |
+          node <<'NODE'
+          const { chromium } = require('./.build/playwright-live/node_modules/playwright-chromium');
+
+          (async () => {
+            const pageURL = process.env.PAGE_URL;
+            const browser = await chromium.launch({ headless: true });
+            const page = await browser.newPage();
+            await page.goto(pageURL + '?v=' + Date.now());
+            await page.fill('#urlInput', new URL('fixtures/generated-large.yaml', pageURL).href);
+            await page.click('#runButton');
+            await page.waitForFunction(
+              () => document.querySelector('#status')?.textContent === 'Passed',
+              null,
+              { timeout: 60000 }
+            );
+            const output = await page.locator('#output').textContent();
+            const throughput = await page.locator('#throughputMetric').textContent();
+            const stdoutMatch = output.match(/stdout:\n([\s\S]*?)\n\nstderr:/);
+            const parsed = JSON.parse(stdoutMatch?.[1] || '{}');
+            if (parsed.documentCount !== 3000) throw new Error(output);
+            if (parsed.firstName !== 'ci-doc-0') throw new Error(output);
+            if (parsed.lastName !== 'ci-doc-2999') throw new Error(output);
+            if (!Number.isFinite(parsed.parseMilliseconds) || parsed.parseMilliseconds <= 0) {
+              throw new Error(`Expected positive parseMilliseconds, got ${parsed.parseMilliseconds}`);
+            }
+            if (!throughput.includes('/s')) {
+              throw new Error(`Expected live throughput metric, got: ${throughput}`);
+            }
+            await browser.close();
+          })();
+          NODE

.github/workflows/wasm.yml

@@ -25,7 +25,7 @@ jobs:
           . "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh"
           swiftly install --use 6.3.2
           swiftly run swift sdk install https://download.swift.org/swift-6.3.2-release/wasm-sdk/swift-6.3.2-RELEASE/swift-6.3.2-RELEASE_wasm.artifactbundle.tar.gz --checksum a61f0584c93283589f8b2f42db05c1f9a182b506c2957271402992655591dd7c +6.3.2
-      - name: Build WASM smoke site
+      - name: Build WASM benchmark site
         run: |
           . "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh"
           bash scripts/build-wasm-site.sh

.gitignore

@@ -3,3 +3,4 @@
 .DS_Store
 WasmSite/*.wasm
 WasmSite/*.wasm.gz
+WasmSite/vendor/

README.md

@@ -55,23 +55,31 @@ Live site:
 
 `https://mihaelamj.github.io/PureYAMLStreaming/`
 
-Build the Swift WASI smoke executable and copy it into the static test site:
+Build the Swift WASI benchmark executable and copy it into the static test site:
 
 ```sh
 bash scripts/build-wasm-site.sh
 ```
 
+The build script requires `npm` so it can stage the pinned browser WASI shim as
+a local `WasmSite/vendor/` asset. The deployed page does not import the shim from
+a runtime CDN.
+
 Serve the site locally:
 
 ```sh
 bash scripts/serve-wasm-site.sh
 ```
 
-Then open `http://localhost:8080` and click **Run WASM Test**. The page loads
-`pureyaml-streaming-wasm-smoke.wasm`, lets you choose from 10+ public real-world
-YAML files, fetches the selected file in JavaScript, feeds the bytes to Swift
-through WASI stdin, and reports fetch time, parse time, throughput, document
-count, and stdout JSON.
+Then open `http://localhost:8080` and click **Run Benchmark**. The page loads
+`pureyaml-streaming-wasm-smoke.wasm`, lets you choose from 10+ public
+multi-megabyte YAML files, fetches the selected file in JavaScript, feeds the
+bytes to Swift through WASI stdin, and reports fetch time, parse time,
+throughput, document count, and stdout JSON.
+
+The browser fetch step currently buffers the selected YAML file before invoking
+the WASM guest. Inside Swift, the parser path counts documents through
+`parseDocuments` callbacks rather than retaining all parsed documents.
 
 ## Relationship To PureYAML
 

Sources/PureYAMLStreamingWasmSmoke/main.swift

@@ -8,6 +8,7 @@ struct SmokeResult: Encodable {
     var documentCount: Int
     var firstName: String?
     var lastName: String?
+    var parseMilliseconds: Double
     var message: String
 }
 
@@ -32,22 +33,33 @@ let fallbackYAML = Data(
 )
 
 do {
-    let input = CommandLine.arguments.count > 1
+    let readsStandardInput = CommandLine.arguments.count > 1
+    let input = readsStandardInput
         ? FileHandle.standardInput.readDataToEndOfFile()
         : Data()
-    let yaml = input.isEmpty ? fallbackYAML : input
+    let yaml = readsStandardInput ? input : fallbackYAML
     let parser = PureYAMLStreaming.Parser(chunkSize: 31)
-    let documents = try parser.collectDocuments(data: yaml)
-    let names = documents.compactMap { document in
-        stringValue("name", in: document.value)
+    let parseStart = Date()
+    var documentCount = 0
+    var firstName: String?
+    var lastName: String?
+    try parser.parseDocuments(data: yaml) { document in
+        let name = stringValue("name", in: document.value)
+        if documentCount == 0 {
+            firstName = name
+        }
+        lastName = name
+        documentCount += 1
     }
-    let ok = !documents.isEmpty
+    let parseMilliseconds = Date().timeIntervalSince(parseStart) * 1_000
+    let ok = documentCount > 0
     let result = SmokeResult(
         ok: ok,
         inputByteCount: yaml.count,
-        documentCount: documents.count,
-        firstName: names.first,
-        lastName: names.last,
+        documentCount: documentCount,
+        firstName: firstName,
+        lastName: lastName,
+        parseMilliseconds: parseMilliseconds,
         message: ok ? "PureYAMLStreaming WASM parse passed" : "PureYAMLStreaming WASM parse found no documents",
     )
     print(try renderJSON(result))
@@ -58,6 +70,7 @@ do {
         documentCount: 0,
         firstName: nil,
         lastName: nil,
+        parseMilliseconds: 0,
         message: String(describing: error),
     )
     print((try? renderJSON(result)) ?? #"{"ok":false,"message":"failed to render error"}"#)

WasmSite/index.html

@@ -13,8 +13,8 @@
         <h1>Live WASM YAML Benchmark</h1>
         <p>
           Pick a public real-world YAML file. The browser fetches it, feeds the
-          bytes to the Swift WASI executable through stdin, and times the
-          document-streaming parse live.
+          bytes to the Swift WASI executable through stdin, and reports
+          Swift-measured document parse timing live.
         </p>
         <label for="sampleSelect">Public YAML example</label>
         <select id="sampleSelect"></select>

WasmSite/main.js

@@ -3,7 +3,7 @@ import {
   File,
   OpenFile,
   WASI,
-} from "https://esm.sh/@bjorn3/browser_wasi_shim@0.3.0";
+} from "./vendor/browser_wasi_shim/index.js";
 
 const runButton = document.querySelector("#runButton");
 const sampleSelect = document.querySelector("#sampleSelect");
@@ -21,52 +21,52 @@ let modulePromise = null;
 
 const samples = [
   {
-    name: "GitHub REST API OpenAPI (9.5 MB)",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/github-rest-api.yaml",
+    name: "GitHub REST API OpenAPI (8.39 MiB)",
+    url: "https://api.apis.guru/v2/specs/github.com/1.1.4/openapi.yaml",
   },
   {
-    name: "Zoom OpenAPI (5.0 MB)",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/api-guru-zoom-openapi.yaml",
+    name: "Loket.nl API OpenAPI (10.62 MiB)",
+    url: "https://api.apis.guru/v2/specs/loket.nl/V2/openapi.yaml",
   },
   {
-    name: "Stripe OpenAPI (3.7 MB)",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/api-guru-stripe-openapi.yaml",
+    name: "Zuora Billing API OpenAPI (6.46 MiB)",
+    url: "https://api.apis.guru/v2/specs/zuora.com/2021-08-20/openapi.yaml",
   },
   {
-    name: "cert-manager Helm values (65 KB)",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/cert-manager-values.yaml",
+    name: "Amazon EC2 OpenAPI (5.36 MiB)",
+    url: "https://api.apis.guru/v2/specs/amazonaws.com/ec2/2016-11-15/openapi.yaml",
   },
   {
-    name: "cert-manager CRD (44 KB)",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/cert-manager-certificate-crd.yaml",
+    name: "Zoom API OpenAPI (4.80 MiB)",
+    url: "https://api.apis.guru/v2/specs/zoom.us/2.0.0/openapi.yaml",
   },
   {
-    name: "Kubernetes Prow presubmits (14 KB)",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/prow-cluster-api-presubmits.yaml",
+    name: "UniCourt Enterprise APIs OpenAPI (4.29 MiB)",
+    url: "https://api.apis.guru/v2/specs/unicourt.com/1.0.0/openapi.yaml",
   },
   {
-    name: "Prometheus config (12 KB)",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/prometheus-conf-good.yml",
+    name: "Kubernetes Swagger (4.26 MiB)",
+    url: "https://api.apis.guru/v2/specs/kubernetes.io/unversioned/swagger.yaml",
   },
   {
-    name: "USPTO OpenAPI",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/openapi-uspto.yaml",
+    name: "Autotask PSA Swagger (3.90 MiB)",
+    url: "https://api.apis.guru/v2/specs/autotask.net/v1/swagger.yaml",
   },
   {
-    name: "OpenAPI petstore expanded",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/openapi-petstore-expanded.yaml",
+    name: "Google Document AI Warehouse OpenAPI (3.61 MiB)",
+    url: "https://api.apis.guru/v2/specs/googleapis.com/contentwarehouse/v1/openapi.yaml",
   },
   {
-    name: "Bitnami Apache values",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/bitnami-apache-values.yaml",
+    name: "Stripe API OpenAPI (3.48 MiB)",
+    url: "https://api.apis.guru/v2/specs/stripe.com/2022-11-15/openapi.yaml",
   },
   {
-    name: "GitHub Actions Swift format",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/github-actions-swift-format.yml",
+    name: "Google Compute Engine OpenAPI (3.32 MiB)",
+    url: "https://api.apis.guru/v2/specs/googleapis.com/compute/v1/openapi.yaml",
   },
   {
-    name: "Docker Compose Prometheus/Grafana",
-    url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/awesome-compose-prometheus-grafana-compose-yaml.yaml",
+    name: "DocuSign REST API OpenAPI (3.14 MiB)",
+    url: "https://api.apis.guru/v2/specs/docusign.net/v2.1/openapi.yaml",
   },
 ];
 
@@ -105,7 +105,7 @@ async function runBenchmark() {
   const moduleLoadMS = performance.now() - moduleLoadStart;
 
   const fetchStart = performance.now();
-  const response = await fetch(cacheBustedURL(sourceURL), { cache: "no-store" });
+  const response = await fetch(sourceURL, { cache: "no-store" });
   if (!response.ok) {
     throw new Error(`Failed to fetch YAML: HTTP ${response.status}`);
   }
@@ -130,11 +130,15 @@ async function runBenchmark() {
     wasi_snapshot_preview1: wasi.wasiImport,
   });
   wasi.start(instance);
-  const parseMS = performance.now() - parseStart;
+  const wallClockParseMS = performance.now() - parseStart;
 
   const parsed = parseSmokeOutput(stdoutText);
   const ok = parsed?.ok === true;
-  const throughput = yamlBytes.byteLength / (parseMS / 1000);
+  if (!Number.isFinite(parsed?.parseMilliseconds)) {
+    throw new Error("WASM output did not include a finite Swift parseMilliseconds value.");
+  }
+  const parseMS = parsed.parseMilliseconds;
+  const throughput = parseMS > 0 ? yamlBytes.byteLength / (parseMS / 1000) : 0;
 
   statusElement.className = ok ? "status pass" : "status fail";
   statusElement.textContent = ok ? "Passed" : "Failed";
@@ -147,7 +151,8 @@ async function runBenchmark() {
     `url: ${sourceURL}`,
     `moduleLoad: ${formatDuration(moduleLoadMS)}`,
     `fetch: ${formatDuration(fetchMS)}`,
-    `parse: ${formatDuration(parseMS)}`,
+    `swiftParse: ${formatDuration(parseMS)}`,
+    `processWallClock: ${formatDuration(wallClockParseMS)}`,
     `throughput: ${formatBytes(throughput)}/s`,
     "",
     "stdout:",
@@ -182,12 +187,6 @@ async function loadWasmBytes(url) {
   return new Response(stream).arrayBuffer();
 }
 
-function cacheBustedURL(url) {
-  const parsedURL = new URL(url, location.href);
-  parsedURL.searchParams.set("_pureyaml_bench", `${Date.now()}-${Math.random().toString(36).slice(2)}`);
-  return parsedURL.href;
-}
-
 function parseSmokeOutput(text) {
   try {
     return JSON.parse(text);

scripts/build-wasm-site.sh

@@ -6,6 +6,7 @@ SDK_ID="${PUREYAML_STREAMING_WASM_SDK:-swift-6.3.2-RELEASE_wasm}"
 SWIFT_SELECTOR="${PUREYAML_STREAMING_WASM_SWIFT_SELECTOR:-+6.3.2}"
 PRODUCT="pureyaml-streaming-wasm-smoke"
 SITE_DIR="$ROOT_DIR/WasmSite"
+VENDOR_DIR="$SITE_DIR/vendor/browser_wasi_shim"
 
 swift_command() {
   if command -v swiftly >/dev/null 2>&1; then
@@ -43,6 +44,22 @@ fi
 
 gzip -9 -c "$STAGED_WASM" > "$STAGED_WASM.gz"
 
+if command -v npm >/dev/null 2>&1; then
+  TMP_NPM_DIR="$(mktemp -d)"
+  trap 'rm -rf "$TMP_NPM_DIR"' EXIT
+  (
+    cd "$TMP_NPM_DIR"
+    npm init -y >/dev/null
+    npm install @bjorn3/browser_wasi_shim@0.3.0 >/dev/null
+  )
+  rm -rf "$VENDOR_DIR"
+  mkdir -p "$VENDOR_DIR"
+  cp "$TMP_NPM_DIR/node_modules/@bjorn3/browser_wasi_shim/dist/"*.js "$VENDOR_DIR/"
+else
+  echo "wasm: npm is required to vendor browser_wasi_shim" >&2
+  exit 2
+fi
+
 echo "WASM test site ready:"
 echo "  $SITE_DIR/index.html"
 echo "  $STAGED_WASM.gz"