From 9fd46932d11d20206ac5e5ed88ec9c991b110ccc Mon Sep 17 00:00:00 2001 From: Mihaela Mihaljevic Date: Sat, 6 Jun 2026 23:20:03 +0200 Subject: [PATCH] fix: harden browser WASM benchmark Make the live benchmark report Swift-measured parse timing, use deterministic browser CI, and deploy only after local and live Pages validation pass. --- .github/workflows/wasm-site.yml | 102 ++++++++++++++++-- .github/workflows/wasm.yml | 2 +- .gitignore | 1 + README.md | 20 ++-- Sources/PureYAMLStreamingWasmSmoke/main.swift | 31 ++++-- WasmSite/index.html | 4 +- WasmSite/main.js | 69 ++++++------ scripts/build-wasm-site.sh | 17 +++ 8 files changed, 187 insertions(+), 59 deletions(-) diff --git a/.github/workflows/wasm-site.yml b/.github/workflows/wasm-site.yml index 947af09..bfaee01 100644 --- a/.github/workflows/wasm-site.yml +++ b/.github/workflows/wasm-site.yml @@ -5,6 +5,16 @@ on: branches: [main] paths: - "Package.swift" + - "Package.resolved" + - "Sources/PureYAMLStreaming/**" + - "Sources/PureYAMLStreamingWasmSmoke/**" + - "WasmSite/**" + - "scripts/build-wasm-site.sh" + - ".github/workflows/wasm-site.yml" + pull_request: + paths: + - "Package.swift" + - "Package.resolved" - "Sources/PureYAMLStreaming/**" - "Sources/PureYAMLStreamingWasmSmoke/**" - "WasmSite/**" @@ -14,11 +24,9 @@ on: permissions: contents: read - pages: write - id-token: write concurrency: - group: pureyaml-streaming-wasm-site + group: pureyaml-streaming-wasm-site-${{ github.ref }} cancel-in-progress: true jobs: @@ -54,6 +62,19 @@ jobs: cp WasmSite/main.js site/main.js cp WasmSite/style.css site/style.css cp WasmSite/pureyaml-streaming-wasm-smoke.wasm.gz site/pureyaml-streaming-wasm-smoke.wasm.gz + cp -R WasmSite/vendor site/vendor + mkdir -p site/fixtures + python3 - <<'PY' + from pathlib import Path + fixture = Path("site/fixtures/generated-large.yaml") + payload = "x" * 1024 + with fixture.open("w", encoding="utf-8") as handle: + for index in range(3000): + handle.write("---\n") + handle.write(f"name: ci-doc-{index}\n") + handle.write(f"enabled: {'true' if index % 2 == 0 else 'false'}\n") + handle.write(f"payload: {payload}\n") + PY ls -lh site - name: Install Playwright Chromium @@ -76,7 +97,11 @@ jobs: const browser = await chromium.launch({ headless: true }); const page = await browser.newPage(); await page.goto('http://127.0.0.1:8766/?v=' + Date.now()); - await page.selectOption('#sampleSelect', { index: 4 }); + const optionCount = await page.locator('#sampleSelect option').count(); + if (optionCount < 10) { + throw new Error(`Expected at least 10 public YAML samples, got ${optionCount}`); + } + await page.fill('#urlInput', 'http://127.0.0.1:8766/fixtures/generated-large.yaml'); await page.click('#runButton'); await page.waitForFunction( () => document.querySelector('#status')?.textContent === 'Passed', @@ -84,9 +109,24 @@ jobs: { timeout: 60000 } ); const output = await page.locator('#output').textContent(); + const bytes = await page.locator('#bytesMetric').textContent(); + const parse = await page.locator('#parseMetric').textContent(); const throughput = await page.locator('#throughputMetric').textContent(); - if (!output.includes('"ok" : true') && !output.includes('"ok":true')) { - throw new Error(output); + const documents = await page.locator('#documentsMetric').textContent(); + const stdoutMatch = output.match(/stdout:\n([\s\S]*?)\n\nstderr:/); + const parsed = JSON.parse(stdoutMatch?.[1] || '{}'); + if (parsed.ok !== true) throw new Error(output); + if (parsed.documentCount !== 3000) throw new Error(output); + if (parsed.firstName !== 'ci-doc-0') throw new Error(output); + if (parsed.lastName !== 'ci-doc-2999') throw new Error(output); + if (!Number.isFinite(parsed.parseMilliseconds) || parsed.parseMilliseconds <= 0) { + throw new Error(`Expected positive parseMilliseconds, got ${parsed.parseMilliseconds}`); + } + if (documents !== '3000') { + throw new Error(`Expected document metric 3000, got: ${documents}`); + } + if (bytes === '-' || parse === '-') { + throw new Error(`Expected benchmark metrics, got bytes=${bytes}, parse=${parse}`); } if (!throughput.includes('/s')) { throw new Error(`Expected throughput metric, got: ${throughput}`); @@ -96,16 +136,22 @@ jobs: NODE - name: Configure Pages + if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: actions/configure-pages@v5 - name: Upload Pages artifact + if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: actions/upload-pages-artifact@v3 with: path: site deploy: needs: build + if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest + permissions: + pages: write + id-token: write environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} @@ -113,3 +159,47 @@ jobs: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v4 + + - name: Install Playwright Chromium + run: | + mkdir -p .build/playwright-live + cd .build/playwright-live + npm init -y + npm install playwright-chromium + npx playwright install --with-deps chromium + + - name: Validate live Pages benchmark + env: + PAGE_URL: ${{ steps.deployment.outputs.page_url }} + run: | + node <<'NODE' + const { chromium } = require('./.build/playwright-live/node_modules/playwright-chromium'); + + (async () => { + const pageURL = process.env.PAGE_URL; + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage(); + await page.goto(pageURL + '?v=' + Date.now()); + await page.fill('#urlInput', new URL('fixtures/generated-large.yaml', pageURL).href); + await page.click('#runButton'); + await page.waitForFunction( + () => document.querySelector('#status')?.textContent === 'Passed', + null, + { timeout: 60000 } + ); + const output = await page.locator('#output').textContent(); + const throughput = await page.locator('#throughputMetric').textContent(); + const stdoutMatch = output.match(/stdout:\n([\s\S]*?)\n\nstderr:/); + const parsed = JSON.parse(stdoutMatch?.[1] || '{}'); + if (parsed.documentCount !== 3000) throw new Error(output); + if (parsed.firstName !== 'ci-doc-0') throw new Error(output); + if (parsed.lastName !== 'ci-doc-2999') throw new Error(output); + if (!Number.isFinite(parsed.parseMilliseconds) || parsed.parseMilliseconds <= 0) { + throw new Error(`Expected positive parseMilliseconds, got ${parsed.parseMilliseconds}`); + } + if (!throughput.includes('/s')) { + throw new Error(`Expected live throughput metric, got: ${throughput}`); + } + await browser.close(); + })(); + NODE diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 9775ea6..17a1d74 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -25,7 +25,7 @@ jobs: . "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" swiftly install --use 6.3.2 swiftly run swift sdk install https://download.swift.org/swift-6.3.2-release/wasm-sdk/swift-6.3.2-RELEASE/swift-6.3.2-RELEASE_wasm.artifactbundle.tar.gz --checksum a61f0584c93283589f8b2f42db05c1f9a182b506c2957271402992655591dd7c +6.3.2 - - name: Build WASM smoke site + - name: Build WASM benchmark site run: | . "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" bash scripts/build-wasm-site.sh diff --git a/.gitignore b/.gitignore index d5dc969..361a693 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .DS_Store WasmSite/*.wasm WasmSite/*.wasm.gz +WasmSite/vendor/ diff --git a/README.md b/README.md index 96adb53..ade7806 100644 --- a/README.md +++ b/README.md @@ -55,23 +55,31 @@ Live site: `https://mihaelamj.github.io/PureYAMLStreaming/` -Build the Swift WASI smoke executable and copy it into the static test site: +Build the Swift WASI benchmark executable and copy it into the static test site: ```sh bash scripts/build-wasm-site.sh ``` +The build script requires `npm` so it can stage the pinned browser WASI shim as +a local `WasmSite/vendor/` asset. The deployed page does not import the shim from +a runtime CDN. + Serve the site locally: ```sh bash scripts/serve-wasm-site.sh ``` -Then open `http://localhost:8080` and click **Run WASM Test**. The page loads -`pureyaml-streaming-wasm-smoke.wasm`, lets you choose from 10+ public real-world -YAML files, fetches the selected file in JavaScript, feeds the bytes to Swift -through WASI stdin, and reports fetch time, parse time, throughput, document -count, and stdout JSON. +Then open `http://localhost:8080` and click **Run Benchmark**. The page loads +`pureyaml-streaming-wasm-smoke.wasm`, lets you choose from 10+ public +multi-megabyte YAML files, fetches the selected file in JavaScript, feeds the +bytes to Swift through WASI stdin, and reports fetch time, parse time, +throughput, document count, and stdout JSON. + +The browser fetch step currently buffers the selected YAML file before invoking +the WASM guest. Inside Swift, the parser path counts documents through +`parseDocuments` callbacks rather than retaining all parsed documents. ## Relationship To PureYAML diff --git a/Sources/PureYAMLStreamingWasmSmoke/main.swift b/Sources/PureYAMLStreamingWasmSmoke/main.swift index 8502d98..7a7e7f3 100644 --- a/Sources/PureYAMLStreamingWasmSmoke/main.swift +++ b/Sources/PureYAMLStreamingWasmSmoke/main.swift @@ -8,6 +8,7 @@ struct SmokeResult: Encodable { var documentCount: Int var firstName: String? var lastName: String? + var parseMilliseconds: Double var message: String } @@ -32,22 +33,33 @@ let fallbackYAML = Data( ) do { - let input = CommandLine.arguments.count > 1 + let readsStandardInput = CommandLine.arguments.count > 1 + let input = readsStandardInput ? FileHandle.standardInput.readDataToEndOfFile() : Data() - let yaml = input.isEmpty ? fallbackYAML : input + let yaml = readsStandardInput ? input : fallbackYAML let parser = PureYAMLStreaming.Parser(chunkSize: 31) - let documents = try parser.collectDocuments(data: yaml) - let names = documents.compactMap { document in - stringValue("name", in: document.value) + let parseStart = Date() + var documentCount = 0 + var firstName: String? + var lastName: String? + try parser.parseDocuments(data: yaml) { document in + let name = stringValue("name", in: document.value) + if documentCount == 0 { + firstName = name + } + lastName = name + documentCount += 1 } - let ok = !documents.isEmpty + let parseMilliseconds = Date().timeIntervalSince(parseStart) * 1_000 + let ok = documentCount > 0 let result = SmokeResult( ok: ok, inputByteCount: yaml.count, - documentCount: documents.count, - firstName: names.first, - lastName: names.last, + documentCount: documentCount, + firstName: firstName, + lastName: lastName, + parseMilliseconds: parseMilliseconds, message: ok ? "PureYAMLStreaming WASM parse passed" : "PureYAMLStreaming WASM parse found no documents", ) print(try renderJSON(result)) @@ -58,6 +70,7 @@ do { documentCount: 0, firstName: nil, lastName: nil, + parseMilliseconds: 0, message: String(describing: error), ) print((try? renderJSON(result)) ?? #"{"ok":false,"message":"failed to render error"}"#) diff --git a/WasmSite/index.html b/WasmSite/index.html index c764d83..d969004 100644 --- a/WasmSite/index.html +++ b/WasmSite/index.html @@ -13,8 +13,8 @@

Live WASM YAML Benchmark

Pick a public real-world YAML file. The browser fetches it, feeds the - bytes to the Swift WASI executable through stdin, and times the - document-streaming parse live. + bytes to the Swift WASI executable through stdin, and reports + Swift-measured document parse timing live.

diff --git a/WasmSite/main.js b/WasmSite/main.js index fa4a7d0..854fea8 100644 --- a/WasmSite/main.js +++ b/WasmSite/main.js @@ -3,7 +3,7 @@ import { File, OpenFile, WASI, -} from "https://esm.sh/@bjorn3/browser_wasi_shim@0.3.0"; +} from "./vendor/browser_wasi_shim/index.js"; const runButton = document.querySelector("#runButton"); const sampleSelect = document.querySelector("#sampleSelect"); @@ -21,52 +21,52 @@ let modulePromise = null; const samples = [ { - name: "GitHub REST API OpenAPI (9.5 MB)", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/github-rest-api.yaml", + name: "GitHub REST API OpenAPI (8.39 MiB)", + url: "https://api.apis.guru/v2/specs/github.com/1.1.4/openapi.yaml", }, { - name: "Zoom OpenAPI (5.0 MB)", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/api-guru-zoom-openapi.yaml", + name: "Loket.nl API OpenAPI (10.62 MiB)", + url: "https://api.apis.guru/v2/specs/loket.nl/V2/openapi.yaml", }, { - name: "Stripe OpenAPI (3.7 MB)", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/api-guru-stripe-openapi.yaml", + name: "Zuora Billing API OpenAPI (6.46 MiB)", + url: "https://api.apis.guru/v2/specs/zuora.com/2021-08-20/openapi.yaml", }, { - name: "cert-manager Helm values (65 KB)", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/cert-manager-values.yaml", + name: "Amazon EC2 OpenAPI (5.36 MiB)", + url: "https://api.apis.guru/v2/specs/amazonaws.com/ec2/2016-11-15/openapi.yaml", }, { - name: "cert-manager CRD (44 KB)", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/cert-manager-certificate-crd.yaml", + name: "Zoom API OpenAPI (4.80 MiB)", + url: "https://api.apis.guru/v2/specs/zoom.us/2.0.0/openapi.yaml", }, { - name: "Kubernetes Prow presubmits (14 KB)", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/prow-cluster-api-presubmits.yaml", + name: "UniCourt Enterprise APIs OpenAPI (4.29 MiB)", + url: "https://api.apis.guru/v2/specs/unicourt.com/1.0.0/openapi.yaml", }, { - name: "Prometheus config (12 KB)", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/prometheus-conf-good.yml", + name: "Kubernetes Swagger (4.26 MiB)", + url: "https://api.apis.guru/v2/specs/kubernetes.io/unversioned/swagger.yaml", }, { - name: "USPTO OpenAPI", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/openapi-uspto.yaml", + name: "Autotask PSA Swagger (3.90 MiB)", + url: "https://api.apis.guru/v2/specs/autotask.net/v1/swagger.yaml", }, { - name: "OpenAPI petstore expanded", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/openapi-petstore-expanded.yaml", + name: "Google Document AI Warehouse OpenAPI (3.61 MiB)", + url: "https://api.apis.guru/v2/specs/googleapis.com/contentwarehouse/v1/openapi.yaml", }, { - name: "Bitnami Apache values", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/bitnami-apache-values.yaml", + name: "Stripe API OpenAPI (3.48 MiB)", + url: "https://api.apis.guru/v2/specs/stripe.com/2022-11-15/openapi.yaml", }, { - name: "GitHub Actions Swift format", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/github-actions-swift-format.yml", + name: "Google Compute Engine OpenAPI (3.32 MiB)", + url: "https://api.apis.guru/v2/specs/googleapis.com/compute/v1/openapi.yaml", }, { - name: "Docker Compose Prometheus/Grafana", - url: "https://raw.githubusercontent.com/mihaelamj/PureYAMLGeekbench/main/Fixtures/real-yaml/awesome-compose-prometheus-grafana-compose-yaml.yaml", + name: "DocuSign REST API OpenAPI (3.14 MiB)", + url: "https://api.apis.guru/v2/specs/docusign.net/v2.1/openapi.yaml", }, ]; @@ -105,7 +105,7 @@ async function runBenchmark() { const moduleLoadMS = performance.now() - moduleLoadStart; const fetchStart = performance.now(); - const response = await fetch(cacheBustedURL(sourceURL), { cache: "no-store" }); + const response = await fetch(sourceURL, { cache: "no-store" }); if (!response.ok) { throw new Error(`Failed to fetch YAML: HTTP ${response.status}`); } @@ -130,11 +130,15 @@ async function runBenchmark() { wasi_snapshot_preview1: wasi.wasiImport, }); wasi.start(instance); - const parseMS = performance.now() - parseStart; + const wallClockParseMS = performance.now() - parseStart; const parsed = parseSmokeOutput(stdoutText); const ok = parsed?.ok === true; - const throughput = yamlBytes.byteLength / (parseMS / 1000); + if (!Number.isFinite(parsed?.parseMilliseconds)) { + throw new Error("WASM output did not include a finite Swift parseMilliseconds value."); + } + const parseMS = parsed.parseMilliseconds; + const throughput = parseMS > 0 ? yamlBytes.byteLength / (parseMS / 1000) : 0; statusElement.className = ok ? "status pass" : "status fail"; statusElement.textContent = ok ? "Passed" : "Failed"; @@ -147,7 +151,8 @@ async function runBenchmark() { `url: ${sourceURL}`, `moduleLoad: ${formatDuration(moduleLoadMS)}`, `fetch: ${formatDuration(fetchMS)}`, - `parse: ${formatDuration(parseMS)}`, + `swiftParse: ${formatDuration(parseMS)}`, + `processWallClock: ${formatDuration(wallClockParseMS)}`, `throughput: ${formatBytes(throughput)}/s`, "", "stdout:", @@ -182,12 +187,6 @@ async function loadWasmBytes(url) { return new Response(stream).arrayBuffer(); } -function cacheBustedURL(url) { - const parsedURL = new URL(url, location.href); - parsedURL.searchParams.set("_pureyaml_bench", `${Date.now()}-${Math.random().toString(36).slice(2)}`); - return parsedURL.href; -} - function parseSmokeOutput(text) { try { return JSON.parse(text); diff --git a/scripts/build-wasm-site.sh b/scripts/build-wasm-site.sh index 9752f0b..06768c2 100755 --- a/scripts/build-wasm-site.sh +++ b/scripts/build-wasm-site.sh @@ -6,6 +6,7 @@ SDK_ID="${PUREYAML_STREAMING_WASM_SDK:-swift-6.3.2-RELEASE_wasm}" SWIFT_SELECTOR="${PUREYAML_STREAMING_WASM_SWIFT_SELECTOR:-+6.3.2}" PRODUCT="pureyaml-streaming-wasm-smoke" SITE_DIR="$ROOT_DIR/WasmSite" +VENDOR_DIR="$SITE_DIR/vendor/browser_wasi_shim" swift_command() { if command -v swiftly >/dev/null 2>&1; then @@ -43,6 +44,22 @@ fi gzip -9 -c "$STAGED_WASM" > "$STAGED_WASM.gz" +if command -v npm >/dev/null 2>&1; then + TMP_NPM_DIR="$(mktemp -d)" + trap 'rm -rf "$TMP_NPM_DIR"' EXIT + ( + cd "$TMP_NPM_DIR" + npm init -y >/dev/null + npm install @bjorn3/browser_wasi_shim@0.3.0 >/dev/null + ) + rm -rf "$VENDOR_DIR" + mkdir -p "$VENDOR_DIR" + cp "$TMP_NPM_DIR/node_modules/@bjorn3/browser_wasi_shim/dist/"*.js "$VENDOR_DIR/" +else + echo "wasm: npm is required to vendor browser_wasi_shim" >&2 + exit 2 +fi + echo "WASM test site ready:" echo " $SITE_DIR/index.html" echo " $STAGED_WASM.gz"