$ node scripts/prove-scenario.ts scripts/scenarios/usage-figures-accessibility.e2e.ts --base 45dbb4436c07001e3bac81c912e63c54c4d9b0af --head a62e473f7c6fc1c4e549045f0044e1c4e4267b47 --upload-command 'ghdrop upload' ⊘ cache disabled Scope: all 42 workspace projects ✓ Lockfile passes supply-chain policies (verified 30m ago) Lockfile is up to date, resolution step is skipped Packages are hard linked from the content-addressable store to the virtual store. Content-addressable store is at: /.pnpm-store/v11 Virtual store is at: node_modules/.pnpm Progress: resolved 1210, reused 1210, downloaded 0, added 1194 Progress: resolved 1773, reused 1773, downloaded 0, added 1773 Packages: +1780 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Progress: resolved 1780, reused 1776, downloaded 0, added 1780, done Agent skills awaiting approval: e2e. Run "pnpm approve" to review them. . prepare$ vp config --no-agent . prepare: Done devDependencies: + @coderabbitai/config 1.2.0 + @effect/tsgo 0.67.1 + @shadcn/lint 0.1.5 + @types/node 24.12.4 + knip 6.34.0 + vite-plus 1.4.0 Done in 3.2s using pnpm v12.14.0 RUN e2e v0.23.0 /scripts run 01a1dfaf-0682-7507-8b0e-0897de769af7 · targets: web, ios ✓ target "web" command at http://127.0.0.1:44073/.well-known/t3/environment ready 4.01s × |web| scenarios/usage-figures-accessibility.e2e.ts > Usage figures expose individually navigable labels and values 11.40s → expected 0 to be greater than or equal to 4 ❯ |web| scenarios/usage-figures-accessibility.e2e.ts (1 test | 1 failed) 11.40s ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ FAIL |web| scenarios/usage-figures-accessibility.e2e.ts > Usage figures expose individually navigable labels and values ASSERTION_FAILED: expected 0 to be greater than or equal to 4 ❯ scenarios/usage-figures-accessibility.e2e.ts:35:25 33| const definitions = await dialog.getByRole('definition').count(); 34| writeFileSync(out+'-result.json', JSON.stringify({sha,terms,defin… 35| await expect(terms).toBeGreaterThanOrEqual(4); | ^ 36| await expect(definitions).toBeGreaterThanOrEqual(4); 37| await expect(overview).toContain('- term: Processed tokens'); ❯ at http://127.0.0.1:44073/usage ❯ trace .e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/trace.md ❯ video .e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/video/video.webm ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed | 0 passed (1) Tests 1 failed | 0 passed (1) Start at 14:49:12 Duration 16.28s (startup 4.01s) Traces .e2e/prove/base/results/ (1 page, trace.md in each test's directory) Screenshots .e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/001-model-stat-tiles.png .e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/002-usage-summary-and-totals.png Markdown .e2e/prove/base/summary.md Failed × |web| scenarios/usage-figures-accessibility.e2e.ts:35 > Usage figures expose individually navigable labels and values ASSERTION_FAILED: expected 0 to be greater than or equal to 4 trace .e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/trace.md Scope: all 42 workspace projects ✓ Lockfile passes supply-chain policies (verified 30m ago) Lockfile is up to date, resolution step is skipped Packages are hard linked from the content-addressable store to the virtual store. Content-addressable store is at: /.pnpm-store/v11 Virtual store is at: node_modules/.pnpm Progress: resolved 1375, reused 1375, downloaded 0, added 1359 Packages: +1780 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Progress: resolved 1780, reused 1776, downloaded 0, added 1780, done Agent skills awaiting approval: e2e. Run "pnpm approve" to review them. . prepare$ vp config --no-agent . prepare: Done devDependencies: + @coderabbitai/config 1.2.0 + @effect/tsgo 0.67.1 + @shadcn/lint 0.1.5 + @types/node 24.12.4 + knip 6.34.0 + vite-plus 1.4.0 Done in 2.8s using pnpm v12.14.0 RUN e2e v0.23.0 /scripts run 01a1dfaf-8220-769d-afc9-7c4c844d5244 · targets: web, ios ✓ target "web" command at http://127.0.0.1:40151/.well-known/t3/environment ready 4.73s ✓ |web| scenarios/usage-figures-accessibility.e2e.ts > Usage figures expose individually navigable labels and values 11.63s ✓ |web| scenarios/usage-figures-accessibility.e2e.ts (1 test) 11.63s Test Files 1 passed (1) Tests 1 passed (1) Start at 14:49:44 Duration 17.36s (startup 4.73s) Traces .e2e/prove/head/results/ (1 page, trace.md in each test's directory) Screenshots .e2e/prove/head/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/001-model-stat-tiles.png .e2e/prove/head/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/002-usage-summary-and-totals.png Markdown .e2e/prove/head/summary.md { "scenario": "Usage figures expose individually navigable labels and values", "failsOnBase": true, "passesOnHead": true, "couldNotRun": null, "baseSha": "45dbb4436c07001e3bac81c912e63c54c4d9b0af", "headSha": "a62e473f7c6fc1c4e549045f0044e1c4e4267b47", "verdict": "Proven: the scenario fails on base `45dbb4436c` and passes on head `a62e473f7c`.", "steps": { "base": [ { "step": "app.open ", "status": "passed" }, { "step": "expect.not.toHaveURL /\\/pair(?:[?#]|$)/", "status": "passed" }, { "step": "app.open /usage", "status": "passed" }, { "step": "locator.tap getByRole(\"button\", name: \"Cost\")", "status": "passed" }, { "step": "locator.tap getByText(\"claude-sonnet-4-5\")", "status": "passed" }, { "step": "app.screenshot model-stat-tiles", "status": "passed" }, { "step": "locator.tap getByRole(\"button\", name: \"Close\").last()", "status": "passed" }, { "step": "app.screenshot usage-summary-and-totals", "status": "passed" }, { "step": "browser.cookies ", "status": "passed" }, { "step": "browser.url ", "status": "passed" } ], "head": [ { "step": "app.open ", "status": "passed" }, { "step": "expect.not.toHaveURL /\\/pair(?:[?#]|$)/", "status": "passed" }, { "step": "app.open /usage", "status": "passed" }, { "step": "locator.tap getByRole(\"button\", name: \"Cost\")", "status": "passed" }, { "step": "locator.tap getByText(\"claude-sonnet-4-5\")", "status": "passed" }, { "step": "app.screenshot model-stat-tiles", "status": "passed" }, { "step": "locator.tap getByRole(\"button\", name: \"Close\").last()", "status": "passed" }, { "step": "app.screenshot usage-summary-and-totals", "status": "passed" }, { "step": "browser.cookies ", "status": "passed" }, { "step": "browser.url ", "status": "passed" } ] }, "captures": [ "/scripts/.e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/001-model-stat-tiles.png", "/scripts/.e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/002-usage-summary-and-totals.png", "/scripts/.e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/003-failure.png", "/scripts/.e2e/prove/head/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/001-model-stat-tiles.png", "/scripts/.e2e/prove/head/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/002-usage-summary-and-totals.png" ], "videos": [ "/scripts/.e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/video/video.webm", "/scripts/.e2e/prove/head/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/video/video.webm" ], "markdown": "### Scenario: Usage figures expose individually navigable labels and values\n\n**Proven: the scenario fails on base `45dbb4436c` and passes on head `a62e473f7c`.**\n\n| | Base `45dbb4436c` | Head `a62e473f7c` |\n| --- | --- | --- |\n| Result | ❌ failed
ASSERTION_FAILED: expected 0 to be greater than or equal to 4 | ✅ passed |\n| model stat tiles | ![Base 45dbb4436c: model stat tiles](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/5af866b02d939492/001-model-stat-tiles.png) | ![Head a62e473f7c: model stat tiles](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/dd95e1bfa4181ce6/001-model-stat-tiles.png) |\n| usage summary and totals | ![Base 45dbb4436c: usage summary and totals](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/f327a124c384dd49/002-usage-summary-and-totals.png) | ![Head a62e473f7c: usage summary and totals](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/ad498e539dbb2551/002-usage-summary-and-totals.png) |\n| failure | ![Base 45dbb4436c: failure](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/f40f900b16c4ba0c/003-failure.png) | |\n", "base": { "sha": "45dbb4436c07001e3bac81c912e63c54c4d9b0af", "status": "failed", "title": "Usage figures expose individually navigable labels and values", "message": "ASSERTION_FAILED: expected 0 to be greater than or equal to 4", "steps": [ { "step": "app.open ", "status": "passed" }, { "step": "expect.not.toHaveURL /\\/pair(?:[?#]|$)/", "status": "passed" }, { "step": "app.open /usage", "status": "passed" }, { "step": "locator.tap getByRole(\"button\", name: \"Cost\")", "status": "passed" }, { "step": "locator.tap getByText(\"claude-sonnet-4-5\")", "status": "passed" }, { "step": "app.screenshot model-stat-tiles", "status": "passed" }, { "step": "locator.tap getByRole(\"button\", name: \"Close\").last()", "status": "passed" }, { "step": "app.screenshot usage-summary-and-totals", "status": "passed" }, { "step": "browser.cookies ", "status": "passed" }, { "step": "browser.url ", "status": "passed" } ], "captures": [ "/scripts/.e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/001-model-stat-tiles.png", "/scripts/.e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/002-usage-summary-and-totals.png", "/scripts/.e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/003-failure.png" ], "videos": [ "/scripts/.e2e/prove/base/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/video/video.webm" ] }, "head": { "sha": "a62e473f7c6fc1c4e549045f0044e1c4e4267b47", "status": "passed", "title": "Usage figures expose individually navigable labels and values", "steps": [ { "step": "app.open ", "status": "passed" }, { "step": "expect.not.toHaveURL /\\/pair(?:[?#]|$)/", "status": "passed" }, { "step": "app.open /usage", "status": "passed" }, { "step": "locator.tap getByRole(\"button\", name: \"Cost\")", "status": "passed" }, { "step": "locator.tap getByText(\"claude-sonnet-4-5\")", "status": "passed" }, { "step": "app.screenshot model-stat-tiles", "status": "passed" }, { "step": "locator.tap getByRole(\"button\", name: \"Close\").last()", "status": "passed" }, { "step": "app.screenshot usage-summary-and-totals", "status": "passed" }, { "step": "browser.cookies ", "status": "passed" }, { "step": "browser.url ", "status": "passed" } ], "captures": [ "/scripts/.e2e/prove/head/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/001-model-stat-tiles.png", "/scripts/.e2e/prove/head/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/screenshots/002-usage-summary-and-totals.png" ], "videos": [ "/scripts/.e2e/prove/head/results/usage-figures-accessibility-usage-figures-expose-c1ecec8671f07fbf/attempt-1/video/video.webm" ] } }