{
  "scenario": "Favorites remain usable when their provider is disabled",
  "failsOnBase": true,
  "passesOnHead": true,
  "baseSha": "3a8c250cb0c8056f2f563091fd412fe5db809fa1",
  "headSha": "2898188e32e8c893afa066d5330049c653523d3d",
  "verdict": "Proven: the scenario fails on base `3a8c250cb0` and passes on head `2898188e32`.",
  "steps": {
    "base": [
      {
        "step": "app.open <pairing link>",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\")",
        "status": "passed"
      },
      {
        "step": "browser.evaluate ",
        "status": "passed"
      },
      {
        "step": "app.open <pairing link>",
        "status": "passed"
      },
      {
        "step": "app.open /settings/providers?instanceId=opencode",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"switch\", name: \"Enable OpenCode\")",
        "status": "passed"
      },
      {
        "step": "locator.scrollIntoView getByRole(\"button\", name: \"Add custom model\")",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\", name: \"Filter models\")",
        "status": "passed"
      },
      {
        "step": "browser.evaluate ",
        "status": "passed"
      },
      {
        "step": "app.screenshot missing-favorite-settings",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"button\", name: \"Remove demo-retired from favorites\")",
        "status": "failed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Close\")",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"dialog\", name: \"Settings\") → hidden",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\")",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Codex\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"option\", name: /Demo active model/)",
        "status": "passed"
      },
      {
        "step": "browser.keyboard.press Escape",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"option\", name: /Demo active model/) → hidden",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot picker-initial",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"option\", name: /Demo active model/)",
        "status": "failed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Codex\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot codex-models-available",
        "status": "passed"
      },
      {
        "step": "browser.keyboard.press Escape",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"option\", name: /Demo active model/) → hidden",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot picker-reopened",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"option\", name: /Demo active model/)",
        "status": "failed"
      }
    ],
    "head": [
      {
        "step": "app.open <pairing link>",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\")",
        "status": "passed"
      },
      {
        "step": "browser.evaluate ",
        "status": "passed"
      },
      {
        "step": "app.open <pairing link>",
        "status": "passed"
      },
      {
        "step": "app.open /settings/providers?instanceId=opencode",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"switch\", name: \"Enable OpenCode\")",
        "status": "passed"
      },
      {
        "step": "locator.scrollIntoView getByRole(\"button\", name: \"Add custom model\")",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\", name: \"Filter models\")",
        "status": "passed"
      },
      {
        "step": "browser.evaluate ",
        "status": "passed"
      },
      {
        "step": "app.screenshot missing-favorite-settings",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"button\", name: \"Remove demo-retired from favorites\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Remove demo-retired from favorites\")",
        "status": "passed"
      },
      {
        "step": "expect.toBeHidden getByRole(\"button\", name: \"Remove demo-retired from favorites\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Close\")",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"dialog\", name: \"Settings\") → hidden",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\")",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Codex\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"option\", name: /Demo active model/)",
        "status": "passed"
      },
      {
        "step": "browser.keyboard.press Escape",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"option\", name: /Demo active model/) → hidden",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot picker-initial",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"option\", name: /Demo active model/)",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Codex\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot codex-models-available",
        "status": "passed"
      },
      {
        "step": "browser.keyboard.press Escape",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"option\", name: /Demo active model/) → hidden",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot picker-reopened",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"option\", name: /Demo active model/)",
        "status": "passed"
      }
    ]
  },
  "captures": [
    {
      "label": "base-001-missing-favorite-settings.png",
      "side": "before",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/937c42cf41225b52/base-001-missing-favorite-settings.png"
    },
    {
      "label": "base-002-picker-initial.png",
      "side": "before",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/20e9277babf2cb79/base-002-picker-initial.png"
    },
    {
      "label": "base-003-codex-models-available.png",
      "side": "before",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/b1768811df779006/base-003-codex-models-available.png"
    },
    {
      "label": "base-004-picker-reopened.png",
      "side": "before",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/4ab057dfc8fc3910/base-004-picker-reopened.png"
    },
    {
      "label": "head-001-missing-favorite-settings.png",
      "side": "after",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/0d5b71d89e173d07/head-001-missing-favorite-settings.png"
    },
    {
      "label": "head-002-picker-initial.png",
      "side": "after",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/8a0af933bd30c7cd/head-002-picker-initial.png"
    },
    {
      "label": "head-003-codex-models-available.png",
      "side": "after",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/c9e5278d61484efe/head-003-codex-models-available.png"
    },
    {
      "label": "head-004-picker-reopened.png",
      "side": "after",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/024adac20162821f/head-004-picker-reopened.png"
    },
    {
      "label": "before-picker.mp4",
      "side": "before",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/ac9d000f5e41b7a4/before-picker.mp4"
    },
    {
      "label": "after-picker.mp4",
      "side": "after",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/547889b16625d77a/after-picker.mp4"
    },
    {
      "label": "after-remove-star.mp4",
      "side": "after",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/958aafd59dfa7af3/after-remove-star.mp4"
    },
    {
      "label": "disabled-provider-favorites.e2e.ts",
      "side": "other",
      "url": "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/c61767b49c9c9828/disabled-provider-favorites.e2e.ts"
    }
  ],
  "videos": [
    "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/ac9d000f5e41b7a4/before-picker.mp4",
    "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/547889b16625d77a/after-picker.mp4"
  ],
  "markdown": "### Evidence\n\nEnable Codex and OpenCode; save only OpenCode favorites, including a synthetic missing slug. Disable OpenCode. In Settings, filter Models for demo-retired and remove its star if available. In an unlocked composer, select Codex / Demo active model, open the picker, switch to Codex, close it, and reopen it.\n\nReal web app, Chromium, 1280×800, light theme, fresh isolated state on each commit. Synthetic preferences were seeded through the app settings RPC; no paid agent turns. Private catalog names are redacted; the synthetic model and empty-Favorites message remain visible.\n\n| | Before [3a8c250cb0](https://github.com/pingdotgg/t3code/commit/3a8c250cb0c8056f2f563091fd412fe5db809fa1) | After [2898188e32](https://github.com/pingdotgg/t3code/commit/2898188e32e8c893afa066d5330049c653523d3d) |\n| --- | --- | --- |\n| Missing favorite in Settings | ![Before: Missing favorite in Settings](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/937c42cf41225b52/base-001-missing-favorite-settings.png) | ![After: Missing favorite in Settings](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/0d5b71d89e173d07/head-001-missing-favorite-settings.png) |\n| Picker opened | ![Before: Picker opened](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/20e9277babf2cb79/base-002-picker-initial.png) | ![After: Picker opened](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/8a0af933bd30c7cd/head-002-picker-initial.png) |\n| Codex models available | ![Before: Codex models available](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/b1768811df779006/base-003-codex-models-available.png) | ![After: Codex models available](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/c9e5278d61484efe/head-003-codex-models-available.png) |\n| Picker reopened | ![Before: Picker reopened](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/4ab057dfc8fc3910/base-004-picker-reopened.png) | ![After: Picker reopened](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/024adac20162821f/head-004-picker-reopened.png) |\n\n[Before: picker interaction](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/ac9d000f5e41b7a4/before-picker.mp4) · [After: picker interaction](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/547889b16625d77a/after-picker.mp4) · [After: remove missing favorite](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/958aafd59dfa7af3/after-remove-star.mp4)\n\n[Scenario and synthetic seed](https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/c61767b49c9c9828/disabled-provider-favorites.e2e.ts)\n\nAI-assisted; verified with the tests and tools listed above.\n",
  "base": {
    "sha": "3a8c250cb0c8056f2f563091fd412fe5db809fa1",
    "status": "failed",
    "title": "Favorites remain usable when their provider is disabled",
    "message": "ASSERTION_FAILED: expect.toBeVisible failed\nlocator: getByRole(\"option\", name: /Demo active model/)\nexpected: visible\nobserved: no node (match count 0)",
    "steps": [
      {
        "step": "app.open <pairing link>",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\")",
        "status": "passed"
      },
      {
        "step": "browser.evaluate ",
        "status": "passed"
      },
      {
        "step": "app.open <pairing link>",
        "status": "passed"
      },
      {
        "step": "app.open /settings/providers?instanceId=opencode",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"switch\", name: \"Enable OpenCode\")",
        "status": "passed"
      },
      {
        "step": "locator.scrollIntoView getByRole(\"button\", name: \"Add custom model\")",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\", name: \"Filter models\")",
        "status": "passed"
      },
      {
        "step": "browser.evaluate ",
        "status": "passed"
      },
      {
        "step": "app.screenshot missing-favorite-settings",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"button\", name: \"Remove demo-retired from favorites\")",
        "status": "failed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Close\")",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"dialog\", name: \"Settings\") → hidden",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\")",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Codex\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"option\", name: /Demo active model/)",
        "status": "passed"
      },
      {
        "step": "browser.keyboard.press Escape",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"option\", name: /Demo active model/) → hidden",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot picker-initial",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"option\", name: /Demo active model/)",
        "status": "failed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Codex\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot codex-models-available",
        "status": "passed"
      },
      {
        "step": "browser.keyboard.press Escape",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"option\", name: /Demo active model/) → hidden",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot picker-reopened",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"option\", name: /Demo active model/)",
        "status": "failed"
      }
    ],
    "captures": [
      "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/937c42cf41225b52/base-001-missing-favorite-settings.png",
      "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/20e9277babf2cb79/base-002-picker-initial.png",
      "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/b1768811df779006/base-003-codex-models-available.png",
      "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/4ab057dfc8fc3910/base-004-picker-reopened.png"
    ],
    "videos": [
      "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/ac9d000f5e41b7a4/before-picker.mp4"
    ]
  },
  "head": {
    "sha": "2898188e32e8c893afa066d5330049c653523d3d",
    "status": "passed",
    "title": "Favorites remain usable when their provider is disabled",
    "steps": [
      {
        "step": "app.open <pairing link>",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\")",
        "status": "passed"
      },
      {
        "step": "browser.evaluate ",
        "status": "passed"
      },
      {
        "step": "app.open <pairing link>",
        "status": "passed"
      },
      {
        "step": "app.open /settings/providers?instanceId=opencode",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"switch\", name: \"Enable OpenCode\")",
        "status": "passed"
      },
      {
        "step": "locator.scrollIntoView getByRole(\"button\", name: \"Add custom model\")",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\", name: \"Filter models\")",
        "status": "passed"
      },
      {
        "step": "browser.evaluate ",
        "status": "passed"
      },
      {
        "step": "app.screenshot missing-favorite-settings",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"button\", name: \"Remove demo-retired from favorites\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Remove demo-retired from favorites\")",
        "status": "passed"
      },
      {
        "step": "expect.toBeHidden getByRole(\"button\", name: \"Remove demo-retired from favorites\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Close\")",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"dialog\", name: \"Settings\") → hidden",
        "status": "passed"
      },
      {
        "step": "locator.fill getByRole(\"textbox\")",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Codex\")",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"option\", name: /Demo active model/)",
        "status": "passed"
      },
      {
        "step": "browser.keyboard.press Escape",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"option\", name: /Demo active model/) → hidden",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot picker-initial",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"option\", name: /Demo active model/)",
        "status": "passed"
      },
      {
        "step": "locator.tap getByRole(\"button\", name: \"Codex\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot codex-models-available",
        "status": "passed"
      },
      {
        "step": "browser.keyboard.press Escape",
        "status": "passed"
      },
      {
        "step": "locator.waitFor getByRole(\"option\", name: /Demo active model/) → hidden",
        "status": "passed"
      },
      {
        "step": "locator.tap locator(\"[data-chat-provider-model-picker]\")",
        "status": "passed"
      },
      {
        "step": "app.screenshot picker-reopened",
        "status": "passed"
      },
      {
        "step": "expect.toBeVisible getByRole(\"option\", name: /Demo active model/)",
        "status": "passed"
      }
    ],
    "captures": [
      "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/0d5b71d89e173d07/head-001-missing-favorite-settings.png",
      "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/8a0af933bd30c7cd/head-002-picker-initial.png",
      "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/c9e5278d61484efe/head-003-codex-models-available.png",
      "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/024adac20162821f/head-004-picker-reopened.png"
    ],
    "videos": [
      "https://gh-file-drop-api-prod-mi5fy3sowv63ufte.pinglabs.workers.dev/f/547889b16625d77a/after-picker.mp4"
    ]
  },
  "method": "Real web app, Chromium, 1280×800, light theme, fresh isolated state on each commit. Synthetic preferences were seeded through the app settings RPC; no paid agent turns. Private catalog names are redacted; the synthetic model and empty-Favorites message remain visible.",
  "stepsDescription": "Enable Codex and OpenCode; save only OpenCode favorites, including a synthetic missing slug. Disable OpenCode. In Settings, filter Models for demo-retired and remove its star if available. In an unlocked composer, select Codex / Demo active model, open the picker, switch to Codex, close it, and reopen it."
}