feat(ci): add code coverage pipeline and E2E mocking with aimock

.github/workflows/code-qa.yml

@@ -49,12 +49,34 @@ jobs:
                 include:
                     - os: ubuntu-latest
                       name: ubuntu-latest
+                      codecov-flag: ubuntu
                     - os: windows-latest
                       name: windows-latest
+                      codecov-flag: windows
         steps:
             - name: Checkout code
               uses: actions/checkout@v4
             - name: Setup Node.js and pnpm
               uses: ./.github/actions/setup-node-pnpm
-            - name: Run unit tests
-              run: pnpm test
+            - name: Cache Turbo
+              uses: actions/cache@v4
+              with:
+                  path: .turbo/cache
+                  key: ${{ runner.os }}-turbo-${{ hashFiles('**/pnpm-lock.yaml') }}-${{ github.sha }}
+                  restore-keys: |
+                      ${{ runner.os }}-turbo-${{ hashFiles('**/pnpm-lock.yaml') }}-
+                      ${{ runner.os }}-turbo-
+            - name: Run unit tests with coverage
+              run: pnpm test:coverage
+            - name: Upload coverage to Codecov
+              uses: codecov/codecov-action@v4
+              with:
+                  files: >-
+                      src/coverage/lcov.info,
+                      webview-ui/coverage/lcov.info,
+                      packages/core/coverage/lcov.info,
+                      packages/cloud/coverage/lcov.info,
+                      packages/telemetry/coverage/lcov.info,
+                      apps/cli/coverage/lcov.info
+                  flags: ${{ matrix.codecov-flag }}
+                  token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/e2e.yml

@@ -0,0 +1,26 @@
+name: E2E Tests (Mocked)
+
+on:
+    workflow_dispatch:
+    pull_request:
+        types: [opened, reopened, ready_for_review, synchronize]
+        branches: [main]
+        paths:
+            - "src/**"
+            - "webview-ui/**"
+            - "apps/vscode-e2e/**"
+            - "packages/core/**"
+
+jobs:
+    e2e-mock:
+        runs-on: ubuntu-latest
+        timeout-minutes: 30
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@v4
+            - name: Setup Node.js and pnpm
+              uses: ./.github/actions/setup-node-pnpm
+            - name: Install xvfb
+              run: sudo apt-get install -y xvfb
+            - name: Run mocked E2E tests
+              run: xvfb-run -a pnpm --filter @roo-code/vscode-e2e test:ci:mock

apps/cli/package.json

@@ -13,6 +13,7 @@
 		"lint": "eslint src --ext .ts --max-warnings=0",
 		"check-types": "tsc --noEmit",
 		"test": "vitest run",
+		"test:coverage": "vitest run --coverage",
 		"test:integration": "tsx scripts/integration/run.ts",
 		"build": "tsup",
 		"build:extension": "pnpm --filter roo-cline bundle",
@@ -45,6 +46,7 @@
 		"ink-testing-library": "^4.0.0",
 		"rimraf": "^6.0.1",
 		"tsup": "^8.4.0",
+		"@vitest/coverage-v8": "^3.2.3",
 		"vitest": "^3.2.3"
 	}
 }

apps/cli/vitest.config.ts

@@ -13,5 +13,10 @@ export default defineConfig({
 		watch: false,
 		testTimeout: 120_000, // 2m for integration tests.
 		include: ["src/**/*.test.ts", "src/**/*.test.tsx"],
+		coverage: {
+			provider: "v8",
+			reporter: ["text", "lcov"],
+			exclude: ["**/*.test.ts", "**/*.test.tsx", "**/*.spec.ts", "**/*.spec.tsx", "**/vitest.config.ts"],
+		},
 	},
 })

apps/vscode-e2e/AGENTS.md

@@ -0,0 +1,91 @@
+# E2E Test Fixture Workflow
+
+E2E tests run against `@copilotkit/aimock` (`LLMock`) — a local HTTP server that replays recorded LLM responses. This makes tests free, deterministic, and CI-friendly.
+
+## How aimock matching works
+
+Fixtures are matched by **substring**: `incoming_last_user_message.includes(fixture.match.userMessage)`. A fixture fires if its match string appears _anywhere_ in the last user message of the API request.
+
+**Critical**: the last user message always contains `<environment_details>` with the current time. Never use a match string that includes a timestamp — it will stop matching on the next run.
+
+Record mode uses **record-on-miss**: if an existing fixture already matches a request, aimock serves it and does **not** re-record. Only unmatched requests are proxied to the real API and saved as `openai-*.json` files.
+
+## Adding a fixture for a new test
+
+1. Write the test in `src/suite/`. Use short, stable, unique text in the task prompt.
+
+2. Clear any stale auto-recorded files first (they accumulate across record runs):
+
+    ```sh
+    git clean -fx apps/vscode-e2e/fixtures/
+    ```
+
+    The `-x` flag is required because `openai-*.json` files are gitignored — `git clean -f` alone silently skips them.
+
+3. Record fixtures (requires an OpenRouter API key with credits):
+
+    ```sh
+    OPENROUTER_API_KEY=<key> pnpm --filter @roo-code/vscode-e2e test:record
+    ```
+
+    This proxies unmatched requests to OpenRouter and writes `fixtures/openai-*.json`. Background
+    calls from the extension will also be recorded here — that's expected, ignore them.
+
+4. Find the auto-recorded file for your test:
+
+    ```sh
+    grep -l "your unique prompt text" apps/vscode-e2e/fixtures/openai-*.json
+    ```
+
+5. Inspect it to find the `response` block (tool calls the LLM made).
+
+6. Create a named fixture file, e.g. `fixtures/my-feature.json`, with a **short stable match string**:
+
+    ```json
+    {
+    	"fixtures": [
+    		{
+    			"match": { "userMessage": "your unique prompt text" },
+    			"response": {
+    				"toolCalls": [
+    					{ "name": "attempt_completion", "arguments": "{\"result\":\"...\"}", "id": "call_001" }
+    				]
+    			}
+    		}
+    	]
+    }
+    ```
+
+    The match string should be unique enough to identify this request but contain **no timestamps, file paths, or environment details**.
+
+7. Delete the `openai-*.json` files — they're gitignored and can't be replayed.
+
+8. Verify in mock mode (no API key needed):
+    ```sh
+    pnpm --filter @roo-code/vscode-e2e test:ci:mock
+    ```
+
+## Multi-turn tests
+
+If the LLM calls a tool first (e.g. `read_file`) and then calls `attempt_completion` after seeing the result, you need two fixtures:
+
+- **Turn 1**: match on the task prompt → respond with the tool call
+- **Turn 2**: match on a stable part of the tool _result_ → respond with `attempt_completion`
+
+The tool result is provided by the extension (not the mock), so its content is deterministic if test files have stable names. Use a stable substring from the tool result as the turn-2 match string.
+
+## 404 errors in logs are expected
+
+Background API calls from the extension (usage collection, initialization) hit aimock with no matching fixture and return 404. These do **not** affect test results — the tests still pass. You'll see `[OpenRouter] API error: { message: '404 No fixture matched' }` in the output; this is normal.
+
+## Running tests
+
+| Command                                                                   | Purpose                                                            |
+| ------------------------------------------------------------------------- | ------------------------------------------------------------------ |
+| `pnpm --filter @roo-code/vscode-e2e test:ci:mock`                         | Replay mode — no API key needed, uses fixtures                     |
+| `OPENROUTER_API_KEY=<key> pnpm --filter @roo-code/vscode-e2e test:record` | Record mode — proxies to real API, writes `openai-*.json`          |
+| `OPENROUTER_API_KEY=<key> pnpm --filter @roo-code/vscode-e2e test:ci`     | Real-API mode — runs against live OpenRouter (for drift detection) |
+
+## Programmatic fixtures (regex matching)
+
+For requests that can't be matched by a stable substring (e.g. "starts with `<environment_details>` but not preceded by a user message"), add a programmatic fixture in `src/runTest.ts` using `mock.addFixture()` with a `RegExp` match. These are only available in replay mode and are not recorded.

apps/vscode-e2e/fixtures/.gitignore

@@ -0,0 +1,3 @@
+# Auto-recorded fixtures have timestamp-based match strings and never replay correctly.
+# Contributors should extract stable fixtures manually from these files, then delete them.
+openai-*.json

apps/vscode-e2e/fixtures/markdown-lists.json

@@ -0,0 +1,60 @@
+{
+	"fixtures": [
+		{
+			"match": {
+				"userMessage": "Please show me an example of an unordered list with the following items: Apple, Banana, Orange"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "attempt_completion",
+						"arguments": "{\"result\":\"Here is an unordered list:\\n- Apple\\n- Banana\\n- Orange\"}",
+						"id": "call_markdown_unordered_001"
+					}
+				]
+			}
+		},
+		{
+			"match": {
+				"userMessage": "Please show me a numbered list with three steps: First step, Second step, Third step"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "attempt_completion",
+						"arguments": "{\"result\":\"Here is a numbered list:\\n1. First step\\n2. Second step\\n3. Third step\"}",
+						"id": "call_markdown_ordered_001"
+					}
+				]
+			}
+		},
+		{
+			"match": {
+				"userMessage": "Please create a nested list with 'Main item' having two sub-items: 'Sub-item A' and 'Sub-item B'"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "attempt_completion",
+						"arguments": "{\"result\":\"Here is a nested list:\\n- Main item\\n  - Sub-item A\\n  - Sub-item B\"}",
+						"id": "call_markdown_nested_001"
+					}
+				]
+			}
+		},
+		{
+			"match": {
+				"userMessage": "Please create a list that has both numbered items and bullet points, mixing ordered and unordered lists"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "attempt_completion",
+						"arguments": "{\"result\":\"Here is a mixed list:\\n1. First numbered item\\n- Bullet point A\\n- Bullet point B\\n2. Second numbered item\"}",
+						"id": "call_markdown_mixed_001"
+					}
+				]
+			}
+		}
+	]
+}

apps/vscode-e2e/fixtures/modes.json

@@ -0,0 +1,18 @@
+{
+	"fixtures": [
+		{
+			"match": {
+				"userMessage": "Use the `switch_mode` tool to switch to ask mode."
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "switch_mode",
+						"arguments": "{\"mode_slug\":\"ask\",\"reason\":\"User requested to switch to ask mode.\"}",
+						"id": "call_modes_switch_001"
+					}
+				]
+			}
+		}
+	]
+}

apps/vscode-e2e/fixtures/task-hello-world.json

@@ -0,0 +1,18 @@
+{
+	"fixtures": [
+		{
+			"match": {
+				"userMessage": "Hello world, what is your name? Respond with 'My name is ...'"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "attempt_completion",
+						"arguments": "{\"result\":\"My name is Roo! I'm your AI coding assistant, here to help you with development tasks.\"}",
+						"id": "call_task_hello_world_001"
+					}
+				]
+			}
+		}
+	]
+}

apps/vscode-e2e/package.json

@@ -6,13 +6,16 @@
 		"check-types": "tsc -p tsconfig.esm.json --noEmit",
 		"format": "prettier --write src",
 		"test:ci": "pnpm -w bundle && pnpm --filter @roo-code/vscode-webview build && pnpm test:run",
+		"test:ci:mock": "pnpm -w bundle && pnpm --filter @roo-code/vscode-webview build && USE_MOCK=true pnpm test:run",
+		"test:record": "AIMOCK_RECORD=true pnpm test:ci",
 		"test:run": "rimraf out && tsc -p tsconfig.json && npx dotenvx run -f .env.local -- node ./out/runTest.js",
 		"clean": "rimraf out .turbo"
 	},
 	"devDependencies": {
 		"@roo-code/config-eslint": "workspace:^",
 		"@roo-code/config-typescript": "workspace:^",
 		"@roo-code/types": "workspace:^",
+		"@copilotkit/aimock": "^1.15.1",
 		"@types/mocha": "^10.0.10",
 		"@types/node": "20.x",
 		"@types/vscode": "^1.95.0",