From 55f16460d03694e484e9fe4528d5929f4f834305 Mon Sep 17 00:00:00 2001
From: Tobi Lutke <tobi@shopify.com>
Date: Tue, 10 Mar 2026 13:28:37 -0400
Subject: [PATCH] fix(ci): guard LLM calls in CI and increase test timeouts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add _ciMode flag to LlamaCpp that throws immediately on embedBatch,
generate, expandQuery, and rerank when CI=true — prevents silent 30s
timeouts. Skip MCP HTTP Transport tests in CI (they instantiate a real
LlamaCpp). Bump vitest/bun test timeouts to 60s for slower CI runners.
---
 .github/workflows/ci.yml      | 4 ++--
 .github/workflows/publish.yml | 2 +-
 src/llm.ts                    | 5 +++++
 test/llm.test.ts              | 1 +
 test/mcp.test.ts              | 2 +-
 5 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8bf578e..84c2770 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -34,7 +34,7 @@ jobs:
       - run: npm install
 
       - name: Tests
-        run: npx vitest run --reporter=verbose test/
+        run: npx vitest run --reporter=verbose --testTimeout 60000 test/
         env:
           CI: true
 
@@ -64,7 +64,7 @@ jobs:
       - run: bun install
 
       - name: Tests
-        run: bun test --timeout 30000 --preload ./src/test-preload.ts test/
+        run: bun test --timeout 60000 --preload ./src/test-preload.ts test/
         env:
           CI: true
           DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 972342d..08c753c 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -23,7 +23,7 @@ jobs:
         run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
 
       - run: bun install
-      - run: bun test --timeout 30000 --preload ./src/test-preload.ts test/
+      - run: bun test --timeout 60000 --preload ./src/test-preload.ts test/
         env:
           CI: true
           LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
diff --git a/src/llm.ts b/src/llm.ts
index 7b5b8d0..485f45a 100644
--- a/src/llm.ts
+++ b/src/llm.ts
@@ -405,6 +405,7 @@ function resolveExpandContextSize(configValue?: number): number {
 }
 
 export class LlamaCpp implements LLM {
+  private readonly _ciMode = !!process.env.CI;
   private llama: Llama | null = null;
   private embedModel: LlamaModel | null = null;
   private embedContexts: LlamaEmbeddingContext[] = [];
@@ -854,6 +855,7 @@ export class LlamaCpp implements LLM {
    * Uses Promise.all for parallel embedding - node-llama-cpp handles batching internally
    */
   async embedBatch(texts: string[]): Promise<(EmbeddingResult | null)[]> {
+    if (this._ciMode) throw new Error("LLM operations are disabled in CI (set CI=true)");
     // Ping activity at start to keep models alive during this operation
     this.touchActivity();
 
@@ -912,6 +914,7 @@ export class LlamaCpp implements LLM {
   }
 
   async generate(prompt: string, options: GenerateOptions = {}): Promise<GenerateResult | null> {
+    if (this._ciMode) throw new Error("LLM operations are disabled in CI (set CI=true)");
     // Ping activity at start to keep models alive during this operation
     this.touchActivity();
 
@@ -971,6 +974,7 @@ export class LlamaCpp implements LLM {
   // ==========================================================================
 
   async expandQuery(query: string, options: { context?: string, includeLexical?: boolean, intent?: string } = {}): Promise<Queryable[]> {
+    if (this._ciMode) throw new Error("LLM operations are disabled in CI (set CI=true)");
     // Ping activity at start to keep models alive during this operation
     this.touchActivity();
 
@@ -1067,6 +1071,7 @@ export class LlamaCpp implements LLM {
     documents: RerankDocument[],
     options: RerankOptions = {}
   ): Promise<RerankResult> {
+    if (this._ciMode) throw new Error("LLM operations are disabled in CI (set CI=true)");
     // Ping activity at start to keep models alive during this operation
     this.touchActivity();
 
diff --git a/test/llm.test.ts b/test/llm.test.ts
index a1d0681..b5de9e0 100644
--- a/test/llm.test.ts
+++ b/test/llm.test.ts
@@ -120,6 +120,7 @@ describe("LlamaCpp expand context size config", () => {
 describe("LlamaCpp rerank deduping", () => {
   test("deduplicates identical document texts before scoring", async () => {
     const llm = new LlamaCpp({}) as any;
+    llm._ciMode = false; // allow unit test even in CI (mocked, no real models)
     const rankAll = vi.fn(async (_query: string, docs: string[]) =>
       docs.map((doc) => doc === "shared chunk" ? 0.9 : 0.2)
     );
diff --git a/test/mcp.test.ts b/test/mcp.test.ts
index 1fadd75..24f13cd 100644
--- a/test/mcp.test.ts
+++ b/test/mcp.test.ts
@@ -897,7 +897,7 @@ describe("MCP Server", () => {
 import { startMcpHttpServer, type HttpServerHandle } from "../src/mcp/server";
 import { enableProductionMode } from "../src/store";
 
-describe("MCP HTTP Transport", () => {
+describe.skipIf(!!process.env.CI)("MCP HTTP Transport", () => {
   let handle: HttpServerHandle;
   let baseUrl: string;
   let httpTestDbPath: string;