From 6db34d727890231d925f73e2a59f9ab7b82ac83c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobias=20L=C3=BCtke?= <tobi@shopify.com>
Date: Sun, 5 Apr 2026 17:11:54 -0400
Subject: [PATCH] fix(llm): catch GPU init failures and fall back to CPU

Adds QMD_LLAMA_GPU env var (set to false/off/none to force CPU) and
wraps getLlama() in try/catch so Vulkan/CUDA init failures on headless
or driverless machines fall back gracefully instead of crashing the
node process with an uncatchable C++ terminate().
---
 src/llm.ts | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/src/llm.ts b/src/llm.ts
index 820eee3..dde9548 100644
--- a/src/llm.ts
+++ b/src/llm.ts
@@ -548,11 +548,32 @@ export class LlamaCpp implements LLM {
    */
   private async ensureLlama(): Promise<Llama> {
     if (!this.llama) {
-      const llama = await getLlama({
-        // attempt to build
-        build: "autoAttempt",
-        logLevel: LlamaLogLevel.error
-      });
+      // Allow override via QMD_LLAMA_GPU: "false" | "off" | "none" forces CPU
+      const gpuOverride = (process.env.QMD_LLAMA_GPU ?? "").toLowerCase();
+      const forceCpu = ["false", "off", "none", "disable", "disabled", "0"].includes(gpuOverride);
+
+      const loadLlama = async (gpu: "auto" | false) =>
+        await getLlama({
+          build: "autoAttempt",
+          logLevel: LlamaLogLevel.error,
+          gpu,
+        });
+
+      let llama: Llama;
+      if (forceCpu) {
+        llama = await loadLlama(false);
+      } else {
+        try {
+          llama = await loadLlama("auto");
+        } catch (err) {
+          // GPU backend (e.g. Vulkan on headless/driverless machines) can throw at init.
+          // Fall back to CPU so qmd still works.
+          process.stderr.write(
+            `QMD Warning: GPU init failed (${err instanceof Error ? err.message : String(err)}), falling back to CPU.\n`
+          );
+          llama = await loadLlama(false);
+        }
+      }
 
       if (llama.gpu === false) {
         process.stderr.write(