From e8229d8bfb0f59da9a5def5d009a90502abac44b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20L=C3=BCtke?= Date: Sat, 9 May 2026 17:56:28 +0000 Subject: [PATCH] Fix Windows CUDA context parallelism --- CHANGELOG.md | 1 + README.md | 2 ++ src/llm.ts | 44 ++++++++++++++++++++++++++++++++++++++++---- test/llm.test.ts | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4191964..7014380 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Hybrid search: weight RRF lists by query type so original FTS and original vector evidence get the intended 2x boost, instead of accidentally boosting the first lexical expansion. #591 - MCP: seed llama.cpp/GGML quiet env vars before launching `qmd mcp` so native logs cannot pollute stdio JSON-RPC framing. #593 - CLI: remove CommonJS `require()` calls from ESM index path normalization so `qmd --index ` no longer crashes with `ERR_AMBIGUOUS_MODULE_SYNTAX` on Node 22+. #634 +- Windows CUDA: serialize llama.cpp embedding/reranking contexts by default to avoid intermittent `ggml-cuda.cu:98` crashes in `qmd query`; set `QMD_EMBED_PARALLELISM` to opt back into parallel contexts if your driver is stable. #519 - GPU: respect explicit `QMD_LLAMA_GPU=metal|vulkan|cuda` backend overrides instead of always using auto GPU selection. #529 - Fix: preserve original filename case in `handelize()`. The previous `.toLowerCase()` call made indexed paths unreachable on case-sensitive diff --git a/README.md b/README.md index 6f31844..02e4b1e 100644 --- a/README.md +++ b/README.md @@ -797,6 +797,8 @@ llm_cache -- Cached LLM responses (query expansion, rerank scores) | Variable | Default | Description | |----------|---------|-------------| | `XDG_CACHE_HOME` | `~/.cache` | Cache directory location | +| `QMD_LLAMA_GPU` | `auto` | Force llama.cpp GPU backend (`metal`, `vulkan`, `cuda`) or disable GPU with `false` | +| `QMD_EMBED_PARALLELISM` | automatic | Override embedding/reranking context parallelism (1-8). Windows CUDA defaults to `1` because parallel CUDA contexts can crash with `ggml-cuda.cu:98`; use Vulkan or raise this only if your driver is stable. | ## How It Works diff --git a/src/llm.ts b/src/llm.ts index 7d2bbe0..d469d36 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -451,7 +451,41 @@ export type LlamaCppConfig = { const DEFAULT_INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000; const DEFAULT_EXPAND_CONTEXT_SIZE = 2048; -type LlamaGpuMode = "auto" | "metal" | "vulkan" | "cuda" | false; +export type LlamaGpuMode = "auto" | "metal" | "vulkan" | "cuda" | false; + +type ParallelismOptions = { + gpu: string | false; + platform?: NodeJS.Platform; + computed: number; + envValue?: string; +}; + +export function resolveParallelismOverride(envValue = process.env.QMD_EMBED_PARALLELISM): number | undefined { + const normalized = envValue?.trim() ?? ""; + if (!normalized) return undefined; + + const parsed = Number(normalized); + if (!Number.isInteger(parsed) || parsed < 1) { + process.stderr.write(`QMD Warning: invalid QMD_EMBED_PARALLELISM="${envValue}", using automatic parallelism.\n`); + return undefined; + } + + return Math.min(8, parsed); +} + +export function resolveSafeParallelism(options: ParallelismOptions): number { + const override = resolveParallelismOverride(options.envValue); + if (override !== undefined) return override; + + // node-llama-cpp/llama.cpp CUDA on Windows is unstable with multiple + // simultaneous contexts (ggml-cuda.cu:98 in #519). Vulkan and CPU do not + // show the same failure mode, so only serialize Windows CUDA by default. + if ((options.platform ?? process.platform) === "win32" && options.gpu === "cuda") { + return 1; + } + + return Math.max(1, options.computed); +} export function resolveLlamaGpuMode(envValue = process.env.QMD_LLAMA_GPU): LlamaGpuMode { const normalized = envValue?.trim().toLowerCase() ?? ""; @@ -726,16 +760,18 @@ export class LlamaCpp implements LLM { const vram = await llama.getVramState(); const freeMB = vram.free / (1024 * 1024); const maxByVram = Math.floor((freeMB * 0.25) / perContextMB); - return Math.max(1, Math.min(8, maxByVram)); + const computed = Math.max(1, Math.min(8, maxByVram)); + return resolveSafeParallelism({ gpu: llama.gpu, computed }); } catch { - return 2; + return resolveSafeParallelism({ gpu: llama.gpu, computed: 2 }); } } // CPU: split cores across contexts. At least 4 threads per context. const cores = llama.cpuMathCores || 4; const maxContexts = Math.floor(cores / 4); - return Math.max(1, Math.min(4, maxContexts)); + const computed = Math.max(1, Math.min(4, maxContexts)); + return resolveSafeParallelism({ gpu: false, computed }); } /** diff --git a/test/llm.test.ts b/test/llm.test.ts index 74b6430..3678bad 100644 --- a/test/llm.test.ts +++ b/test/llm.test.ts @@ -13,6 +13,8 @@ import { getDefaultLlamaCpp, disposeDefaultLlamaCpp, resolveLlamaGpuMode, + resolveParallelismOverride, + resolveSafeParallelism, withLLMSession, canUnloadLLM, SessionReleasedError, @@ -88,6 +90,44 @@ describe("QMD_LLAMA_GPU resolution", () => { }); }); +describe("LLM context parallelism safety", () => { + test("defaults Windows CUDA to one context to avoid ggml-cuda.cu:98 crashes", () => { + expect(resolveSafeParallelism({ + gpu: "cuda", + platform: "win32", + computed: 8, + envValue: undefined, + })).toBe(1); + }); + + test("keeps non-Windows and non-CUDA backends on computed parallelism", () => { + expect(resolveSafeParallelism({ gpu: "cuda", platform: "linux", computed: 8 })).toBe(8); + expect(resolveSafeParallelism({ gpu: "vulkan", platform: "win32", computed: 8 })).toBe(8); + expect(resolveSafeParallelism({ gpu: false, platform: "win32", computed: 4 })).toBe(4); + }); + + test("QMD_EMBED_PARALLELISM overrides the Windows CUDA safety default", () => { + expect(resolveSafeParallelism({ + gpu: "cuda", + platform: "win32", + computed: 8, + envValue: "2", + })).toBe(2); + }); + + test("QMD_EMBED_PARALLELISM clamps invalid values and warns", () => { + const stderrSpy = vi.spyOn(process.stderr, "write").mockReturnValue(true); + try { + expect(resolveParallelismOverride("0")).toBeUndefined(); + expect(resolveParallelismOverride("bad")).toBeUndefined(); + expect(stderrSpy).toHaveBeenCalledTimes(2); + expect(String(stderrSpy.mock.calls[0]?.[0] || "")).toContain("QMD_EMBED_PARALLELISM"); + } finally { + stderrSpy.mockRestore(); + } + }); +}); + describe("LlamaCpp expand context size config", () => { const defaultExpandContextSize = 2048;