From 1f757379e2152bdc1f446e0fda342899beb0d2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20L=C3=BCtke?= Date: Sat, 16 May 2026 18:37:47 +0000 Subject: [PATCH] Fix GPU status guidance and benchmark warnings --- CHANGELOG.md | 2 ++ src/cli/qmd.ts | 20 ++++++++++++++------ src/llm.ts | 7 +++++-- test/cli.test.ts | 7 +++++-- test/llm.test.ts | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16cd7cf..dde514f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Fixes +- GPU/status: `qmd status` now uses the same embedding model identity as `qmd embed` when computing pending embeddings, so URI-backed embeddings are not incorrectly reported as pending under the legacy `embeddinggemma` alias. +- GPU status: `qmd status` now always shows GPU mode/configuration without unsafe native probing, and CPU-fallback warnings point to `QMD_STATUS_DEVICE_PROBE=1 qmd status` for an actual backend probe. The no-GPU warning is emitted once per process instead of once per LLM instance during benchmarks. - GPU: add `QMD_FORCE_CPU=1` / `--no-gpu` to bypass CUDA/Vulkan/Metal probing entirely, and route native llama.cpp stdout noise to stderr so JSON output stays parseable during search/query commands. - Snippet line numbers: `qmd_query` (MCP), HTTP `/query`, and `qmd query` (CLI JSON output and snippet headers) now return absolute source-file diff --git a/src/cli/qmd.ts b/src/cli/qmd.ts index 25a2a0d..fa01896 100755 --- a/src/cli/qmd.ts +++ b/src/cli/qmd.ts @@ -311,8 +311,8 @@ function formatETA(seconds: number): string { // Check index health and print warnings/tips -function checkIndexHealth(db: Database): void { - const { needsEmbedding, totalDocs, daysStale } = getIndexHealth(db); +function checkIndexHealth(db: Database, model: string = resolveEmbedModelForCli()): void { + const { needsEmbedding, totalDocs, daysStale } = getIndexHealth(db, model); // Warn if many docs need embedding if (needsEmbedding > 0) { @@ -410,7 +410,8 @@ async function showStatus(): Promise { // Overall stats const totalDocs = db.prepare(`SELECT COUNT(*) as count FROM documents WHERE active = 1`).get() as { count: number }; const vectorCount = db.prepare(`SELECT COUNT(*) as count FROM content_vectors`).get() as { count: number }; - const needsEmbedding = getHashesNeedingEmbedding(db); + const statusEmbedModel = resolveEmbedModelForCli(); + const needsEmbedding = getHashesNeedingEmbedding(db, undefined, statusEmbedModel); // Most recent update across all collections const mostRecent = db.prepare(`SELECT MAX(modified_at) as latest FROM documents WHERE active = 1`).get() as { latest: string | null }; @@ -545,9 +546,16 @@ async function showStatus(): Promise { // Device / GPU info // Important: probing node-llama-cpp can abort the whole process on machines with // incompatible GPU drivers (for example Vulkan loader present but no usable driver). - // Keep `qmd status` safe by default and make the expensive/native probe opt-in. - if (process.env.QMD_STATUS_DEVICE_PROBE === "1") { - console.log(`\n${c.bold}Device${c.reset}`); + // Keep the native probe opt-in, but always show how QMD is configured and how to probe. + console.log(`\n${c.bold}Device${c.reset}`); + const configuredGpuMode = process.env.QMD_FORCE_CPU && !["false", "off", "none", "disable", "disabled", "0"].includes(process.env.QMD_FORCE_CPU.trim().toLowerCase()) + ? "CPU forced (QMD_FORCE_CPU)" + : (process.env.QMD_LLAMA_GPU?.trim() || "auto"); + console.log(` Mode: ${configuredGpuMode}`); + if (process.env.QMD_STATUS_DEVICE_PROBE !== "1") { + console.log(` Status: ${c.dim}not probed${c.reset} (set QMD_STATUS_DEVICE_PROBE=1 to test GPU/CPU backend)`); + } else { + console.log(` Status: probing native llama backend...`); try { const llm = getDefaultLlamaCpp(); const device = await llm.getDeviceInfo({ allowBuild: false }); diff --git a/src/llm.ts b/src/llm.ts index bab9e5f..1a6c43b 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -31,6 +31,7 @@ async function loadNodeLlamaCpp(): Promise { export function setNodeLlamaCppModuleForTest(module: NodeLlamaCppModule | null): void { nodeLlamaCppImport = module ? Promise.resolve(module) : null; failedGpuInitModes.clear(); + noGpuAccelerationWarningShown = false; } type StdoutWrite = typeof process.stdout.write; @@ -579,6 +580,7 @@ function resolveExpandContextSize(configValue?: number): number { } const failedGpuInitModes = new Set(); +let noGpuAccelerationWarningShown = false; export class LlamaCpp implements LLM { private readonly _ciMode = !!process.env.CI; @@ -760,9 +762,10 @@ export class LlamaCpp implements LLM { } } - if (llama.gpu === false) { + if (llama.gpu === false && !noGpuAccelerationWarningShown) { + noGpuAccelerationWarningShown = true; process.stderr.write( - "QMD Warning: no GPU acceleration, running on CPU (slow). Run 'qmd status' for details.\n" + "QMD Warning: no GPU acceleration, running on CPU (slow). Run 'QMD_STATUS_DEVICE_PROBE=1 qmd status' for device details.\n" ); } this.llama = llama; diff --git a/test/cli.test.ts b/test/cli.test.ts index 1b551f2..ee9d5f5 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -470,10 +470,13 @@ describe("CLI Status Command", () => { expect(stdout).toContain("Collection"); }); - test("skips device probing by default", async () => { + test("shows device mode without native probing by default", async () => { const { stdout, exitCode } = await runQmd(["status"]); expect(exitCode).toBe(0); - expect(stdout).not.toContain("Device"); + expect(stdout).toContain("Device"); + expect(stdout).toContain("Mode:"); + expect(stdout).toContain("not probed"); + expect(stdout).toContain("QMD_STATUS_DEVICE_PROBE=1"); }); }); diff --git a/test/llm.test.ts b/test/llm.test.ts index 2fc03cd..21e6f66 100644 --- a/test/llm.test.ts +++ b/test/llm.test.ts @@ -178,6 +178,40 @@ describe("native llama stdout containment", () => { else process.env.QMD_FORCE_CPU = prevForceCpu; } }); + + test("warns about CPU fallback only once per process", async () => { + const prevGpu = process.env.QMD_LLAMA_GPU; + const prevForceCpu = process.env.QMD_FORCE_CPU; + process.env.QMD_LLAMA_GPU = "false"; + delete process.env.QMD_FORCE_CPU; + + setNodeLlamaCppModuleForTest({ + LlamaLogLevel: { error: "error" }, + resolveModelFile: vi.fn(), + LlamaChatSession: vi.fn() as any, + getLlama: vi.fn(async () => ({ gpu: false, cpuMathCores: 4 }) as any), + }); + + const stderrSpy = vi.spyOn(process.stderr, "write").mockReturnValue(true); + try { + const first = new LlamaCpp(); + const second = new LlamaCpp(); + + await (first as any).ensureLlama(); + await (second as any).ensureLlama(); + + const stderr = String(stderrSpy.mock.calls.map(call => call[0]).join("")); + expect(stderr.match(/no GPU acceleration/g)?.length).toBe(1); + expect(stderr).toContain("QMD_STATUS_DEVICE_PROBE=1 qmd status"); + } finally { + stderrSpy.mockRestore(); + setNodeLlamaCppModuleForTest(null); + if (prevGpu === undefined) delete process.env.QMD_LLAMA_GPU; + else process.env.QMD_LLAMA_GPU = prevGpu; + if (prevForceCpu === undefined) delete process.env.QMD_FORCE_CPU; + else process.env.QMD_FORCE_CPU = prevForceCpu; + } + }); }); describe("LLM context parallelism safety", () => {