Fix AST grammar packaging for Bun installs

CHANGELOG.md

@@ -24,6 +24,9 @@
 - Store: keep content rows referenced by inactive documents during orphan
   cleanup so `qmd update` preserves soft-deleted tombstones for removed
   files. #585
+- Packaging: install AST grammar WASM packages as required dependencies so
+  Bun global installs include TypeScript/TSX/JavaScript grammars, and add a
+  `smoke:package-grammars` verification command. #595
 
 ## [2.1.0] - 2026-04-05
 

bun.lock

@@ -11,6 +11,10 @@
         "node-llama-cpp": "3.18.1",
         "picomatch": "4.0.4",
         "sqlite-vec": "0.1.9",
+        "tree-sitter-go": "0.23.4",
+        "tree-sitter-python": "0.23.4",
+        "tree-sitter-rust": "0.24.0",
+        "tree-sitter-typescript": "0.23.2",
         "web-tree-sitter": "0.26.7",
         "yaml": "2.8.3",
         "zod": "4.2.1",
@@ -26,10 +30,6 @@
         "sqlite-vec-linux-arm64": "0.1.9",
         "sqlite-vec-linux-x64": "0.1.9",
         "sqlite-vec-windows-x64": "0.1.9",
-        "tree-sitter-go": "0.23.4",
-        "tree-sitter-python": "0.23.4",
-        "tree-sitter-rust": "0.24.0",
-        "tree-sitter-typescript": "0.23.2",
       },
       "peerDependencies": {
         "typescript": "^5.9.3",
@@ -509,7 +509,7 @@
 
     "node-abi": ["node-abi@3.87.0", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-+CGM1L1CgmtheLcBuleyYOn7NWPVu0s0EJH2C4puxgEZb9h8QpR9G2dBfZJOAUhi7VQxuBPMd0hiISWcTyiYyQ=="],
 
-    "node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
+    "node-addon-api": ["node-addon-api@8.7.0", "", {}, "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA=="],
 
     "node-api-headers": ["node-api-headers@1.8.0", "", {}, "sha512-jfnmiKWjRAGbdD1yQS28bknFM1tbHC1oucyuMPjmkEs+kpiu76aRs40WlTmBmyEgzDM76ge1DQ7XJ3R5deiVjQ=="],
 
@@ -773,8 +773,6 @@
 
     "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
 
-    "node-llama-cpp/node-addon-api": ["node-addon-api@8.7.0", "", {}, "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA=="],
-
     "ora/cli-spinners": ["cli-spinners@3.4.0", "", {}, "sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw=="],
 
     "postcss/nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
@@ -793,6 +791,16 @@
 
     "tinyglobby/picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="],
 
+    "tree-sitter-go/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
+
+    "tree-sitter-javascript/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
+
+    "tree-sitter-python/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
+
+    "tree-sitter-rust/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
+
+    "tree-sitter-typescript/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
+
     "vite/picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="],
 
     "vitest/picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="],

package.json

@@ -17,6 +17,7 @@
   "files": [
     "bin/",
     "dist/",
+    "scripts/check-package-grammars.mjs",
     "LICENSE",
     "CHANGELOG.md"
   ],
@@ -31,7 +32,8 @@
     "vsearch": "tsx src/cli/qmd.ts vsearch",
     "rerank": "tsx src/cli/qmd.ts rerank",
     "inspector": "npx @modelcontextprotocol/inspector tsx src/cli/qmd.ts mcp",
-    "release": "./scripts/release.sh"
+    "release": "./scripts/release.sh",
+    "smoke:package-grammars": "node scripts/check-package-grammars.mjs"
   },
   "publishConfig": {
     "access": "public"
@@ -53,18 +55,18 @@
     "sqlite-vec": "0.1.9",
     "web-tree-sitter": "0.26.7",
     "yaml": "2.8.3",
-    "zod": "4.2.1"
+    "zod": "4.2.1",
+    "tree-sitter-go": "0.23.4",
+    "tree-sitter-python": "0.23.4",
+    "tree-sitter-rust": "0.24.0",
+    "tree-sitter-typescript": "0.23.2"
   },
   "optionalDependencies": {
     "sqlite-vec-darwin-arm64": "0.1.9",
     "sqlite-vec-darwin-x64": "0.1.9",
     "sqlite-vec-linux-arm64": "0.1.9",
     "sqlite-vec-linux-x64": "0.1.9",
-    "sqlite-vec-windows-x64": "0.1.9",
+    "sqlite-vec-windows-x64": "0.1.9"
-    "tree-sitter-go": "0.23.4",
-    "tree-sitter-python": "0.23.4",
-    "tree-sitter-rust": "0.24.0",
-    "tree-sitter-typescript": "0.23.2"
   },
   "devDependencies": {
     "@types/better-sqlite3": "7.6.13",

scripts/check-package-grammars.mjs

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+
+const grammars = [
+  "tree-sitter-typescript/tree-sitter-typescript.wasm",
+  "tree-sitter-typescript/tree-sitter-tsx.wasm",
+  "tree-sitter-python/tree-sitter-python.wasm",
+  "tree-sitter-go/tree-sitter-go.wasm",
+  "tree-sitter-rust/tree-sitter-rust.wasm",
+];
+
+let ok = true;
+for (const grammar of grammars) {
+  try {
+    const resolved = require.resolve(grammar);
+    console.log(`ok ${grammar} -> ${resolved}`);
+  } catch (err) {
+    ok = false;
+    console.error(`missing ${grammar}`);
+    console.error(err instanceof Error ? err.message : String(err));
+  }
+}
+
+if (!ok) {
+  console.error("\nAST grammar package smoke check failed. Run `bun install` locally or repair a broken global install with the matching `bun add tree-sitter-...@<version>` command shown by `qmd status`.");
+  process.exit(1);
+}

src/ast.ts

@@ -63,15 +63,22 @@ export function detectLanguage(filepath: string): SupportedLanguage | null {
 /**
  * Maps language to the npm package and wasm filename for the grammar.
  */
-const GRAMMAR_MAP: Record<SupportedLanguage, { pkg: string; wasm: string }> = {
+const GRAMMAR_MAP: Record<SupportedLanguage, { pkg: string; wasm: string; version: string }> = {
-  typescript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm" },
+  typescript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm", version: "0.23.2" },
-  tsx:        { pkg: "tree-sitter-typescript", wasm: "tree-sitter-tsx.wasm" },
+  tsx:        { pkg: "tree-sitter-typescript", wasm: "tree-sitter-tsx.wasm",        version: "0.23.2" },
-  javascript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm" },
+  javascript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm", version: "0.23.2" },
-  python:     { pkg: "tree-sitter-python",     wasm: "tree-sitter-python.wasm" },
+  python:     { pkg: "tree-sitter-python",     wasm: "tree-sitter-python.wasm",     version: "0.23.4" },
-  go:         { pkg: "tree-sitter-go",         wasm: "tree-sitter-go.wasm" },
+  go:         { pkg: "tree-sitter-go",         wasm: "tree-sitter-go.wasm",         version: "0.23.4" },
-  rust:       { pkg: "tree-sitter-rust",        wasm: "tree-sitter-rust.wasm" },
+  rust:       { pkg: "tree-sitter-rust",       wasm: "tree-sitter-rust.wasm",       version: "0.24.0" },
 };
 
+export function formatGrammarLoadError(language: SupportedLanguage, err: unknown): string {
+  const grammar = GRAMMAR_MAP[language];
+  const detail = err instanceof Error ? err.message : String(err);
+  return `${grammar.pkg}/${grammar.wasm} failed to load (${detail}); falling back to regex chunking. ` +
+    `Repair a broken global install with: bun add ${grammar.pkg}@${grammar.version}`;
+}
+
 // =============================================================================
 // Per-Language Query Definitions
 // =============================================================================
@@ -176,6 +183,9 @@ let initPromise: Promise<void> | null = null;
 /** Languages that have already failed to load — warn only once per process. */
 const failedLanguages = new Set<string>();
 
+/** Last grammar load error by language, for status output. */
+const grammarLoadErrors = new Map<SupportedLanguage, string>();
+
 /** Cached grammar load promises. */
 const grammarCache = new Map<string, Promise<LanguageType>>();
 
@@ -228,7 +238,9 @@ async function loadGrammar(language: SupportedLanguage): Promise<LanguageType |
   } catch (err) {
     failedLanguages.add(language);
     grammarCache.delete(wasmKey);
-    console.warn(`[qmd] Failed to load tree-sitter grammar for ${language}: ${err}`);
+    const message = formatGrammarLoadError(language, err);
+    grammarLoadErrors.set(language, message);
+    console.warn(`[qmd] AST grammar unavailable for ${language}: ${message}`);
     return null;
   }
 }
@@ -345,7 +357,7 @@ export async function getASTStatus(): Promise<{
         getQuery(lang, grammar);
         languages.push({ language: lang, available: true });
       } else {
-        languages.push({ language: lang, available: false, error: "grammar failed to load" });
+        languages.push({ language: lang, available: false, error: grammarLoadErrors.get(lang) ?? "grammar failed to load" });
       }
     } catch (err) {
       languages.push({

test/ast.test.ts

@@ -6,7 +6,7 @@
  */
 
 import { describe, test, expect } from "vitest";
-import { detectLanguage, getASTBreakPoints, extractSymbols } from "../src/ast.js";
+import { detectLanguage, getASTBreakPoints, extractSymbols, formatGrammarLoadError } from "../src/ast.js";
 import type { SupportedLanguage } from "../src/ast.js";
 
 // =============================================================================
@@ -315,6 +315,16 @@ describe("getASTBreakPoints - error handling", () => {
     // Should either return some partial break points or empty array — not throw
     expect(Array.isArray(points)).toBe(true);
   });
+
+  test("explains missing grammar packages with a repair command", () => {
+    const msg = formatGrammarLoadError(
+      "typescript",
+      new Error("Cannot find module 'tree-sitter-typescript/tree-sitter-typescript.wasm'"),
+    );
+    expect(msg).toContain("tree-sitter-typescript");
+    expect(msg).toContain("bun add tree-sitter-typescript@0.23.2");
+    expect(msg).toContain("falling back to regex");
+  });
 });
 
 // =============================================================================

test/package.test.ts

@@ -0,0 +1,27 @@
+import { describe, expect, test } from "vitest";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+const root = new URL("..", import.meta.url);
+const pkg = JSON.parse(readFileSync(new URL("package.json", root), "utf8"));
+
+describe("package grammar distribution", () => {
+  test("installs AST grammar wasm packages as required runtime dependencies", () => {
+    for (const dep of ["tree-sitter-typescript", "tree-sitter-python", "tree-sitter-go", "tree-sitter-rust"]) {
+      expect(pkg.dependencies, `${dep} should be a required dependency`).toHaveProperty(dep);
+      expect(pkg.optionalDependencies ?? {}, `${dep} should not be optional`).not.toHaveProperty(dep);
+    }
+  });
+
+  test("documents a packaging smoke check for grammar wasm availability", () => {
+    expect(pkg.scripts, "package.json scripts").toHaveProperty("smoke:package-grammars");
+    expect(String(pkg.scripts["smoke:package-grammars"])).toContain("check-package-grammars");
+
+    expect(pkg.files, "published package files").toContain("scripts/check-package-grammars.mjs");
+
+    const scriptPath = join(root.pathname, "scripts", "check-package-grammars.mjs");
+    const script = readFileSync(scriptPath, "utf8");
+    expect(script).toContain("tree-sitter-typescript/tree-sitter-typescript.wasm");
+    expect(script).toContain("tree-sitter-typescript/tree-sitter-tsx.wasm");
+  });
+});