ai-workspace-services/qmd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix AST grammar packaging for Bun installs

Browse Source
This commit is contained in:
Tobi Lütke 2026-05-09 17:53:58 +00:00
parent 004714af48
commit 3f055e705d
No known key found for this signature in database
7 changed files with 115 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -24,6 +24,9 @@
- Store: keep content rows referenced by inactive documents during orphan
cleanup so `qmd update` preserves soft-deleted tombstones for removed
files. #585
- Packaging: install AST grammar WASM packages as required dependencies so
Bun global installs include TypeScript/TSX/JavaScript grammars, and add a
`smoke:package-grammars` verification command. #595
## [2.1.0] - 2026-04-05

22
bun.lock
View File

@ -11,6 +11,10 @@
"node-llama-cpp": "3.18.1",
"picomatch": "4.0.4",
"sqlite-vec": "0.1.9",
"tree-sitter-go": "0.23.4",
"tree-sitter-python": "0.23.4",
"tree-sitter-rust": "0.24.0",
"tree-sitter-typescript": "0.23.2",
"web-tree-sitter": "0.26.7",
"yaml": "2.8.3",
"zod": "4.2.1",
@ -26,10 +30,6 @@
"sqlite-vec-linux-arm64": "0.1.9",
"sqlite-vec-linux-x64": "0.1.9",
"sqlite-vec-windows-x64": "0.1.9",
"tree-sitter-go": "0.23.4",
"tree-sitter-python": "0.23.4",
"tree-sitter-rust": "0.24.0",
"tree-sitter-typescript": "0.23.2",
},
"peerDependencies": {
"typescript": "^5.9.3",
@ -509,7 +509,7 @@
"node-abi": ["node-abi@3.87.0", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-+CGM1L1CgmtheLcBuleyYOn7NWPVu0s0EJH2C4puxgEZb9h8QpR9G2dBfZJOAUhi7VQxuBPMd0hiISWcTyiYyQ=="],
"node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
"node-addon-api": ["node-addon-api@8.7.0", "", {}, "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA=="],
"node-api-headers": ["node-api-headers@1.8.0", "", {}, "sha512-jfnmiKWjRAGbdD1yQS28bknFM1tbHC1oucyuMPjmkEs+kpiu76aRs40WlTmBmyEgzDM76ge1DQ7XJ3R5deiVjQ=="],
@ -773,8 +773,6 @@
"micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
"node-llama-cpp/node-addon-api": ["node-addon-api@8.7.0", "", {}, "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA=="],
"ora/cli-spinners": ["cli-spinners@3.4.0", "", {}, "sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw=="],
"postcss/nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
@ -793,6 +791,16 @@
"tinyglobby/picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="],
"tree-sitter-go/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
"tree-sitter-javascript/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
"tree-sitter-python/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
"tree-sitter-rust/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
"tree-sitter-typescript/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
"vite/picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="],
"vitest/picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="],

16
package.json
View File

@ -17,6 +17,7 @@
"files": [
"bin/",
"dist/",
"scripts/check-package-grammars.mjs",
"LICENSE",
"CHANGELOG.md"
],
@ -31,7 +32,8 @@
"vsearch": "tsx src/cli/qmd.ts vsearch",
"rerank": "tsx src/cli/qmd.ts rerank",
"inspector": "npx @modelcontextprotocol/inspector tsx src/cli/qmd.ts mcp",
"release": "./scripts/release.sh"
"release": "./scripts/release.sh",
"smoke:package-grammars": "node scripts/check-package-grammars.mjs"
},
"publishConfig": {
"access": "public"
@ -53,18 +55,18 @@
"sqlite-vec": "0.1.9",
"web-tree-sitter": "0.26.7",
"yaml": "2.8.3",
"zod": "4.2.1"
"zod": "4.2.1",
"tree-sitter-go": "0.23.4",
"tree-sitter-python": "0.23.4",
"tree-sitter-rust": "0.24.0",
"tree-sitter-typescript": "0.23.2"
},
"optionalDependencies": {
"sqlite-vec-darwin-arm64": "0.1.9",
"sqlite-vec-darwin-x64": "0.1.9",
"sqlite-vec-linux-arm64": "0.1.9",
"sqlite-vec-linux-x64": "0.1.9",
"sqlite-vec-windows-x64": "0.1.9",
"tree-sitter-go": "0.23.4",
"tree-sitter-python": "0.23.4",
"tree-sitter-rust": "0.24.0",
"tree-sitter-typescript": "0.23.2"
"sqlite-vec-windows-x64": "0.1.9"
},
"devDependencies": {
"@types/better-sqlite3": "7.6.13",

29
scripts/check-package-grammars.mjs Normal file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env node
import { createRequire } from "node:module";
const require = createRequire(import.meta.url);
const grammars = [
"tree-sitter-typescript/tree-sitter-typescript.wasm",
"tree-sitter-typescript/tree-sitter-tsx.wasm",
"tree-sitter-python/tree-sitter-python.wasm",
"tree-sitter-go/tree-sitter-go.wasm",
"tree-sitter-rust/tree-sitter-rust.wasm",
];
let ok = true;
for (const grammar of grammars) {
try {
const resolved = require.resolve(grammar);
console.log(`ok ${grammar} -> ${resolved}`);
} catch (err) {
ok = false;
console.error(`missing ${grammar}`);
console.error(err instanceof Error ? err.message : String(err));
}
}
if (!ok) {
console.error("\nAST grammar package smoke check failed. Run `bun install` locally or repair a broken global install with the matching `bun add tree-sitter-...@<version>` command shown by `qmd status`.");
process.exit(1);
}

30
src/ast.ts
View File

@ -63,15 +63,22 @@ export function detectLanguage(filepath: string): SupportedLanguage | null {
/**
* Maps language to the npm package and wasm filename for the grammar.
*/
const GRAMMAR_MAP: Record<SupportedLanguage, { pkg: string; wasm: string }> = {
typescript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm" },
tsx: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-tsx.wasm" },
javascript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm" },
python: { pkg: "tree-sitter-python", wasm: "tree-sitter-python.wasm" },
go: { pkg: "tree-sitter-go", wasm: "tree-sitter-go.wasm" },
rust: { pkg: "tree-sitter-rust", wasm: "tree-sitter-rust.wasm" },
const GRAMMAR_MAP: Record<SupportedLanguage, { pkg: string; wasm: string; version: string }> = {
typescript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm", version: "0.23.2" },
tsx: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-tsx.wasm", version: "0.23.2" },
javascript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm", version: "0.23.2" },
python: { pkg: "tree-sitter-python", wasm: "tree-sitter-python.wasm", version: "0.23.4" },
go: { pkg: "tree-sitter-go", wasm: "tree-sitter-go.wasm", version: "0.23.4" },
rust: { pkg: "tree-sitter-rust", wasm: "tree-sitter-rust.wasm", version: "0.24.0" },
};
export function formatGrammarLoadError(language: SupportedLanguage, err: unknown): string {
const grammar = GRAMMAR_MAP[language];
const detail = err instanceof Error ? err.message : String(err);
return `${grammar.pkg}/${grammar.wasm} failed to load (${detail}); falling back to regex chunking. ` +
`Repair a broken global install with: bun add ${grammar.pkg}@${grammar.version}`;
}
// =============================================================================
// Per-Language Query Definitions
// =============================================================================
@ -176,6 +183,9 @@ let initPromise: Promise<void> | null = null;
/** Languages that have already failed to load — warn only once per process. */
const failedLanguages = new Set<string>();
/** Last grammar load error by language, for status output. */
const grammarLoadErrors = new Map<SupportedLanguage, string>();
/** Cached grammar load promises. */
const grammarCache = new Map<string, Promise<LanguageType>>();
@ -228,7 +238,9 @@ async function loadGrammar(language: SupportedLanguage): Promise<LanguageType |
} catch (err) {
failedLanguages.add(language);
grammarCache.delete(wasmKey);
console.warn(`[qmd] Failed to load tree-sitter grammar for ${language}: ${err}`);
const message = formatGrammarLoadError(language, err);
grammarLoadErrors.set(language, message);
console.warn(`[qmd] AST grammar unavailable for ${language}: ${message}`);
return null;
}
}
@ -345,7 +357,7 @@ export async function getASTStatus(): Promise<{
getQuery(lang, grammar);
languages.push({ language: lang, available: true });
} else {
languages.push({ language: lang, available: false, error: "grammar failed to load" });
languages.push({ language: lang, available: false, error: grammarLoadErrors.get(lang) ?? "grammar failed to load" });
}
} catch (err) {
languages.push({

12
test/ast.test.ts
View File

@ -6,7 +6,7 @@
*/
import { describe, test, expect } from "vitest";
import { detectLanguage, getASTBreakPoints, extractSymbols } from "../src/ast.js";
import { detectLanguage, getASTBreakPoints, extractSymbols, formatGrammarLoadError } from "../src/ast.js";
import type { SupportedLanguage } from "../src/ast.js";
// =============================================================================
@ -315,6 +315,16 @@ describe("getASTBreakPoints - error handling", () => {
// Should either return some partial break points or empty array — not throw
expect(Array.isArray(points)).toBe(true);
});
test("explains missing grammar packages with a repair command", () => {
const msg = formatGrammarLoadError(
"typescript",
new Error("Cannot find module 'tree-sitter-typescript/tree-sitter-typescript.wasm'"),
);
expect(msg).toContain("tree-sitter-typescript");
expect(msg).toContain("bun add tree-sitter-typescript@0.23.2");
expect(msg).toContain("falling back to regex");
});
});
// =============================================================================

27
test/package.test.ts Normal file
View File

@ -0,0 +1,27 @@
import { describe, expect, test } from "vitest";
import { readFileSync } from "node:fs";
import { join } from "node:path";
const root = new URL("..", import.meta.url);
const pkg = JSON.parse(readFileSync(new URL("package.json", root), "utf8"));
describe("package grammar distribution", () => {
test("installs AST grammar wasm packages as required runtime dependencies", () => {
for (const dep of ["tree-sitter-typescript", "tree-sitter-python", "tree-sitter-go", "tree-sitter-rust"]) {
expect(pkg.dependencies, `${dep} should be a required dependency`).toHaveProperty(dep);
expect(pkg.optionalDependencies ?? {}, `${dep} should not be optional`).not.toHaveProperty(dep);
}
});
test("documents a packaging smoke check for grammar wasm availability", () => {
expect(pkg.scripts, "package.json scripts").toHaveProperty("smoke:package-grammars");
expect(String(pkg.scripts["smoke:package-grammars"])).toContain("check-package-grammars");
expect(pkg.files, "published package files").toContain("scripts/check-package-grammars.mjs");
const scriptPath = join(root.pathname, "scripts", "check-package-grammars.mjs");
const script = readFileSync(scriptPath, "utf8");
expect(script).toContain("tree-sitter-typescript/tree-sitter-typescript.wasm");
expect(script).toContain("tree-sitter-typescript/tree-sitter-tsx.wasm");
});
});