diff --git a/CHANGELOG.md b/CHANGELOG.md index 54d84bd..cde9802 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ - Store: keep content rows referenced by inactive documents during orphan cleanup so `qmd update` preserves soft-deleted tombstones for removed files. #585 +- Packaging: install AST grammar WASM packages as required dependencies so + Bun global installs include TypeScript/TSX/JavaScript grammars, and add a + `smoke:package-grammars` verification command. #595 ## [2.1.0] - 2026-04-05 diff --git a/bun.lock b/bun.lock index a96f096..651b00f 100644 --- a/bun.lock +++ b/bun.lock @@ -11,6 +11,10 @@ "node-llama-cpp": "3.18.1", "picomatch": "4.0.4", "sqlite-vec": "0.1.9", + "tree-sitter-go": "0.23.4", + "tree-sitter-python": "0.23.4", + "tree-sitter-rust": "0.24.0", + "tree-sitter-typescript": "0.23.2", "web-tree-sitter": "0.26.7", "yaml": "2.8.3", "zod": "4.2.1", @@ -26,10 +30,6 @@ "sqlite-vec-linux-arm64": "0.1.9", "sqlite-vec-linux-x64": "0.1.9", "sqlite-vec-windows-x64": "0.1.9", - "tree-sitter-go": "0.23.4", - "tree-sitter-python": "0.23.4", - "tree-sitter-rust": "0.24.0", - "tree-sitter-typescript": "0.23.2", }, "peerDependencies": { "typescript": "^5.9.3", @@ -509,7 +509,7 @@ "node-abi": ["node-abi@3.87.0", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-+CGM1L1CgmtheLcBuleyYOn7NWPVu0s0EJH2C4puxgEZb9h8QpR9G2dBfZJOAUhi7VQxuBPMd0hiISWcTyiYyQ=="], - "node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="], + "node-addon-api": ["node-addon-api@8.7.0", "", {}, "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA=="], "node-api-headers": ["node-api-headers@1.8.0", "", {}, "sha512-jfnmiKWjRAGbdD1yQS28bknFM1tbHC1oucyuMPjmkEs+kpiu76aRs40WlTmBmyEgzDM76ge1DQ7XJ3R5deiVjQ=="], @@ -773,8 +773,6 @@ "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], - "node-llama-cpp/node-addon-api": ["node-addon-api@8.7.0", "", {}, "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA=="], - "ora/cli-spinners": ["cli-spinners@3.4.0", "", {}, "sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw=="], "postcss/nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], @@ -793,6 +791,16 @@ "tinyglobby/picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], + "tree-sitter-go/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="], + + "tree-sitter-javascript/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="], + + "tree-sitter-python/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="], + + "tree-sitter-rust/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="], + + "tree-sitter-typescript/node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="], + "vite/picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], "vitest/picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], diff --git a/package.json b/package.json index 0ec04c9..59a878a 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "files": [ "bin/", "dist/", + "scripts/check-package-grammars.mjs", "LICENSE", "CHANGELOG.md" ], @@ -31,7 +32,8 @@ "vsearch": "tsx src/cli/qmd.ts vsearch", "rerank": "tsx src/cli/qmd.ts rerank", "inspector": "npx @modelcontextprotocol/inspector tsx src/cli/qmd.ts mcp", - "release": "./scripts/release.sh" + "release": "./scripts/release.sh", + "smoke:package-grammars": "node scripts/check-package-grammars.mjs" }, "publishConfig": { "access": "public" @@ -53,18 +55,18 @@ "sqlite-vec": "0.1.9", "web-tree-sitter": "0.26.7", "yaml": "2.8.3", - "zod": "4.2.1" + "zod": "4.2.1", + "tree-sitter-go": "0.23.4", + "tree-sitter-python": "0.23.4", + "tree-sitter-rust": "0.24.0", + "tree-sitter-typescript": "0.23.2" }, "optionalDependencies": { "sqlite-vec-darwin-arm64": "0.1.9", "sqlite-vec-darwin-x64": "0.1.9", "sqlite-vec-linux-arm64": "0.1.9", "sqlite-vec-linux-x64": "0.1.9", - "sqlite-vec-windows-x64": "0.1.9", - "tree-sitter-go": "0.23.4", - "tree-sitter-python": "0.23.4", - "tree-sitter-rust": "0.24.0", - "tree-sitter-typescript": "0.23.2" + "sqlite-vec-windows-x64": "0.1.9" }, "devDependencies": { "@types/better-sqlite3": "7.6.13", diff --git a/scripts/check-package-grammars.mjs b/scripts/check-package-grammars.mjs new file mode 100644 index 0000000..45d7854 --- /dev/null +++ b/scripts/check-package-grammars.mjs @@ -0,0 +1,29 @@ +#!/usr/bin/env node +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); + +const grammars = [ + "tree-sitter-typescript/tree-sitter-typescript.wasm", + "tree-sitter-typescript/tree-sitter-tsx.wasm", + "tree-sitter-python/tree-sitter-python.wasm", + "tree-sitter-go/tree-sitter-go.wasm", + "tree-sitter-rust/tree-sitter-rust.wasm", +]; + +let ok = true; +for (const grammar of grammars) { + try { + const resolved = require.resolve(grammar); + console.log(`ok ${grammar} -> ${resolved}`); + } catch (err) { + ok = false; + console.error(`missing ${grammar}`); + console.error(err instanceof Error ? err.message : String(err)); + } +} + +if (!ok) { + console.error("\nAST grammar package smoke check failed. Run `bun install` locally or repair a broken global install with the matching `bun add tree-sitter-...@` command shown by `qmd status`."); + process.exit(1); +} diff --git a/src/ast.ts b/src/ast.ts index 5f8194e..a83dbc1 100644 --- a/src/ast.ts +++ b/src/ast.ts @@ -63,15 +63,22 @@ export function detectLanguage(filepath: string): SupportedLanguage | null { /** * Maps language to the npm package and wasm filename for the grammar. */ -const GRAMMAR_MAP: Record = { - typescript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm" }, - tsx: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-tsx.wasm" }, - javascript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm" }, - python: { pkg: "tree-sitter-python", wasm: "tree-sitter-python.wasm" }, - go: { pkg: "tree-sitter-go", wasm: "tree-sitter-go.wasm" }, - rust: { pkg: "tree-sitter-rust", wasm: "tree-sitter-rust.wasm" }, +const GRAMMAR_MAP: Record = { + typescript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm", version: "0.23.2" }, + tsx: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-tsx.wasm", version: "0.23.2" }, + javascript: { pkg: "tree-sitter-typescript", wasm: "tree-sitter-typescript.wasm", version: "0.23.2" }, + python: { pkg: "tree-sitter-python", wasm: "tree-sitter-python.wasm", version: "0.23.4" }, + go: { pkg: "tree-sitter-go", wasm: "tree-sitter-go.wasm", version: "0.23.4" }, + rust: { pkg: "tree-sitter-rust", wasm: "tree-sitter-rust.wasm", version: "0.24.0" }, }; +export function formatGrammarLoadError(language: SupportedLanguage, err: unknown): string { + const grammar = GRAMMAR_MAP[language]; + const detail = err instanceof Error ? err.message : String(err); + return `${grammar.pkg}/${grammar.wasm} failed to load (${detail}); falling back to regex chunking. ` + + `Repair a broken global install with: bun add ${grammar.pkg}@${grammar.version}`; +} + // ============================================================================= // Per-Language Query Definitions // ============================================================================= @@ -176,6 +183,9 @@ let initPromise: Promise | null = null; /** Languages that have already failed to load — warn only once per process. */ const failedLanguages = new Set(); +/** Last grammar load error by language, for status output. */ +const grammarLoadErrors = new Map(); + /** Cached grammar load promises. */ const grammarCache = new Map>(); @@ -228,7 +238,9 @@ async function loadGrammar(language: SupportedLanguage): Promise { // Should either return some partial break points or empty array — not throw expect(Array.isArray(points)).toBe(true); }); + + test("explains missing grammar packages with a repair command", () => { + const msg = formatGrammarLoadError( + "typescript", + new Error("Cannot find module 'tree-sitter-typescript/tree-sitter-typescript.wasm'"), + ); + expect(msg).toContain("tree-sitter-typescript"); + expect(msg).toContain("bun add tree-sitter-typescript@0.23.2"); + expect(msg).toContain("falling back to regex"); + }); }); // ============================================================================= diff --git a/test/package.test.ts b/test/package.test.ts new file mode 100644 index 0000000..018087d --- /dev/null +++ b/test/package.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const root = new URL("..", import.meta.url); +const pkg = JSON.parse(readFileSync(new URL("package.json", root), "utf8")); + +describe("package grammar distribution", () => { + test("installs AST grammar wasm packages as required runtime dependencies", () => { + for (const dep of ["tree-sitter-typescript", "tree-sitter-python", "tree-sitter-go", "tree-sitter-rust"]) { + expect(pkg.dependencies, `${dep} should be a required dependency`).toHaveProperty(dep); + expect(pkg.optionalDependencies ?? {}, `${dep} should not be optional`).not.toHaveProperty(dep); + } + }); + + test("documents a packaging smoke check for grammar wasm availability", () => { + expect(pkg.scripts, "package.json scripts").toHaveProperty("smoke:package-grammars"); + expect(String(pkg.scripts["smoke:package-grammars"])).toContain("check-package-grammars"); + + expect(pkg.files, "published package files").toContain("scripts/check-package-grammars.mjs"); + + const scriptPath = join(root.pathname, "scripts", "check-package-grammars.mjs"); + const script = readFileSync(scriptPath, "utf8"); + expect(script).toContain("tree-sitter-typescript/tree-sitter-typescript.wasm"); + expect(script).toContain("tree-sitter-typescript/tree-sitter-tsx.wasm"); + }); +});