diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index fb44112..2411872 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,9 +1,10 @@ -{"id":"qmd-4ru","title":"Update document retrieval for new schema","description":"Functions like getDocument, findDocument, getMultipleDocuments need to work with new schema (path instead of filepath, content joins, virtual paths).","status":"in_progress","priority":0,"issue_type":"task","created_at":"2025-12-12T15:29:53.911881-05:00","updated_at":"2025-12-12T15:30:10.835834-05:00","dependencies":[{"issue_id":"qmd-4ru","depends_on_id":"qmd-ama","type":"discovered-from","created_at":"2025-12-12T15:29:53.912607-05:00","created_by":"daemon"}]} +{"id":"qmd-4ru","title":"Update document retrieval for new schema","description":"Functions like getDocument, findDocument, getMultipleDocuments need to work with new schema (path instead of filepath, content joins, virtual paths).","status":"closed","priority":0,"issue_type":"task","created_at":"2025-12-12T15:29:53.911881-05:00","updated_at":"2025-12-12T15:56:11.054888-05:00","closed_at":"2025-12-12T15:56:11.054888-05:00","dependencies":[{"issue_id":"qmd-4ru","depends_on_id":"qmd-ama","type":"discovered-from","created_at":"2025-12-12T15:29:53.912607-05:00","created_by":"daemon"}]} +{"id":"qmd-afe","title":"implement qmd collection rename, which changes the global path prefix for the collection","description":"","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-12T15:55:54.779325-05:00","updated_at":"2025-12-12T15:55:54.779325-05:00"} {"id":"qmd-ama","title":"Refactor database system","description":"All documents should be stored as content addressable hash, e.g. hash, doc, created_at,\n┃ updated_at. documents should be a file system layer on top e.g. collection, path, hash,\n┃ created_at, updated_at. (collection,path)\n┃\n┃\n\n┃ All documents should be stored as content addressable hash, e.g. hash, doc, created_at,\n┃ updated_at. documents should be a file system layer on top e.g. collection_id, path, hash,\n┃ created_at, updated_at. (collection,path) is unique. There is also collection which stores PWD\n┃ + glob pattern, name (\\w+). Every document is treated as path qmd://collection.name/","notes":"## Completed\n- ✅ Implemented content-addressable storage (content table with hash→doc mapping)\n- ✅ Refactored documents table as file system layer (collection_id, path, hash)\n- ✅ Added collection names (e.g., \"pages\", \"journals\", \"archive\")\n- ✅ Implemented virtual paths (qmd://collection-name/path/to/file.md)\n- ✅ Added hierarchical context support (collection-scoped)\n- ✅ Successfully migrated existing database\n- ✅ Updated search functions to work with new schema\n- ✅ Updated indexing logic to use content-addressable storage\n- ✅ Orphaned content hash cleanup\n\n## Still TODO\n- Fix migration SQL to properly extract basename (currently needs manual fix)\n- Implement `qmd collection add . --name \u003cname\u003e --mask '**/*.md'`\n- Implement `qmd ls [path]` for exploring virtual file tree","status":"closed","priority":2,"issue_type":"task","created_at":"2025-12-10T10:57:35.497489-05:00","updated_at":"2025-12-12T15:39:48.879143-05:00","closed_at":"2025-12-12T15:39:48.879143-05:00"} {"id":"qmd-bx1","title":"Fix migration SQL for proper basename extraction","description":"The migration currently generates collection names incorrectly (uses full path instead of basename). Need to fix the SQL in migrateToContentAddressable to properly extract the directory basename.","status":"closed","priority":1,"issue_type":"bug","created_at":"2025-12-12T15:29:53.757723-05:00","updated_at":"2025-12-12T15:50:29.349134-05:00","closed_at":"2025-12-12T15:50:29.349134-05:00","dependencies":[{"issue_id":"qmd-bx1","depends_on_id":"qmd-ama","type":"discovered-from","created_at":"2025-12-12T15:29:53.758524-05:00","created_by":"daemon"}]} {"id":"qmd-c0m","title":"Comprehensive CLI review and consistency pass","description":"Review entire CLI command structure:\n- Consistent naming (add vs create, remove vs delete)\n- Consistent flag usage (--name, --mask, etc)\n- Update help text for all commands\n- Ensure virtual paths work everywhere\n- Test all commands end-to-end","status":"open","priority":1,"issue_type":"task","created_at":"2025-12-12T15:29:38.083564-05:00","updated_at":"2025-12-12T15:29:38.083564-05:00"} {"id":"qmd-deh","title":"Refactor database introduce qmd collection *","description":"","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-10T10:56:04.516137-05:00","updated_at":"2025-12-10T10:56:04.516137-05:00"} -{"id":"qmd-dmi","title":"Implement 'qmd collection' commands","description":"Add explicit collection management:\n- qmd collection add . --name \u003cname\u003e --mask '**/*.md'\n- qmd collection list\n- qmd collection remove \u003cname\u003e\n\nThis gives users control over collection names and patterns.","status":"open","priority":1,"issue_type":"feature","created_at":"2025-12-12T15:29:53.810666-05:00","updated_at":"2025-12-12T15:29:53.810666-05:00","dependencies":[{"issue_id":"qmd-dmi","depends_on_id":"qmd-ama","type":"discovered-from","created_at":"2025-12-12T15:29:53.811294-05:00","created_by":"daemon"}]} +{"id":"qmd-dmi","title":"Implement 'qmd collection' commands","description":"Add explicit collection management:\n- qmd collection add . --name \u003cname\u003e --mask '**/*.md'\n- qmd collection list\n- qmd collection remove \u003cname\u003e\n\nThis gives users control over collection names and patterns.","status":"closed","priority":1,"issue_type":"feature","created_at":"2025-12-12T15:29:53.810666-05:00","updated_at":"2025-12-12T16:02:08.079158-05:00","closed_at":"2025-12-12T16:02:08.079158-05:00","dependencies":[{"issue_id":"qmd-dmi","depends_on_id":"qmd-ama","type":"discovered-from","created_at":"2025-12-12T15:29:53.811294-05:00","created_by":"daemon"}]} {"id":"qmd-e2c","title":"Implement 'qmd ls' command","description":"Add command to explore virtual file tree:\n- qmd ls → list all collections\n- qmd ls \u003ccollection\u003e → list files in collection\n- qmd ls \u003ccollection\u003e/\u003cpath\u003e → list files under path\nOutput: flat list of qmd:// paths","status":"closed","priority":1,"issue_type":"feature","created_at":"2025-12-12T15:29:53.859804-05:00","updated_at":"2025-12-12T15:55:12.777701-05:00","closed_at":"2025-12-12T15:55:12.777701-05:00","dependencies":[{"issue_id":"qmd-e2c","depends_on_id":"qmd-ama","type":"discovered-from","created_at":"2025-12-12T15:29:53.860535-05:00","created_by":"daemon"}]} {"id":"qmd-j9z","title":"Add unit tests for content addressable hashes","description":"add same file from multiple places and verify that they both point at same hash. drop one collection and the content stays.","status":"open","priority":3,"issue_type":"task","created_at":"2025-12-12T15:39:15.459504-05:00","updated_at":"2025-12-12T15:39:15.459504-05:00"} {"id":"qmd-p1h","title":"Create collection add|remove","description":"","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-10T10:57:00.717864-05:00","updated_at":"2025-12-10T10:57:00.717864-05:00"} diff --git a/cli.test.ts b/cli.test.ts index 3ac3862..7487a48 100644 --- a/cli.test.ts +++ b/cli.test.ts @@ -560,3 +560,58 @@ describe("CLI ls Command", () => { expect(stderr).toContain("Collection not found"); }); }); + +describe("CLI Collection Commands", () => { + let localDbPath: string; + + beforeEach(async () => { + // Use a fresh database for this test suite + localDbPath = getFreshDbPath(); + // Index some files first to create a collection + await runQmd(["add", "."], { dbPath: localDbPath }); + }); + + test("lists collections", async () => { + const { stdout, exitCode } = await runQmd(["collection", "list"], { dbPath: localDbPath }); + expect(exitCode).toBe(0); + expect(stdout).toContain("Collections"); + expect(stdout).toContain("fixtures"); + expect(stdout).toContain("Path:"); + expect(stdout).toContain("Pattern:"); + expect(stdout).toContain("Files:"); + }); + + test("removes a collection", async () => { + // First verify the collection exists + const { stdout: listBefore } = await runQmd(["collection", "list"], { dbPath: localDbPath }); + expect(listBefore).toContain("fixtures"); + + // Remove it + const { stdout, exitCode } = await runQmd(["collection", "remove", "fixtures"], { dbPath: localDbPath }); + expect(exitCode).toBe(0); + expect(stdout).toContain("✓ Removed collection 'fixtures'"); + expect(stdout).toContain("Deleted"); + + // Verify it's gone + const { stdout: listAfter } = await runQmd(["collection", "list"], { dbPath: localDbPath }); + expect(listAfter).not.toContain("fixtures"); + }); + + test("handles removing non-existent collection", async () => { + const { stderr, exitCode } = await runQmd(["collection", "remove", "nonexistent"], { dbPath: localDbPath }); + expect(exitCode).toBe(1); + expect(stderr).toContain("Collection not found"); + }); + + test("handles missing remove argument", async () => { + const { stderr, exitCode } = await runQmd(["collection", "remove"], { dbPath: localDbPath }); + expect(exitCode).toBe(1); + expect(stderr).toContain("Usage:"); + }); + + test("handles unknown subcommand", async () => { + const { stderr, exitCode } = await runQmd(["collection", "invalid"], { dbPath: localDbPath }); + expect(exitCode).toBe(1); + expect(stderr).toContain("Unknown subcommand"); + }); +}); diff --git a/qmd.ts b/qmd.ts index a5b5b71..c01fdaa 100755 --- a/qmd.ts +++ b/qmd.ts @@ -1229,6 +1229,137 @@ function listFiles(pathArg?: string): void { closeDb(); } +// Collection management commands +function collectionList(): void { + const db = getDb(); + + const collections = db.prepare(` + SELECT + c.id, + c.name, + c.pwd, + c.glob_pattern, + c.created_at, + c.updated_at, + COUNT(d.id) as file_count + FROM collections c + LEFT JOIN documents d ON d.collection_id = c.id AND d.active = 1 + GROUP BY c.id + ORDER BY c.name + `).all() as { + id: number; + name: string; + pwd: string; + glob_pattern: string; + created_at: string; + updated_at: string; + file_count: number; + }[]; + + if (collections.length === 0) { + console.log("No collections found. Run 'qmd add .' to create one."); + closeDb(); + return; + } + + console.log(`${c.bold}Collections (${collections.length}):${c.reset}\n`); + + for (const coll of collections) { + const updatedAt = new Date(coll.updated_at); + const timeAgo = formatTimeAgo(updatedAt); + + console.log(`${c.cyan}${coll.name}${c.reset}`); + console.log(` ${c.dim}Path:${c.reset} ${coll.pwd}`); + console.log(` ${c.dim}Pattern:${c.reset} ${coll.glob_pattern}`); + console.log(` ${c.dim}Files:${c.reset} ${coll.file_count}`); + console.log(` ${c.dim}Updated:${c.reset} ${timeAgo}`); + console.log(); + } + + closeDb(); +} + +async function collectionAdd(pwd: string, globPattern: string, name?: string): Promise { + const db = getDb(); + + // If name not provided, generate from pwd basename + if (!name) { + const parts = pwd.split('/').filter(Boolean); + name = parts[parts.length - 1] || 'root'; + } + + // Check if collection with this name already exists + const existing = getCollectionByName(db, name); + if (existing) { + console.error(`${c.yellow}Collection '${name}' already exists.${c.reset}`); + console.error(`Use a different name with --name `); + closeDb(); + process.exit(1); + } + + // Check if a collection with this pwd+glob already exists + const existingPwdGlob = db.prepare(` + SELECT id, name FROM collections WHERE pwd = ? AND glob_pattern = ? + `).get(pwd, globPattern) as { id: number; name: string } | null; + + if (existingPwdGlob) { + console.error(`${c.yellow}A collection already exists for this path and pattern:${c.reset}`); + console.error(` Name: ${existingPwdGlob.name}`); + console.error(` Path: ${pwd}`); + console.error(` Pattern: ${globPattern}`); + console.error(`\nUse 'qmd add ${globPattern}' to update it, or remove it first with 'qmd collection remove ${existingPwdGlob.name}'`); + closeDb(); + process.exit(1); + } + + closeDb(); + + // Create the collection and index files + console.log(`Creating collection '${name}'...`); + await indexFiles(globPattern); + console.log(`${c.green}✓${c.reset} Collection '${name}' created successfully`); +} + +function collectionRemove(name: string): void { + const db = getDb(); + + const coll = getCollectionByName(db, name); + if (!coll) { + console.error(`${c.yellow}Collection not found: ${name}${c.reset}`); + console.error(`Run 'qmd collection list' to see available collections.`); + closeDb(); + process.exit(1); + } + + // Get file count + const fileCount = db.prepare(` + SELECT COUNT(*) as count FROM documents WHERE collection_id = ? AND active = 1 + `).get(coll.id) as { count: number }; + + // Delete documents + db.prepare(`DELETE FROM documents WHERE collection_id = ?`).run(coll.id); + + // Delete contexts + db.prepare(`DELETE FROM path_contexts WHERE collection_id = ?`).run(coll.id); + + // Delete collection + db.prepare(`DELETE FROM collections WHERE id = ?`).run(coll.id); + + // Clean up orphaned content hashes + const cleanupResult = db.prepare(` + DELETE FROM content + WHERE hash NOT IN (SELECT DISTINCT hash FROM documents WHERE active = 1) + `).run(); + + console.log(`${c.green}✓${c.reset} Removed collection '${name}'`); + console.log(` Deleted ${fileCount.count} documents`); + if (cleanupResult.changes > 0) { + console.log(` Cleaned up ${cleanupResult.changes} orphaned content hashes`); + } + + closeDb(); +} + async function dropCollection(globPattern: string): Promise { const db = getDb(); const pwd = getPwd(); @@ -2137,6 +2268,9 @@ function parseCLI() { collection: { type: "string", short: "c" }, // Filter by collection // Add options drop: { type: "boolean" }, + // Collection options + name: { type: "string" }, // collection name + mask: { type: "string" }, // glob pattern // Embed options force: { type: "boolean", short: "f" }, // Get options @@ -2193,6 +2327,9 @@ function showHelp(): void { console.log(" qmd get [:line] [-l N] [--from N] - Get document (optionally from line, max N lines)"); console.log(" qmd multi-get [-l N] [--max-bytes N] - Get multiple docs by glob or comma-separated list"); console.log(" qmd ls [collection[/path]] - List collections or files in a collection"); + console.log(" qmd collection list - List all collections with details"); + console.log(" qmd collection add [path] --name --mask - Create collection explicitly"); + console.log(" qmd collection remove - Remove a collection by name"); console.log(" qmd status - Show index status and collections"); console.log(" qmd update - Re-index all collections"); console.log(" qmd embed [-f] - Create vector embeddings (chunks ~6KB each)"); @@ -2374,6 +2511,43 @@ switch (cli.command) { break; } + case "collection": { + const subcommand = cli.args[0]; + switch (subcommand) { + case "list": { + collectionList(); + break; + } + + case "add": { + const pwd = cli.args[1] || getPwd(); + const resolvedPwd = pwd === '.' ? getPwd() : getRealPath(resolve(pwd)); + const globPattern = cli.values.mask as string || DEFAULT_GLOB; + const name = cli.values.name as string | undefined; + + await collectionAdd(resolvedPwd, globPattern, name); + break; + } + + case "remove": + case "rm": { + if (!cli.args[1]) { + console.error("Usage: qmd collection remove "); + console.error(" Use 'qmd collection list' to see available collections"); + process.exit(1); + } + collectionRemove(cli.args[1]); + break; + } + + default: + console.error(`Unknown subcommand: ${subcommand}`); + console.error("Available: list, add, remove"); + process.exit(1); + } + break; + } + case "status": showStatus(); break;