Fix FTS schema and getStatus() to use YAML collections
1. Fixed FTS table schema mismatch: - Updated initDb() to create FTS with filepath, title, body columns - Updated migration to match the same schema - Fixed triggers to properly populate FTS on insert/update - Removed legacy path_contexts index 2. Fixed getStatus() function: - Load collections from YAML instead of querying collections table - Query documents table for counts per collection - Changed CollectionInfo type to use 'name' instead of 'id' 3. Fixed searchFTS() collectionId parameter: - Removed query to collections table - Made collectionId parameter legacy/deprecated These fixes resolve the FTS trigger errors during qmd update. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Tobi Lutke
parent
86f4374ba3
commit
6542e1271c
@ -1,7 +1,8 @@
|
||||
{"id":"qmd-0ic","title":"in qmd status, list all the additonal contexts under the collections that match","description":"","status":"closed","priority":2,"issue_type":"task","created_at":"2025-12-12T16:41:42.126194-05:00","updated_at":"2025-12-12T17:14:48.268119-05:00","closed_at":"2025-12-12T17:14:48.268119-05:00"}
|
||||
{"id":"qmd-18s","title":"Move cleanup/maintenance DB operations to store.ts","description":"Move cleanup operations from cleanup() command to store.ts. Create methods like deleteInactiveDocuments(), vacuumDatabase(), cleanupOrphanedContent(), etc.","status":"closed","priority":2,"issue_type":"task","created_at":"2025-12-12T16:36:21.815781-05:00","updated_at":"2025-12-12T16:42:36.896806-05:00","closed_at":"2025-12-12T16:42:36.896806-05:00","dependencies":[{"issue_id":"qmd-18s","depends_on_id":"qmd-29c","type":"parent-child","created_at":"2025-12-12T16:37:03.014111-05:00","created_by":"daemon"}]}
|
||||
{"id":"qmd-1xd","title":"Update tests for YAML-based collections","description":"Update all tests to use YAML config instead of DB collections. Update test helpers to create temporary YAML configs.","status":"in_progress","priority":1,"issue_type":"task","created_at":"2025-12-13T09:54:53.349545-05:00","updated_at":"2025-12-13T10:27:15.760204-05:00","dependencies":[{"issue_id":"qmd-1xd","depends_on_id":"qmd-thw","type":"blocks","created_at":"2025-12-13T09:55:08.14305-05:00","created_by":"daemon"}]}
|
||||
{"id":"qmd-1xd","title":"Update tests for YAML-based collections","description":"Update all tests to use YAML config instead of DB collections. Update test helpers to create temporary YAML configs.","notes":"Test suite has been updated for YAML-based collections. 92 tests passing, 4 skipped, 10 failing.\n\nThe 4 skipped tests call getStatus() which has a bug (queries non-existent collections table).\n\nThe 10 failing tests are due to bugs in store.ts functions (findDocument, getDocumentBody, getDocument, findSimilarFiles, matchFilesByGlob) that need to be updated to use YAML configuration. These are production code bugs, not test bugs.","status":"closed","priority":1,"issue_type":"task","created_at":"2025-12-13T09:54:53.349545-05:00","updated_at":"2025-12-13T11:37:16.935866-05:00","closed_at":"2025-12-13T11:37:16.935866-05:00","dependencies":[{"issue_id":"qmd-1xd","depends_on_id":"qmd-thw","type":"blocks","created_at":"2025-12-13T09:55:08.14305-05:00","created_by":"daemon"}]}
|
||||
{"id":"qmd-29c","title":"Move all database operations from qmd.ts to store.ts","description":"Currently qmd.ts has ~70 direct database operations (db.prepare, db.exec). All database operations should be moved to store.ts to improve separation of concerns. qmd.ts should only use high-level methods from store.ts that don't require direct SQL knowledge.","notes":"Phase 1 complete: Moved collection operations (listCollections, removeCollection, renameCollection) to store.ts. Created 4 subtasks for remaining work: document indexing, context management, embeddings, and cleanup operations.","status":"closed","priority":2,"issue_type":"task","created_at":"2025-12-12T16:32:13.722223-05:00","updated_at":"2025-12-12T16:49:53.829124-05:00","closed_at":"2025-12-12T16:49:53.829124-05:00"}
|
||||
{"id":"qmd-2gn","title":"Fix store.ts functions to use YAML collections","description":"Update findDocument(), getDocumentBody(), getDocument(), findSimilarFiles(), matchFilesByGlob(), and getStatus() to use YAML collection configuration instead of querying the collections table. These functions currently fail because they try to query the non-existent collections table.","status":"in_progress","priority":1,"issue_type":"bug","created_at":"2025-12-13T11:37:22.706882-05:00","updated_at":"2025-12-13T12:27:48.33024-05:00"}
|
||||
{"id":"qmd-3z9","title":"Design YAML schema and create collections.ts module","description":"Create collections.ts to manage YAML-based collection configuration at ~/.config/qmd/index.yml. Define TypeScript types for collections and contexts. Implement load/save functions with Bun's native YAML support.","design":"YAML structure:\n```yaml\n# Global context for all collections\nglobal_context: \"...\"\n\ncollections:\n name:\n path: /absolute/path\n pattern: \"**/*.md\"\n context:\n \"/path/prefix\": \"Description\"\n \"/\": \"Root context\"\n```\n\nTypeScript types:\n- Collection: { path, pattern, context }\n- CollectionConfig: { global_context?, collections }\n- Functions: loadConfig(), saveConfig(), getCollection(), listCollections()","status":"closed","priority":1,"issue_type":"task","created_at":"2025-12-13T09:54:52.586027-05:00","updated_at":"2025-12-13T09:56:57.309927-05:00","closed_at":"2025-12-13T09:56:57.309927-05:00"}
|
||||
{"id":"qmd-4ru","title":"Update document retrieval for new schema","description":"Functions like getDocument, findDocument, getMultipleDocuments need to work with new schema (path instead of filepath, content joins, virtual paths).","status":"closed","priority":0,"issue_type":"task","created_at":"2025-12-12T15:29:53.911881-05:00","updated_at":"2025-12-12T15:56:11.054888-05:00","closed_at":"2025-12-12T15:56:11.054888-05:00","dependencies":[{"issue_id":"qmd-4ru","depends_on_id":"qmd-ama","type":"discovered-from","created_at":"2025-12-12T15:29:53.912607-05:00","created_by":"daemon"}]}
|
||||
{"id":"qmd-4u4","title":"Move embedding/vector DB operations to store.ts","description":"Move vector indexing DB operations from vectorIndex() to store.ts. Create methods like getHashesForEmbedding(), insertEmbedding(), clearEmbeddings(), etc.","status":"closed","priority":2,"issue_type":"task","created_at":"2025-12-12T16:36:21.683434-05:00","updated_at":"2025-12-12T16:42:40.42653-05:00","closed_at":"2025-12-12T16:42:40.42653-05:00","dependencies":[{"issue_id":"qmd-4u4","depends_on_id":"qmd-29c","type":"parent-child","created_at":"2025-12-12T16:37:02.944591-05:00","created_by":"daemon"}]}
|
||||
|
||||
137
src/store.ts
137
src/store.ts
@ -278,21 +278,26 @@ function initializeDatabase(db: Database): void {
|
||||
)
|
||||
`);
|
||||
|
||||
// FTS - index path and content (joined from content table)
|
||||
// FTS - index filepath (collection/path), title, and content
|
||||
db.exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
|
||||
path, body,
|
||||
filepath, title, body,
|
||||
tokenize='porter unicode61'
|
||||
)
|
||||
`);
|
||||
|
||||
// Triggers to keep FTS in sync
|
||||
db.exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS documents_ai AFTER INSERT ON documents BEGIN
|
||||
INSERT INTO documents_fts(rowid, path, body)
|
||||
SELECT new.id, new.path, c.doc
|
||||
FROM content c
|
||||
WHERE c.hash = new.hash;
|
||||
CREATE TRIGGER IF NOT EXISTS documents_ai AFTER INSERT ON documents
|
||||
WHEN new.active = 1
|
||||
BEGIN
|
||||
INSERT INTO documents_fts(rowid, filepath, title, body)
|
||||
SELECT
|
||||
new.id,
|
||||
new.collection || '/' || new.path,
|
||||
new.title,
|
||||
(SELECT doc FROM content WHERE hash = new.hash)
|
||||
WHERE new.active = 1;
|
||||
END
|
||||
`);
|
||||
|
||||
@ -303,11 +308,19 @@ function initializeDatabase(db: Database): void {
|
||||
`);
|
||||
|
||||
db.exec(`
|
||||
CREATE TRIGGER IF NOT EXISTS documents_au AFTER UPDATE ON documents BEGIN
|
||||
UPDATE documents_fts
|
||||
SET path = new.path,
|
||||
body = (SELECT doc FROM content WHERE hash = new.hash)
|
||||
WHERE rowid = new.id;
|
||||
CREATE TRIGGER IF NOT EXISTS documents_au AFTER UPDATE ON documents
|
||||
BEGIN
|
||||
-- Delete from FTS if no longer active
|
||||
DELETE FROM documents_fts WHERE rowid = old.id AND new.active = 0;
|
||||
|
||||
-- Update FTS if still/newly active
|
||||
INSERT OR REPLACE INTO documents_fts(rowid, filepath, title, body)
|
||||
SELECT
|
||||
new.id,
|
||||
new.collection || '/' || new.path,
|
||||
new.title,
|
||||
(SELECT doc FROM content WHERE hash = new.hash)
|
||||
WHERE new.active = 1;
|
||||
END
|
||||
`);
|
||||
}
|
||||
@ -508,20 +521,25 @@ function migrateToContentAddressable(db: Database): void {
|
||||
db.exec("DROP TABLE collections_old");
|
||||
db.exec("DROP TABLE path_contexts_old");
|
||||
|
||||
// Recreate FTS and triggers
|
||||
// Recreate FTS and triggers (matching migrated schema)
|
||||
db.exec(`
|
||||
CREATE VIRTUAL TABLE documents_fts USING fts5(
|
||||
path, body,
|
||||
filepath, title, body,
|
||||
tokenize='porter unicode61'
|
||||
)
|
||||
`);
|
||||
|
||||
db.exec(`
|
||||
CREATE TRIGGER documents_ai AFTER INSERT ON documents BEGIN
|
||||
INSERT INTO documents_fts(rowid, path, body)
|
||||
SELECT new.id, new.path, c.doc
|
||||
FROM content c
|
||||
WHERE c.hash = new.hash;
|
||||
CREATE TRIGGER documents_ai AFTER INSERT ON documents
|
||||
WHEN new.active = 1
|
||||
BEGIN
|
||||
INSERT INTO documents_fts(rowid, filepath, title, body)
|
||||
SELECT
|
||||
new.id,
|
||||
new.collection || '/' || new.path,
|
||||
new.title,
|
||||
(SELECT doc FROM content WHERE hash = new.hash)
|
||||
WHERE new.active = 1;
|
||||
END
|
||||
`);
|
||||
|
||||
@ -532,19 +550,27 @@ function migrateToContentAddressable(db: Database): void {
|
||||
`);
|
||||
|
||||
db.exec(`
|
||||
CREATE TRIGGER documents_au AFTER UPDATE ON documents BEGIN
|
||||
UPDATE documents_fts
|
||||
SET path = new.path,
|
||||
body = (SELECT doc FROM content WHERE hash = new.hash)
|
||||
WHERE rowid = new.id;
|
||||
CREATE TRIGGER documents_au AFTER UPDATE ON documents
|
||||
BEGIN
|
||||
-- Delete from FTS if no longer active
|
||||
DELETE FROM documents_fts WHERE rowid = old.id AND new.active = 0;
|
||||
|
||||
-- Update FTS if still/newly active
|
||||
INSERT OR REPLACE INTO documents_fts(rowid, filepath, title, body)
|
||||
SELECT
|
||||
new.id,
|
||||
new.collection || '/' || new.path,
|
||||
new.title,
|
||||
(SELECT doc FROM content WHERE hash = new.hash)
|
||||
WHERE new.active = 1;
|
||||
END
|
||||
`);
|
||||
|
||||
// Populate FTS from migrated data
|
||||
console.log("Rebuilding full-text search index...");
|
||||
db.exec(`
|
||||
INSERT INTO documents_fts(rowid, path, body)
|
||||
SELECT d.id, d.path, c.doc
|
||||
INSERT INTO documents_fts(rowid, filepath, title, body)
|
||||
SELECT d.id, d.collection || '/' || d.path, d.title, c.doc
|
||||
FROM documents d
|
||||
JOIN content c ON c.hash = d.hash
|
||||
WHERE d.active = 1
|
||||
@ -554,7 +580,6 @@ function migrateToContentAddressable(db: Database): void {
|
||||
db.exec(`CREATE INDEX idx_documents_collection ON documents(collection, active)`);
|
||||
db.exec(`CREATE INDEX idx_documents_hash ON documents(hash)`);
|
||||
db.exec(`CREATE INDEX idx_documents_path ON documents(path, active)`);
|
||||
db.exec(`CREATE INDEX idx_path_contexts_collection ON path_contexts(collection_id, path_prefix)`);
|
||||
|
||||
db.exec("COMMIT");
|
||||
console.log("Migration complete!");
|
||||
@ -846,7 +871,7 @@ export type MultiGetResult = {
|
||||
};
|
||||
|
||||
export type CollectionInfo = {
|
||||
id: number;
|
||||
name: string;
|
||||
path: string;
|
||||
pattern: string;
|
||||
documents: number;
|
||||
@ -1595,12 +1620,11 @@ export function searchFTS(db: Database, query: string, limit: number = 20, colle
|
||||
const params: (string | number)[] = [ftsQuery];
|
||||
|
||||
if (collectionId !== undefined) {
|
||||
// Convert collectionId to collection name for filtering
|
||||
const coll = db.prepare(`SELECT name FROM collections WHERE id = ?`).get(collectionId) as { name: string } | null;
|
||||
if (coll) {
|
||||
sql += ` AND d.collection = ?`;
|
||||
params.push(coll.name);
|
||||
}
|
||||
// Note: collectionId is a legacy parameter that should be phased out
|
||||
// Collections are now managed in YAML. For now, we interpret it as a collection name filter.
|
||||
// This code path is likely unused as collection filtering should be done at CLI level.
|
||||
sql += ` AND d.collection = ?`;
|
||||
params.push(String(collectionId));
|
||||
}
|
||||
|
||||
sql += ` ORDER BY score LIMIT ?`;
|
||||
@ -2212,15 +2236,34 @@ export type MultiGetFile = {
|
||||
// =============================================================================
|
||||
|
||||
export function getStatus(db: Database): IndexStatus {
|
||||
const collections = db.prepare(`
|
||||
SELECT c.id, c.pwd, c.glob_pattern, c.created_at,
|
||||
COUNT(d.id) as active_count,
|
||||
MAX(d.modified_at) as last_doc_update
|
||||
FROM collections c
|
||||
LEFT JOIN documents d ON d.collection = c.name AND d.active = 1
|
||||
GROUP BY c.id
|
||||
ORDER BY last_doc_update DESC
|
||||
`).all() as { id: number; pwd: string; glob_pattern: string; created_at: string; active_count: number; last_doc_update: string | null }[];
|
||||
// Load collections from YAML
|
||||
const yamlCollections = collectionsListCollections();
|
||||
|
||||
// Get document counts and last update times for each collection
|
||||
const collections = yamlCollections.map(col => {
|
||||
const stats = db.prepare(`
|
||||
SELECT
|
||||
COUNT(*) as active_count,
|
||||
MAX(modified_at) as last_doc_update
|
||||
FROM documents
|
||||
WHERE collection = ? AND active = 1
|
||||
`).get(col.name) as { active_count: number; last_doc_update: string | null };
|
||||
|
||||
return {
|
||||
name: col.name,
|
||||
path: col.path,
|
||||
pattern: col.pattern,
|
||||
documents: stats.active_count,
|
||||
lastUpdated: stats.last_doc_update || new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
// Sort by last update time (most recent first)
|
||||
collections.sort((a, b) => {
|
||||
if (!a.lastUpdated) return 1;
|
||||
if (!b.lastUpdated) return -1;
|
||||
return new Date(b.lastUpdated).getTime() - new Date(a.lastUpdated).getTime();
|
||||
});
|
||||
|
||||
const totalDocs = (db.prepare(`SELECT COUNT(*) as c FROM documents WHERE active = 1`).get() as { c: number }).c;
|
||||
const needsEmbedding = getHashesNeedingEmbedding(db);
|
||||
@ -2230,13 +2273,7 @@ export function getStatus(db: Database): IndexStatus {
|
||||
totalDocuments: totalDocs,
|
||||
needsEmbedding,
|
||||
hasVectorIndex: hasVectors,
|
||||
collections: collections.map(col => ({
|
||||
id: col.id,
|
||||
path: col.pwd,
|
||||
pattern: col.glob_pattern,
|
||||
documents: col.active_count,
|
||||
lastUpdated: col.last_doc_update || col.created_at,
|
||||
})),
|
||||
collections,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user