From fde542cd0d99617484ef3f34e2f7b05b12df2af6 Mon Sep 17 00:00:00 2001 From: Sebastian Kouba Date: Thu, 5 Mar 2026 19:17:44 +0100 Subject: [PATCH] feat: add ignore patterns for collections Add an optional 'ignore' field to collection config that accepts an array of glob patterns to exclude from indexing. This allows collections to skip specific subdirectories without needing separate collections. Example YAML config: personal: path: ~/personal_synced pattern: '**/*.md' ignore: - 'Sessions/**' - 'archive/**' The ignore patterns are passed to fast-glob's ignore option alongside the existing hardcoded excludes (node_modules, .git, etc). Already-indexed files matching new ignore patterns are deactivated on the next update. Changes: - Add ignore?: string[] to Collection interface - Pass ignore patterns through to fast-glob in indexFiles() - Show ignore patterns in collection list/status output - 5 new CLI integration tests covering the feature --- src/collections.ts | 1 + src/qmd.ts | 16 +++++-- test/cli.test.ts | 113 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 4 deletions(-) diff --git a/src/collections.ts b/src/collections.ts index f10018d..923b81f 100644 --- a/src/collections.ts +++ b/src/collections.ts @@ -27,6 +27,7 @@ export type ContextMap = Record; export interface Collection { path: string; // Absolute path to index pattern: string; // Glob pattern (e.g., "**/*.md") + ignore?: string[]; // Glob patterns to exclude (e.g., ["Sessions/**"]) context?: ContextMap; // Optional context definitions update?: string; // Optional bash command to run during qmd update includeByDefault?: boolean; // Include in queries by default (default: true) diff --git a/src/qmd.ts b/src/qmd.ts index 367d5f5..3ecc0c9 100755 --- a/src/qmd.ts +++ b/src/qmd.ts @@ -522,7 +522,7 @@ async function updateCollections(): Promise { } } - await indexFiles(col.pwd, col.glob_pattern, col.name, true); + await indexFiles(col.pwd, col.glob_pattern, col.name, true, yamlCol?.ignore); console.log(""); } @@ -1306,6 +1306,9 @@ function collectionList(): void { console.log(`${c.cyan}${coll.name}${c.reset} ${c.dim}(qmd://${coll.name}/)${c.reset}${excludeTag}`); console.log(` ${c.dim}Pattern:${c.reset} ${coll.glob_pattern}`); + if (yamlColl?.ignore?.length) { + console.log(` ${c.dim}Ignore:${c.reset} ${yamlColl.ignore.join(', ')}`); + } console.log(` ${c.dim}Files:${c.reset} ${coll.active_count}`); console.log(` ${c.dim}Updated:${c.reset} ${timeAgo}`); console.log(); @@ -1348,7 +1351,8 @@ async function collectionAdd(pwd: string, globPattern: string, name?: string): P // Create the collection and index files console.log(`Creating collection '${collName}'...`); - await indexFiles(pwd, globPattern, collName); + const newColl = getCollectionFromYaml(collName); + await indexFiles(pwd, globPattern, collName, false, newColl?.ignore); console.log(`${c.green}✓${c.reset} Collection '${collName}' created successfully`); } @@ -1397,7 +1401,7 @@ function collectionRename(oldName: string, newName: string): void { console.log(` Virtual paths updated: ${c.cyan}qmd://${oldName}/${c.reset} → ${c.cyan}qmd://${newName}/${c.reset}`); } -async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GLOB, collectionName?: string, suppressEmbedNotice: boolean = false): Promise { +async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GLOB, collectionName?: string, suppressEmbedNotice: boolean = false, ignorePatterns?: string[]): Promise { const db = getDb(); const resolvedPwd = pwd || getPwd(); const now = new Date().toISOString(); @@ -1414,12 +1418,16 @@ async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GLOB, coll console.log(`Collection: ${resolvedPwd} (${globPattern})`); progress.indeterminate(); + const allIgnore = [ + ...excludeDirs.map(d => `**/${d}/**`), + ...(ignorePatterns || []), + ]; const allFiles: string[] = await fastGlob(globPattern, { cwd: resolvedPwd, onlyFiles: true, followSymbolicLinks: false, dot: false, - ignore: excludeDirs.map(d => `**/${d}/**`), + ignore: allIgnore, }); // Filter hidden files/folders (dot: false handles top-level but not nested) const files = allFiles.filter(file => { diff --git a/test/cli.test.ts b/test/cli.test.ts index 8c96107..bd9d160 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -782,6 +782,119 @@ describe("CLI Collection Commands", () => { }); }); +// ============================================================================= +// Collection Ignore Patterns +// ============================================================================= + +describe("collection ignore patterns", () => { + let localDbPath: string; + let localConfigDir: string; + let ignoreTestDir: string; + + beforeAll(async () => { + const env = await createIsolatedTestEnv("ignore-patterns"); + localDbPath = env.dbPath; + localConfigDir = env.configDir; + + // Create directory structure with subdirectories to ignore + ignoreTestDir = join(testDir, "ignore-fixtures"); + await mkdir(join(ignoreTestDir, "notes"), { recursive: true }); + await mkdir(join(ignoreTestDir, "sessions"), { recursive: true }); + await mkdir(join(ignoreTestDir, "sessions", "2026-03"), { recursive: true }); + await mkdir(join(ignoreTestDir, "archive"), { recursive: true }); + + // Files that should be indexed + await writeFile(join(ignoreTestDir, "readme.md"), "# Main readme\nThis should be indexed."); + await writeFile(join(ignoreTestDir, "notes", "note1.md"), "# Note 1\nThis is a personal note."); + + // Files that should be ignored + await writeFile(join(ignoreTestDir, "sessions", "session1.md"), "# Session 1\nThis session should be ignored."); + await writeFile(join(ignoreTestDir, "sessions", "2026-03", "session2.md"), "# Session 2\nNested session should also be ignored."); + await writeFile(join(ignoreTestDir, "archive", "old.md"), "# Old stuff\nThis archive file should be ignored."); + }); + + test("ignore patterns exclude matching files from indexing", async () => { + // Write YAML config with ignore patterns + await writeFile( + join(localConfigDir, "index.yml"), + `collections: + ignoretst: + path: ${ignoreTestDir} + pattern: "**/*.md" + ignore: + - "sessions/**" + - "archive/**" +` + ); + + const { stdout, exitCode } = await runQmd(["update"], { + cwd: ignoreTestDir, + dbPath: localDbPath, + configDir: localConfigDir, + }); + expect(exitCode).toBe(0); + // Should index 2 files (readme.md + notes/note1.md), not 5 + expect(stdout).toContain("2 new"); + }); + + test("ignored files are not searchable", async () => { + const { stdout, exitCode } = await runQmd(["search", "session", "-n", "10"], { + cwd: ignoreTestDir, + dbPath: localDbPath, + configDir: localConfigDir, + }); + // Should find no results since sessions/ was ignored + if (exitCode === 0) { + expect(stdout).not.toContain("session1"); + expect(stdout).not.toContain("session2"); + } + }); + + test("non-ignored files are searchable", async () => { + const { stdout, exitCode } = await runQmd(["search", "personal note", "-n", "10"], { + cwd: ignoreTestDir, + dbPath: localDbPath, + configDir: localConfigDir, + }); + expect(exitCode).toBe(0); + expect(stdout).toContain("note1"); + }); + + test("status shows ignore patterns", async () => { + const { stdout, exitCode } = await runQmd(["collection", "list"], { + cwd: ignoreTestDir, + dbPath: localDbPath, + configDir: localConfigDir, + }); + expect(exitCode).toBe(0); + expect(stdout).toContain("Ignore:"); + expect(stdout).toContain("sessions/**"); + expect(stdout).toContain("archive/**"); + }); + + test("collection without ignore indexes all files", async () => { + // Create a second collection without ignore + const env2 = await createIsolatedTestEnv("no-ignore"); + await writeFile( + join(env2.configDir, "index.yml"), + `collections: + allfiles: + path: ${ignoreTestDir} + pattern: "**/*.md" +` + ); + + const { stdout, exitCode } = await runQmd(["update"], { + cwd: ignoreTestDir, + dbPath: env2.dbPath, + configDir: env2.configDir, + }); + expect(exitCode).toBe(0); + // Should index all 5 files + expect(stdout).toContain("5 new"); + }); +}); + // ============================================================================= // Output Format Tests - qmd:// URIs, context, and docid // =============================================================================