Merge pull request #698 from tobi/feat/literal-path-storage
fix: store literal filesystem paths, drop handelize() at index time
This commit is contained in:
Tobias Lütke
GitHub
commit
3f751cd0f0
23
CHANGELOG.md
23
CHANGELOG.md
@ -2,6 +2,29 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- Filesystem paths with special characters (`#`, `&`, spaces, `[]`, `()`, etc.)
|
||||
now round-trip correctly through index → search → get. Previously
|
||||
`reindexCollection` called `handelize()` on relative paths before storing
|
||||
them, turning `# Meeting - 234232 3432 __ 5.md` into
|
||||
`Meeting-234232-3432-5.md` and making `qmd get <actual-path>`,
|
||||
`qmd get --full-path`, and `qmd ls` return dead or garbled paths. Paths are
|
||||
now stored verbatim. Existing indexes auto-migrate on the next `qmd update`.
|
||||
|
||||
- FTS5 search now correctly matches dotted version strings like `2026.4.10`. The
|
||||
`porter unicode61` tokenizer splits on dots (storing `2026`, `4`, `10` as
|
||||
separate tokens), but the query sanitizer was stripping dots and producing
|
||||
`2026410` which never matched. Dotted terms are now split and ANDed together
|
||||
so version-string searches work as expected (#563).
|
||||
- HTTP REST endpoints `/query` and `/search` now return `qmd://collection/path`
|
||||
URIs in the `file` field, matching the output format used by the CLI and MCP
|
||||
resource URIs. Previously the raw `displayPath` (`collection/path`) was
|
||||
returned without the scheme prefix (#576).
|
||||
- The embed session `maxDuration` is now env-configurable via
|
||||
`QMD_EMBED_MAX_DURATION_MS` (default: 30 min). This prevents large-corpus
|
||||
embeddings from being aborted by the hardcoded 30-minute ceiling (#673).
|
||||
|
||||
## [2.5.3] - 2026-05-28
|
||||
|
||||
### Features
|
||||
|
||||
@ -1824,7 +1824,8 @@ async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GLOB, coll
|
||||
|
||||
for (const relativeFile of files) {
|
||||
const filepath = getRealPath(resolve(resolvedPwd, relativeFile));
|
||||
const path = handelize(relativeFile); // Normalize path for token-friendliness
|
||||
// Store the literal relative path — handelize() is NOT applied at index time.
|
||||
const path = relativeFile.replace(/\\/g, '/');
|
||||
seenPaths.add(path);
|
||||
|
||||
let content: string;
|
||||
|
||||
@ -731,7 +731,7 @@ export async function startMcpHttpServer(
|
||||
const { line, snippet } = extractSnippet(r.body, String(primaryQuery), 300, r.bestChunkPos, r.bestChunk.length, typeof params.intent === "string" ? params.intent : undefined);
|
||||
return {
|
||||
docid: `#${r.docid}`,
|
||||
file: r.displayPath,
|
||||
file: `qmd://${encodeQmdPath(r.displayPath)}`,
|
||||
title: r.title,
|
||||
score: Math.round(r.score * 100) / 100,
|
||||
context: r.context,
|
||||
|
||||
68
src/store.ts
68
src/store.ts
@ -1306,7 +1306,10 @@ export async function reindexCollection(
|
||||
|
||||
for (const relativeFile of files) {
|
||||
const filepath = getRealPath(resolve(collectionPath, relativeFile));
|
||||
const path = handelize(relativeFile);
|
||||
// Store the literal relative path so the filesystem path can always be
|
||||
// reconstructed as: resolve(collection.path, storedPath).
|
||||
// handelize() is NOT applied at index time — it is display-only.
|
||||
const path = normalizePathSeparators(relativeFile);
|
||||
seenPaths.add(path);
|
||||
|
||||
let content: string;
|
||||
@ -2493,12 +2496,34 @@ export function findOrMigrateLegacyDocument(
|
||||
const existing = findActiveDocument(db, collectionName, path);
|
||||
if (existing) return existing;
|
||||
|
||||
const legacy = db.prepare(`
|
||||
// Case-insensitive match (legacy normalization: e.g. "README.md" → "readme.md").
|
||||
const legacyCase = db.prepare(`
|
||||
SELECT id, hash, title FROM documents
|
||||
WHERE collection = ? AND path COLLATE NOCASE = ? AND active = 1
|
||||
ORDER BY id
|
||||
LIMIT 1
|
||||
`).get(collectionName, path) as { id: number; hash: string; title: string } | undefined;
|
||||
|
||||
// Handalized-path match: existing DBs indexed with handelize() stored slugged paths
|
||||
// like "Budget-Revenue-Q4-2024.md" for a raw path like "Budget & Revenue (Q4) [2024].md".
|
||||
// Try matching the handalized form of the incoming raw path against the DB so that
|
||||
// qmd update on an old index can rename the row to the literal path.
|
||||
let legacyHandalized: { id: number; hash: string; title: string } | undefined;
|
||||
try {
|
||||
const handleized = handelize(path);
|
||||
if (handleized !== path) {
|
||||
legacyHandalized = db.prepare(`
|
||||
SELECT id, hash, title FROM documents
|
||||
WHERE collection = ? AND path = ? AND active = 1
|
||||
ORDER BY id
|
||||
LIMIT 1
|
||||
`).get(collectionName, handleized) as { id: number; hash: string; title: string } | undefined;
|
||||
}
|
||||
} catch {
|
||||
// handelize throws on invalid paths; just skip
|
||||
}
|
||||
|
||||
const legacy = legacyCase ?? legacyHandalized;
|
||||
if (!legacy) return null;
|
||||
|
||||
// Wrap rename + FTS rebuild in a transaction for atomicity.
|
||||
@ -3249,6 +3274,27 @@ function sanitizeHyphenatedTerm(term: string): string {
|
||||
return term.split('-').map(t => sanitizeFTS5Term(t)).filter(t => t).join(' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a token is a dotted version/version-like string (e.g., 2026.4.10, 3.14.0).
|
||||
* Returns true if splitting on dots yields at least 2 non-empty parts consisting of
|
||||
* word/digit characters only. This avoids incorrectly splitting tokens with leading/
|
||||
* trailing dots. Version strings like "2026.4.10" split into ["2026","4","10"] (3 parts).
|
||||
*/
|
||||
function isDottedToken(token: string): boolean {
|
||||
const parts = token.split('.');
|
||||
return parts.length >= 2 && parts.every(p => p.length > 0 && /^[\p{L}\p{N}_]+$/u.test(p));
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize a dotted term into individual FTS5 tokens joined with AND.
|
||||
* e.g. "2026.4.10" → '"2026"* AND "4"* AND "10"*'
|
||||
* The AND ensures all parts must appear, matching how the porter tokenizer
|
||||
* indexes dotted strings.
|
||||
*/
|
||||
function sanitizeDottedTerm(term: string): string {
|
||||
return term.split('.').map(t => sanitizeFTS5Term(t)).filter(t => t).map(t => `"${t}"*`).join(' AND ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse lex query syntax into FTS5 query.
|
||||
*
|
||||
@ -3325,6 +3371,24 @@ function buildFTS5Query(query: string): string | null {
|
||||
positive.push(ftsPhrase);
|
||||
}
|
||||
}
|
||||
} else if (isDottedToken(term)) {
|
||||
// Handle dotted version strings: 2026.4.10, 3.14.0, v1.2.3
|
||||
// The porter tokenizer splits on dots, so the index has individual tokens.
|
||||
// We AND all parts together so the query matches documents containing all parts.
|
||||
const sanitized = sanitizeDottedTerm(term);
|
||||
if (sanitized) {
|
||||
// sanitizeDottedTerm already wraps each part in quotes with prefix match
|
||||
if (negated) {
|
||||
// Wrap multi-token AND expression in parens for NOT negation
|
||||
negative.push(`(${sanitized})`);
|
||||
} else {
|
||||
// Flatten individual AND'd terms into the positive list so they combine
|
||||
// correctly with other terms (avoids double-wrapping in outer AND).
|
||||
for (const part of sanitized.split(' AND ')) {
|
||||
positive.push(part.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (containsCjk(term)) {
|
||||
const sanitized = sanitizeFTS5Phrase(term);
|
||||
if (sanitized) {
|
||||
|
||||
@ -889,6 +889,33 @@ describe("MCP Server", () => {
|
||||
expect(typeof col.documents).toBe("number");
|
||||
}
|
||||
});
|
||||
|
||||
test("REST /query and /search file field uses qmd:// URI prefix (#576)", () => {
|
||||
// Regression test: the HTTP REST endpoint was returning r.displayPath (e.g.
|
||||
// "docs/readme.md") instead of "qmd://docs/readme.md", while the CLI and MCP
|
||||
// resource URIs always use the qmd:// scheme. This simulates the fix: the REST
|
||||
// handler now applies encodeQmdPath and prepends "qmd://".
|
||||
const results = searchFTS(testDb, "readme", 5);
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
|
||||
// Simulate what the fixed REST handler produces for each result
|
||||
const restResponseItems = results.map(r => ({
|
||||
docid: `#${r.docid}`,
|
||||
file: `qmd://${r.displayPath.split('/').map(s => encodeURIComponent(s)).join('/')}`,
|
||||
title: r.title,
|
||||
score: Math.round(r.score * 100) / 100,
|
||||
}));
|
||||
|
||||
// Every file field must start with qmd://
|
||||
for (const item of restResponseItems) {
|
||||
expect(item.file).toMatch(/^qmd:\/\//);
|
||||
}
|
||||
|
||||
// Spot-check the readme result
|
||||
const readmeItem = restResponseItems.find(item => item.file.includes("readme"));
|
||||
expect(readmeItem).toBeDefined();
|
||||
expect(readmeItem!.file).toBe("qmd://docs/readme.md");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
414
test/path-fidelity.test.ts
Normal file
414
test/path-fidelity.test.ts
Normal file
@ -0,0 +1,414 @@
|
||||
/**
|
||||
* Path Fidelity Tests
|
||||
*
|
||||
* Verifies that QMD stores literal filesystem paths (not handalized slugs) so
|
||||
* that paths with special characters — spaces, #, &, @, [], (), etc. — round-
|
||||
* trip correctly through index → search → get → full-path.
|
||||
*
|
||||
* This covers the five breakage points found before the literal-path fix:
|
||||
* 1. search --json `file` field shows handalized slug instead of real path
|
||||
* 2. `qmd get --full-path` silently falls back (resolveVirtualPath built
|
||||
* a non-existent path from the slug, existsSync returned false)
|
||||
* 3. `qmd get <actual-fs-path>` returns "Document not found"
|
||||
* 4. `qmd ls` shows handalized slugs
|
||||
* 5. `toVirtualPath(db, absPath)` returns null
|
||||
*
|
||||
* Also covers backward-compat migration: an index created with the old
|
||||
* handalize-at-index-time code can be updated with `qmd update` and the paths
|
||||
* are renamed to their literal forms in-place.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from "vitest";
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "fs/promises";
|
||||
import { existsSync, realpathSync } from "fs";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import { spawn } from "child_process";
|
||||
import { fileURLToPath } from "url";
|
||||
import { dirname } from "path";
|
||||
import YAML from "yaml";
|
||||
import { openDatabase } from "../src/db.js";
|
||||
import type { Database } from "../src/db.js";
|
||||
import {
|
||||
createStore,
|
||||
toVirtualPath,
|
||||
insertDocument,
|
||||
insertContent,
|
||||
hashContent,
|
||||
handelize,
|
||||
normalizePathSeparators,
|
||||
syncConfigToDb,
|
||||
} from "../src/store.js";
|
||||
import type { CollectionConfig } from "../src/collections.js";
|
||||
|
||||
const thisDir = dirname(fileURLToPath(import.meta.url));
|
||||
const projectRoot = join(thisDir, "..");
|
||||
const qmdScript = join(projectRoot, "src", "cli", "qmd.ts");
|
||||
const isBunRuntime = typeof (globalThis as { Bun?: unknown }).Bun !== "undefined";
|
||||
const tsxCli = join(projectRoot, "node_modules", "tsx", "dist", "cli.mjs");
|
||||
|
||||
async function runQmd(
|
||||
args: string[],
|
||||
opts: { cwd: string; dbPath: string; configDir: string; env?: Record<string, string> }
|
||||
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
|
||||
const runner = isBunRuntime
|
||||
? { command: process.execPath, args: [qmdScript, ...args] }
|
||||
: { command: process.execPath, args: [tsxCli, qmdScript, ...args] };
|
||||
|
||||
const proc = spawn(runner.command, runner.args, {
|
||||
cwd: opts.cwd,
|
||||
env: {
|
||||
...process.env,
|
||||
INDEX_PATH: opts.dbPath,
|
||||
QMD_CONFIG_DIR: opts.configDir,
|
||||
PWD: opts.cwd,
|
||||
QMD_DOCTOR_DEVICE_PROBE: "0",
|
||||
...(opts.env ?? {}),
|
||||
},
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
proc.stdout?.on("data", (c: Buffer) => { stdout += c.toString(); });
|
||||
proc.stderr?.on("data", (c: Buffer) => { stderr += c.toString(); });
|
||||
const exitCode = await new Promise<number>((res, rej) => {
|
||||
proc.once("error", rej);
|
||||
proc.on("close", (code) => res(code ?? 1));
|
||||
});
|
||||
return { stdout, stderr, exitCode };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test environment setup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let testDir: string;
|
||||
|
||||
// Files with names that previously broke due to handalize() at index time.
|
||||
const crazyFiles: Array<{ name: string; content: string }> = [
|
||||
{
|
||||
name: "# Meeting - 234232 3432 __ 5.md",
|
||||
content: "# Meeting - 234232 3432 // 5\n\nSome meeting content with searchterm-alpha.\n",
|
||||
},
|
||||
{
|
||||
name: "Budget & Revenue (Q4) [2024].md",
|
||||
content: "# Budget & Revenue Q4 2024\n\nFinancial overview searchterm-beta.\n",
|
||||
},
|
||||
{
|
||||
name: "normal-file.md",
|
||||
content: "# Normal File\n\nPlain filename, should always work.\n",
|
||||
},
|
||||
];
|
||||
|
||||
const crazySubFiles: Array<{ name: string; content: string }> = [
|
||||
{
|
||||
name: "Notes #42 - foo@bar.md",
|
||||
content: "# Notes #42\n\nSubdir file with searchterm-gamma.\n",
|
||||
},
|
||||
];
|
||||
|
||||
beforeAll(async () => {
|
||||
testDir = await mkdtemp(join(tmpdir(), "qmd-path-fidelity-"));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await rm(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// Helper: create a fresh isolated test environment with a corpus of crazy filenames.
|
||||
async function createCrazyCollection(prefix: string): Promise<{
|
||||
collectionDir: string;
|
||||
dbPath: string;
|
||||
configDir: string;
|
||||
}> {
|
||||
const envDir = join(testDir, prefix);
|
||||
const collectionDir = join(envDir, "corpus");
|
||||
const dbPath = join(envDir, "test.sqlite");
|
||||
const configDir = join(envDir, "config");
|
||||
|
||||
await mkdir(collectionDir, { recursive: true });
|
||||
await mkdir(join(collectionDir, "subdir"), { recursive: true });
|
||||
await mkdir(configDir, { recursive: true });
|
||||
|
||||
// Resolve symlinks so the path matches what getRealPath() stores in the DB.
|
||||
// On macOS /tmp is a symlink to /private/tmp; without this normalisation
|
||||
// toVirtualPath() and --full-path resolution fail.
|
||||
const realCollectionDir = realpathSync(collectionDir);
|
||||
|
||||
for (const f of crazyFiles) {
|
||||
await writeFile(join(collectionDir, f.name), f.content);
|
||||
}
|
||||
for (const f of crazySubFiles) {
|
||||
await writeFile(join(collectionDir, "subdir", f.name), f.content);
|
||||
}
|
||||
|
||||
// Write empty YAML config — `collection add` will populate it
|
||||
await writeFile(join(configDir, "index.yml"), "collections: {}\n");
|
||||
|
||||
return { collectionDir: realCollectionDir, dbPath, configDir };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests: store-level path storage
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("Path fidelity — store level", () => {
|
||||
test("reindexCollection stores literal relative paths, not handalized slugs", async () => {
|
||||
const { collectionDir, dbPath, configDir } = await createCrazyCollection("store-unit");
|
||||
|
||||
// Run `collection add` to index
|
||||
const add = await runQmd(
|
||||
["collection", "add", collectionDir, "--name", "crazytest"],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(add.exitCode, `collection add failed: ${add.stderr}`).toBe(0);
|
||||
|
||||
// Inspect the DB directly
|
||||
const db = openDatabase(dbPath);
|
||||
const rows = db.prepare(
|
||||
"SELECT path FROM documents WHERE active = 1 ORDER BY path"
|
||||
).all() as { path: string }[];
|
||||
db.close();
|
||||
|
||||
const paths = rows.map((r) => r.path);
|
||||
|
||||
// Must contain literal filenames — not handalized slugs
|
||||
expect(paths).toContain("# Meeting - 234232 3432 __ 5.md");
|
||||
expect(paths).toContain("Budget & Revenue (Q4) [2024].md");
|
||||
expect(paths).toContain("normal-file.md");
|
||||
expect(paths).toContain("subdir/Notes #42 - foo@bar.md");
|
||||
|
||||
// Must NOT contain handalized versions
|
||||
expect(paths).not.toContain("Meeting-234232-3432-5.md");
|
||||
expect(paths).not.toContain("Budget-Revenue-Q4-2024.md");
|
||||
expect(paths).not.toContain("subdir/Notes-42-foo-bar.md");
|
||||
});
|
||||
|
||||
test("toVirtualPath returns non-null for crazy-named files", async () => {
|
||||
const { collectionDir, dbPath, configDir } = await createCrazyCollection("store-to-virtual");
|
||||
const add = await runQmd(
|
||||
["collection", "add", collectionDir, "--name", "crazytest"],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(add.exitCode).toBe(0);
|
||||
|
||||
const rawDb = openDatabase(dbPath);
|
||||
const result = toVirtualPath(rawDb, join(collectionDir, "Budget & Revenue (Q4) [2024].md"));
|
||||
rawDb.close();
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result).toBe(`qmd://crazytest/Budget & Revenue (Q4) [2024].md`);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI integration tests — the five original breakage points
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("Path fidelity — CLI integration", () => {
|
||||
let collectionDir: string;
|
||||
let dbPath: string;
|
||||
let configDir: string;
|
||||
|
||||
// Index once for the whole describe block (read-only tests share it)
|
||||
beforeAll(async () => {
|
||||
({ collectionDir, dbPath, configDir } = await createCrazyCollection("cli-shared"));
|
||||
const add = await runQmd(
|
||||
["collection", "add", collectionDir, "--name", "crazytest"],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(add.exitCode, `collection add failed: ${add.stderr}`).toBe(0);
|
||||
});
|
||||
|
||||
test("(1) search --json file field contains literal path, not handalized slug", async () => {
|
||||
const { stdout, exitCode } = await runQmd(
|
||||
["search", "searchterm-alpha", "--json"],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(exitCode).toBe(0);
|
||||
|
||||
const results = JSON.parse(stdout) as Array<{ file: string }>;
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
|
||||
const meetingResult = results.find((r) => r.file.includes("Meeting"));
|
||||
expect(meetingResult).toBeDefined();
|
||||
// Must contain the literal filename fragment
|
||||
expect(meetingResult!.file).toContain("# Meeting - 234232 3432 __ 5.md");
|
||||
// Must not contain the handalized version
|
||||
expect(meetingResult!.file).not.toContain("Meeting-234232-3432-5.md");
|
||||
});
|
||||
|
||||
test("(2) get --full-path resolves to real filesystem path for crazy-named file", async () => {
|
||||
const virtualPath = `qmd://crazytest/Budget & Revenue (Q4) [2024].md`;
|
||||
const { stdout, exitCode } = await runQmd(
|
||||
["get", virtualPath, "--full-path"],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(exitCode, `get failed: ${stdout}`).toBe(0);
|
||||
|
||||
const header = stdout.split("\n")[0]!;
|
||||
// Should show a real filesystem path, not a qmd:// virtual path
|
||||
expect(header).not.toMatch(/^qmd:\/\//);
|
||||
// Should include the literal filename
|
||||
expect(header).toContain("Budget & Revenue (Q4) [2024].md");
|
||||
// The resolved filesystem path should exist — strip the trailing docid (#abc123)
|
||||
const fsPath = header.trim().replace(/\s+#[a-f0-9]{6}$/, "");
|
||||
// Path may be absolute or relative-to-collectionDir; resolve against collectionDir
|
||||
const absPath = fsPath.startsWith("/") ? fsPath : join(collectionDir, fsPath.replace(/^\.\//, ""));
|
||||
expect(existsSync(absPath), `resolved path does not exist: ${absPath}`).toBe(true);
|
||||
});
|
||||
test("(3) get <actual-fs-path> finds the document", async () => {
|
||||
const fsPath = join(collectionDir, "Budget & Revenue (Q4) [2024].md");
|
||||
const { stdout, exitCode, stderr } = await runQmd(
|
||||
["get", fsPath],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(exitCode, `get by fs path failed: ${stderr}`).toBe(0);
|
||||
// Header should contain the document identifier
|
||||
expect(stdout).toContain("Budget & Revenue (Q4) [2024].md");
|
||||
});
|
||||
|
||||
test("(3b) get <actual-fs-path> finds subdir file with crazy name", async () => {
|
||||
const fsPath = join(collectionDir, "subdir", "Notes #42 - foo@bar.md");
|
||||
const { stdout, exitCode, stderr } = await runQmd(
|
||||
["get", fsPath],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(exitCode, `get subdir file failed: ${stderr}`).toBe(0);
|
||||
expect(stdout).toContain("Notes #42 - foo@bar.md");
|
||||
});
|
||||
|
||||
test("(4) ls shows literal paths, not handalized slugs", async () => {
|
||||
const { stdout, exitCode } = await runQmd(
|
||||
["ls", "crazytest"],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(exitCode).toBe(0);
|
||||
|
||||
// Literal paths must appear
|
||||
expect(stdout).toContain("# Meeting - 234232 3432 __ 5.md");
|
||||
expect(stdout).toContain("Budget & Revenue (Q4) [2024].md");
|
||||
expect(stdout).toContain("Notes #42 - foo@bar.md");
|
||||
|
||||
// Handalized slugs must NOT appear
|
||||
expect(stdout).not.toContain("Meeting-234232-3432-5.md");
|
||||
expect(stdout).not.toContain("Budget-Revenue-Q4-2024.md");
|
||||
expect(stdout).not.toContain("Notes-42-foo-bar.md");
|
||||
});
|
||||
|
||||
test("(5) search --json returns docid that can be fetched back", async () => {
|
||||
const { stdout: searchOut, exitCode: searchExit } = await runQmd(
|
||||
["search", "searchterm-beta", "--json"],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(searchExit).toBe(0);
|
||||
|
||||
const results = JSON.parse(searchOut) as Array<{ docid: string; file: string }>;
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
|
||||
const hit = results[0]!;
|
||||
expect(hit.docid).toMatch(/^#[a-f0-9]{6}$/);
|
||||
|
||||
// Fetch by docid — must work
|
||||
const { stdout: getOut, exitCode: getExit } = await runQmd(
|
||||
["get", hit.docid],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(getExit, `get by docid failed`).toBe(0);
|
||||
expect(getOut).toContain("Budget & Revenue (Q4) [2024].md");
|
||||
});
|
||||
|
||||
test("normal filenames are still stored correctly (regression)", async () => {
|
||||
const { stdout, exitCode } = await runQmd(
|
||||
["search", "Plain filename", "--json"],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(exitCode).toBe(0);
|
||||
const results = JSON.parse(stdout) as Array<{ file: string }>;
|
||||
const hit = results.find((r) => r.file.includes("normal-file"));
|
||||
expect(hit).toBeDefined();
|
||||
expect(hit!.file).toContain("normal-file.md");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Migration test: old handalized DB upgraded by `qmd update`
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("Path fidelity — migration from handalized index", () => {
|
||||
test("qmd update migrates handalized paths to literal paths in existing index", async () => {
|
||||
const { collectionDir, dbPath, configDir } = await createCrazyCollection("migration");
|
||||
|
||||
// Manually build an old-style DB using handalize() (simulates pre-fix index)
|
||||
const store = createStore(dbPath);
|
||||
const now = new Date().toISOString();
|
||||
// Write and sync a config that points at the collection so `qmd update` knows where it is
|
||||
const migrationYaml = `collections:\n crazytest:\n path: "${collectionDir}"\n mask: "**/*.md"\n`;
|
||||
await writeFile(join(configDir, "index.yml"), migrationYaml);
|
||||
const config = YAML.parse(migrationYaml) as CollectionConfig;
|
||||
syncConfigToDb(store.db, config);
|
||||
|
||||
// Insert documents with handalized paths (old behavior)
|
||||
for (const f of crazyFiles) {
|
||||
const relPath = normalizePathSeparators(f.name);
|
||||
const handleized = handelize(relPath);
|
||||
const hash = await hashContent(f.content);
|
||||
insertContent(store.db, hash, f.content, now);
|
||||
insertDocument(store.db, "crazytest", handleized, `Title ${f.name}`, hash, now, now);
|
||||
}
|
||||
const subFile = crazySubFiles[0]!;
|
||||
const subRel = `subdir/${subFile.name}`;
|
||||
const subHandelized = handelize(subRel);
|
||||
const subHash = await hashContent(subFile.content);
|
||||
insertContent(store.db, subHash, subFile.content, now);
|
||||
insertDocument(store.db, "crazytest", subHandelized, "Sub title", subHash, now, now);
|
||||
store.close();
|
||||
|
||||
// Verify the old DB has handalized paths
|
||||
const dbBefore = openDatabase(dbPath);
|
||||
const pathsBefore = (dbBefore.prepare(
|
||||
"SELECT path FROM documents WHERE active = 1 ORDER BY path"
|
||||
).all() as { path: string }[]).map((r) => r.path);
|
||||
dbBefore.close();
|
||||
|
||||
expect(pathsBefore).toContain("Meeting-234232-3432-5.md");
|
||||
expect(pathsBefore).toContain("Budget-Revenue-Q4-2024.md");
|
||||
expect(pathsBefore).not.toContain("# Meeting - 234232 3432 __ 5.md");
|
||||
|
||||
// Run `qmd update` with the new code — should migrate paths in-place
|
||||
const update = await runQmd(
|
||||
["update"],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(update.exitCode, `qmd update failed: ${update.stderr}`).toBe(0);
|
||||
|
||||
// Verify the DB now has literal paths
|
||||
const dbAfter = openDatabase(dbPath);
|
||||
const pathsAfter = (dbAfter.prepare(
|
||||
"SELECT path FROM documents WHERE active = 1 ORDER BY path"
|
||||
).all() as { path: string }[]).map((r) => r.path);
|
||||
dbAfter.close();
|
||||
|
||||
expect(pathsAfter).toContain("# Meeting - 234232 3432 __ 5.md");
|
||||
expect(pathsAfter).toContain("Budget & Revenue (Q4) [2024].md");
|
||||
expect(pathsAfter).toContain("normal-file.md");
|
||||
expect(pathsAfter).toContain("subdir/Notes #42 - foo@bar.md");
|
||||
|
||||
// Handalized slugs must be gone
|
||||
expect(pathsAfter).not.toContain("Meeting-234232-3432-5.md");
|
||||
expect(pathsAfter).not.toContain("Budget-Revenue-Q4-2024.md");
|
||||
|
||||
// Search must work after migration
|
||||
const { stdout: searchOut, exitCode: searchExit } = await runQmd(
|
||||
["search", "searchterm-alpha", "--json"],
|
||||
{ cwd: collectionDir, dbPath, configDir }
|
||||
);
|
||||
expect(searchExit).toBe(0);
|
||||
const results = JSON.parse(searchOut) as Array<{ file: string }>;
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
const meetingResult = results.find((r) => r.file.includes("Meeting"));
|
||||
expect(meetingResult).toBeDefined();
|
||||
expect(meetingResult!.file).toContain("# Meeting - 234232 3432 __ 5.md");
|
||||
});
|
||||
});
|
||||
@ -1604,6 +1604,39 @@ describe("FTS Search", () => {
|
||||
|
||||
await cleanupTestDb(store);
|
||||
});
|
||||
|
||||
test("searchFTS matches dotted version strings like 2026.4.10 (#563)", async () => {
|
||||
// Regression test: porter unicode61 tokenizer splits on dots, so the index
|
||||
// stores "2026", "4", "10" as separate tokens. Before the fix, sanitizeFTS5Term
|
||||
// stripped the dots producing "2026410" which never matched anything.
|
||||
const store = await createTestStore();
|
||||
const collectionName = await createTestCollection();
|
||||
|
||||
await insertTestDocument(store.db, collectionName, {
|
||||
name: "release-notes",
|
||||
title: "Release Notes",
|
||||
body: "## Release 2026.4.10\n\nThis version introduces new features and bug fixes.",
|
||||
displayPath: "test/release-notes.md",
|
||||
});
|
||||
|
||||
// A document that does NOT contain the version string
|
||||
await insertTestDocument(store.db, collectionName, {
|
||||
name: "other-doc",
|
||||
title: "Other Document",
|
||||
body: "Unrelated content about gardening and cooking.",
|
||||
displayPath: "test/other.md",
|
||||
});
|
||||
|
||||
const results = store.searchFTS("2026.4.10", 10);
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
expect(results.map(r => r.displayPath)).toContain(`${collectionName}/test/release-notes.md`);
|
||||
|
||||
// Partial version should also work
|
||||
const partial = store.searchFTS("2026.4", 10);
|
||||
expect(partial.map(r => r.displayPath)).toContain(`${collectionName}/test/release-notes.md`);
|
||||
|
||||
await cleanupTestDb(store);
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
|
||||
Loading…
Reference in New Issue
Block a user