ai-workspace-services/qmd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change format to /only:lex (slash prefix)

Browse Source
This commit is contained in:
Tobi Lütke 2026-01-31 16:24:18 +00:00
parent 46ff098361
commit 20aef8a3e9
3 changed files with 4455 additions and 4455 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8888
finetune/data/qmd_only_variants.jsonl
View File

File diff suppressed because it is too large Load Diff

8
finetune/generate_only_variants.py
View File

@ -81,8 +81,8 @@ def generate_only_variants(input_query: str, output: str) -> list[dict]:
for only_type in ["lex", "vec", "hyde"]:
formatted = format_output(parsed, only_type)
if formatted:
# Add the 'only:' suffix to the query (no space after colon)
new_query = f"{input_query} only:{only_type}"
# Add the '/only:' suffix to the query (slash prefix)
new_query = f"{input_query} /only:{only_type}"
variants.append({
"input": new_query,
"output": formatted,
@ -120,8 +120,8 @@ def process_file(input_path: Path) -> list[dict]:
if not input_query or not output:
continue
# Skip if query already has 'only:' suffix
if " only: " in input_query.lower():
# Skip if query already has '/only:' suffix
if " /only:" in input_query.lower():
continue
# Skip duplicates

14
finetune/reward.py
View File

@ -27,8 +27,8 @@ from collections import Counter
# =============================================================================
# "only:" mode patterns - when query ends with these, expect only that type
# Format: "query only:lex" (no space after colon)
ONLY_MODE_PATTERN = re.compile(r'\s+only:(lex|vec|hyde)\s*$', re.IGNORECASE)
# Format: "query /only:lex" (slash prefix, no space after colon)
ONLY_MODE_PATTERN = re.compile(r'\s+/only:(lex|vec|hyde)\s*$', re.IGNORECASE)
STOPWORDS = frozenset({
'the', 'a', 'an', 'is', 'are', 'to', 'for', 'of', 'in',
@ -578,11 +578,11 @@ if __name__ == "__main__":
("how to use React hooks", "lex: React hooks tutorial\nlex: useEffect useState\nvec: how to use React hooks in functional components"),
("auth", "<think>Let me think...</think>\nlex: auth"),
("auth", "lex: auth\nThis is some explanation\nvec: more"),
# "only:" mode tests (no space after colon)
("auth only:lex", "lex: auth setup\nlex: authentication config\nlex: login credentials"),
("auth only:lex", "lex: auth setup\nvec: how to configure authentication"), # should fail - has vec
("React hooks only:vec", "vec: how to use React hooks in functional components\nvec: useState and useEffect patterns in React"),
("PostgreSQL indexing only:hyde", "hyde: PostgreSQL uses B-tree indexes by default. Create indexes with CREATE INDEX idx_name ON table(column). EXPLAIN ANALYZE shows whether queries use indexes efficiently."),
# "/only:" mode tests (slash prefix)
("auth /only:lex", "lex: auth setup\nlex: authentication config\nlex: login credentials"),
("auth /only:lex", "lex: auth setup\nvec: how to configure authentication"), # should fail - has vec
("React hooks /only:vec", "vec: how to use React hooks in functional components\nvec: useState and useEffect patterns in React"),
("PostgreSQL indexing /only:hyde", "hyde: PostgreSQL uses B-tree indexes by default. Create indexes with CREATE INDEX idx_name ON table(column). EXPLAIN ANALYZE shows whether queries use indexes efficiently."),
]
for query, expansion in tests: