ai-workspace-services/qmd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor evals into separate run and score scripts

Browse Source 
New structure:
- evals/run.py: Generate model outputs to JSONL
- evals/score.py: Score outputs with detailed breakdown
- evals/queries.txt: Test queries (26 total)

Features:
- Supports both HF Hub and local model paths
- Named entity preservation scoring
- Chat template leakage detection
- Strict format validation (every line must be lex:/vec:/hyde:)
- Generic phrase detection

Usage:
  uv run evals/run.py --model tobil/qmd-query-expansion-0.6B-v4
  uv run evals/score.py evals/results_*.jsonl

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Tobi Lutke 2026-01-25 00:40:33 -05:00
parent 857a85ab58
commit f6a6716c44
No known key found for this signature in database
8 changed files with 688 additions and 1794 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
finetune/README.md
View File

@ -80,11 +80,14 @@ See `PROMPT_FORMAT.md` for complete specification.
finetune/
├── train.py # SFT training (uses YAML config)
├── rl.py # GRPO/RL training (uses YAML config)
├── evaluate_model.py # Evaluate finetuned models
├── tui.py # Interactive testing interface
├── configs/
│ ├── sft_v4.yaml # SFT training config
│ └── grpo_v4.yaml # GRPO training config
├── evals/
│ ├── run.py # Generate model outputs to JSONL
│ ├── score.py # Score outputs from JSONL
│ └── queries.txt # Test queries
├── dataset/
│ ├── prepare_data.py # Prepare training data
│ ├── clean_data.py # Data quality improvements
@ -118,7 +121,11 @@ hf jobs uv run --flavor a10g-large --timeout 2h --secrets HF_TOKEN \
### 3. Evaluate
```bash
uv run evaluate_model.py --model tobil/qmd-query-expansion-0.6B-v4
# Generate outputs
uv run evals/run.py --model tobil/qmd-query-expansion-0.6B-v4
# Score them
uv run evals/score.py evals/results_tobil_qmd-query-expansion-0.6B-v4.jsonl
```
### 4. Interactive Testing

3
finetune/evals/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
# Generated results (re-run evals locally)
results_*.jsonl
scores_*.json

42
finetune/evals/queries.txt Normal file
View File

@ -0,0 +1,42 @@
# Test queries for QMD query expansion evaluation
# One query per line, comments start with #
# Technical documentation
how to configure authentication
typescript async await
docker compose networking
git rebase vs merge
react useEffect cleanup
# Short/ambiguous queries
auth
config
setup
api
# Named entities (critical for entity preservation testing)
who is TDS motorsports
React hooks tutorial
Docker container networking
Kubernetes pod deployment
AWS Lambda functions
# Personal notes / journals style
meeting notes project kickoff
ideas for new feature
todo list app architecture
# Research / learning
what is dependency injection
difference between sql and nosql
kubernetes vs docker swarm
# Error/debugging
connection timeout error
memory leak debugging
cors error fix
# Complex queries
how to implement caching with redis in nodejs
best practices for api rate limiting
setting up ci cd pipeline with github actions

195
finetune/evals/run.py Normal file
View File

@ -0,0 +1,195 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "transformers>=4.45.0",
# "peft>=0.7.0",
# "torch",
# "huggingface_hub",
# "accelerate",
# ]
# ///
"""
Generate query expansions from a model and save to JSONL.
Usage:
uv run evals/run.py --model tobil/qmd-query-expansion-0.6B-v4
uv run evals/run.py --model ./local-model --queries evals/queries.txt
uv run evals/run.py --model tobil/qmd-query-expansion-0.6B-v4 --output results.jsonl
"""
import argparse
import json
import sys
from datetime import datetime
from pathlib import Path
import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
def load_queries(path: str) -> list[str]:
"""Load queries from file, one per line, ignoring comments."""
queries = []
with open(path) as f:
for line in f:
line = line.strip()
if line and not line.startswith("#"):
queries.append(line)
return queries
def load_model(model_path: str, base_model: str = None):
"""
Load the model (supports Hub URLs, local adapters, and merged models).
Args:
model_path: HF Hub ID (tobil/model), local adapter dir, or merged model dir
base_model: Base model for tokenizer (auto-detected if not provided)
"""
model_path = Path(model_path) if not model_path.startswith(("http", "hf://")) and "/" not in model_path[:20] else model_path
is_local = isinstance(model_path, Path) and model_path.exists()
# Check if it's an adapter or full model
is_adapter = False
if is_local:
adapter_config = model_path / "adapter_config.json"
is_adapter = adapter_config.exists()
if is_adapter and not base_model:
# Read base model from adapter config
import json
with open(adapter_config) as f:
config = json.load(f)
base_model = config.get("base_model_name_or_path", "Qwen/Qwen3-0.6B")
else:
# For Hub models, assume adapter unless --base-model not provided
is_adapter = base_model is not None
# Default base model
if not base_model:
base_model = "Qwen/Qwen3-0.6B"
print(f"Loading tokenizer from {base_model}...", file=sys.stderr)
tokenizer = AutoTokenizer.from_pretrained(base_model)
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
if is_adapter:
print(f"Loading base model {base_model}...", file=sys.stderr)
base = AutoModelForCausalLM.from_pretrained(
base_model,
torch_dtype=torch.bfloat16,
device_map="auto",
)
print(f"Loading adapter from {model_path}...", file=sys.stderr)
model = PeftModel.from_pretrained(base, str(model_path))
else:
print(f"Loading model from {model_path}...", file=sys.stderr)
model = AutoModelForCausalLM.from_pretrained(
str(model_path),
torch_dtype=torch.bfloat16,
device_map="auto",
)
model.eval()
return model, tokenizer
def generate_expansion(model, tokenizer, query: str, max_new_tokens: int = 200) -> str:
"""Generate query expansion using Qwen3 chat template with /no_think."""
messages = [{"role": "user", "content": f"/no_think Expand this search query: {query}"}]
prompt = tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True
)
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
with torch.no_grad():
outputs = model.generate(
**inputs,
max_new_tokens=max_new_tokens,
temperature=0.7,
do_sample=True,
pad_token_id=tokenizer.pad_token_id,
eos_token_id=tokenizer.eos_token_id,
)
full_output = tokenizer.decode(outputs[0], skip_special_tokens=True)
# Extract assistant response
if "\nassistant\n" in full_output:
expansion = full_output.split("\nassistant\n")[-1].strip()
elif "assistant\n" in full_output:
expansion = full_output.split("assistant\n")[-1].strip()
else:
expansion = full_output[len(prompt):].strip()
# Remove any <think> tags
if expansion.startswith("<think>"):
think_end = expansion.find("</think>")
if think_end != -1:
expansion = expansion[think_end + 8:].strip()
return expansion
def main():
parser = argparse.ArgumentParser(description="Generate query expansions")
parser.add_argument("--model", required=True, help="Model path (Hub or local)")
parser.add_argument("--base-model", default="Qwen/Qwen3-0.6B", help="Base model for tokenizer")
parser.add_argument("--queries", default="evals/queries.txt", help="Queries file")
parser.add_argument("--output", help="Output JSONL file (default: evals/results_{model_name}.jsonl)")
parser.add_argument("--max-tokens", type=int, default=200, help="Max tokens to generate")
args = parser.parse_args()
# Determine output path
if args.output:
output_path = Path(args.output)
else:
model_name = args.model.replace("/", "_").replace(".", "_")
output_path = Path(f"evals/results_{model_name}.jsonl")
# Load queries
queries = load_queries(args.queries)
print(f"Loaded {len(queries)} queries from {args.queries}", file=sys.stderr)
# Load model
model, tokenizer = load_model(args.model, args.base_model)
# Generate expansions
print(f"Generating expansions...", file=sys.stderr)
results = []
for i, query in enumerate(queries, 1):
print(f"[{i}/{len(queries)}] {query}", file=sys.stderr)
expansion = generate_expansion(model, tokenizer, query, args.max_tokens)
results.append({
"query": query,
"expansion": expansion,
})
# Write results
output_path.parent.mkdir(parents=True, exist_ok=True)
with open(output_path, "w") as f:
# Write metadata as first line
metadata = {
"_meta": True,
"model": args.model,
"base_model": args.base_model,
"timestamp": datetime.now().isoformat(),
"num_queries": len(queries),
}
f.write(json.dumps(metadata) + "\n")
# Write results
for result in results:
f.write(json.dumps(result) + "\n")
print(f"Results saved to: {output_path}", file=sys.stderr)
print(str(output_path)) # Print path to stdout for piping
if __name__ == "__main__":
main()

439
finetune/evals/score.py Normal file
View File

@ -0,0 +1,439 @@
# /// script
# requires-python = ">=3.10"
# dependencies = []
# ///
"""
Score query expansion results from a JSONL file.
Usage:
uv run evals/score.py evals/results_model.jsonl
uv run evals/score.py evals/results_model.jsonl --output scores.json
uv run evals/score.py evals/results_model.jsonl --verbose
"""
import argparse
import json
import re
import sys
from collections import Counter
from pathlib import Path
# ============== SCORING CONSTANTS ==============
STOPWORDS = {'the', 'a', 'an', 'is', 'are', 'to', 'for', 'of', 'in', 'and', 'or', 'it', 'this', 'that', 'be', 'with', 'as', 'on', 'by'}
KEY_TERM_STOPWORDS = {'what', 'is', 'how', 'to', 'the', 'a', 'an', 'in', 'on', 'for', 'of',
'and', 'or', 'with', 'my', 'your', 'do', 'does', 'can', 'i', 'me', 'we',
'who', 'where', 'when', 'why', 'which', 'find', 'get', 'show', 'tell'}
GENERIC_LEX_PHRASES = {
'find information about', 'search for', 'look up', 'get information',
'learn about', 'information on', 'details about', 'find out about',
'what is', 'how to', 'guide to', 'help with'
}
# ============== HELPER FUNCTIONS ==============
def extract_named_entities(query: str) -> set:
"""Extract named entities from query using simple heuristics."""
entities = set()
words = query.split()
prev_was_entity = False
for i, word in enumerate(words):
clean = word.strip('.,!?:;()[]"\'')
if not clean:
prev_was_entity = False
continue
is_entity = False
# All-caps words (acronyms): TDS, API, GPU
if clean.isupper() and len(clean) >= 2:
entities.add(clean.lower())
is_entity = True
# Capitalized words (not first word)
elif i > 0 and clean[0].isupper() and clean.lower() not in KEY_TERM_STOPWORDS:
entities.add(clean.lower())
is_entity = True
# Technical terms: node.js, C++
elif any(c in clean for c in '.+-#@') and len(clean) >= 2:
entities.add(clean.lower())
is_entity = True
# CamelCase: JavaScript
elif len(clean) > 1 and any(c.isupper() for c in clean[1:]) and clean[0].isupper():
entities.add(clean.lower())
is_entity = True
# Word following an entity (compound names)
elif prev_was_entity and clean.lower() not in KEY_TERM_STOPWORDS:
entities.add(clean.lower())
is_entity = True
prev_was_entity = is_entity
return entities
def get_key_terms(query: str) -> set:
"""Get key terms (non-stopwords) from query."""
words = set(query.lower().split())
return words - KEY_TERM_STOPWORDS
def lex_preserves_key_terms(lex_line: str, query: str) -> bool:
"""Check if lex line preserves key terms from query."""
key_terms = get_key_terms(query)
if not key_terms:
return True
lex_words = set(lex_line.lower().split())
return bool(key_terms & lex_words)
def lex_preserves_entities(lex_line: str, entities: set) -> bool:
"""Check if lex line contains at least one named entity."""
if not entities:
return True
lex_lower = lex_line.lower()
return any(entity in lex_lower for entity in entities)
def lex_is_generic(lex_line: str) -> bool:
"""Check if lex line is a generic filler phrase."""
lex_lower = lex_line.lower().strip()
for phrase in GENERIC_LEX_PHRASES:
if phrase in lex_lower or lex_lower.startswith(phrase.split()[0]):
remaining = lex_lower
for word in phrase.split():
remaining = remaining.replace(word, '', 1).strip()
if len(remaining) < 3:
return True
return False
def parse_expansion(text: str) -> dict:
"""Parse expansion into structured format."""
lines = text.strip().split("\n")
result = {"lex": [], "vec": [], "hyde": [], "invalid": []}
for line in lines:
line = line.strip()
if not line:
continue
if line.startswith("lex:"):
result["lex"].append(line[4:].strip())
elif line.startswith("vec:"):
result["vec"].append(line[4:].strip())
elif line.startswith("hyde:"):
result["hyde"].append(line[5:].strip())
else:
result["invalid"].append(line)
return result
def edit_distance_simple(a: str, b: str) -> int:
"""Simple word-level edit distance."""
words_a = set(a.lower().split())
words_b = set(b.lower().split())
return len(words_a ^ words_b)
def is_diverse(a: str, b: str, min_distance: int = 2) -> bool:
"""Check if two strings are sufficiently different."""
a, b = a.lower().strip(), b.lower().strip()
if a == b:
return False
if a in b or b in a:
return False
return edit_distance_simple(a, b) >= min_distance
def echoes_query(expansion: str, query: str) -> bool:
"""Check if expansion is just echoing the query."""
exp = expansion.lower().strip()
q = query.lower().strip()
if exp == q:
return True
if q in exp and len(exp) < len(q) + 10:
return True
return False
def word_repetition_penalty(text: str) -> int:
"""Count penalty for repeated words."""
words = re.findall(r'\b\w+\b', text.lower())
counts = Counter(words)
penalty = 0
for word, count in counts.items():
if count >= 3 and word not in STOPWORDS and len(word) > 2:
penalty += (count - 2) * 2
return penalty
# ============== MAIN SCORING FUNCTION ==============
def score_expansion(query: str, expansion: str) -> dict:
"""Score an expansion. Returns detailed breakdown."""
text = expansion.strip()
deductions = []
# HARD FAIL: Chat template artifacts
if any(token in text for token in ['<|im_start|>', '<|im_end|>', '<think>', '</think>',
'\nassistant\n', '\nuser\n', '<|endoftext|>']):
return {
"format": 0, "diversity": 0, "hyde": 0, "quality": 0, "entity": 0,
"total": 0, "max_possible": 100, "percentage": 0, "rating": "Failed",
"deductions": ["CHAT TEMPLATE LEAKAGE"],
"parsed": {"lex": [], "vec": [], "hyde": [], "invalid": [text[:100]]},
"entities_detected": [],
}
# HARD FAIL: Every line must start with lex:, vec:, or hyde:
for line in text.split("\n"):
line = line.strip()
if not line:
continue
if not line.startswith(("lex:", "vec:", "hyde:")):
return {
"format": 0, "diversity": 0, "hyde": 0, "quality": 0, "entity": 0,
"total": 0, "max_possible": 100, "percentage": 0, "rating": "Failed",
"deductions": [f"INVALID LINE: {line[:50]}"],
"parsed": parse_expansion(expansion),
"entities_detected": [],
}
parsed = parse_expansion(expansion)
# FORMAT (0-30)
format_score = 0
if parsed["lex"]:
format_score += 10
else:
deductions.append("missing lex:")
if parsed["vec"]:
format_score += 10
else:
deductions.append("missing vec:")
format_score += 10 # No invalid lines (guaranteed by hard fail)
# DIVERSITY (0-30)
diversity_score = 0
types_present = sum(1 for t in ["lex", "vec"] if parsed[t])
if types_present >= 2:
diversity_score += 10
else:
deductions.append("only one type")
total_expansions = len(parsed["lex"]) + len(parsed["vec"])
if total_expansions >= 2:
diversity_score += 5
lex_score = 5
for i, a in enumerate(parsed["lex"]):
for b in parsed["lex"][i+1:]:
if not is_diverse(a, b, 2):
lex_score -= 2
deductions.append(f"lex duplicate: {a[:20]}...")
diversity_score += max(0, lex_score)
vec_score = 5
for i, a in enumerate(parsed["vec"]):
for b in parsed["vec"][i+1:]:
if not is_diverse(a, b, 3):
vec_score -= 2
deductions.append(f"vec duplicate: {a[:20]}...")
diversity_score += max(0, vec_score)
echo_score = 5
for exp in parsed["lex"] + parsed["vec"]:
if echoes_query(exp, query):
echo_score -= 3
deductions.append(f"echoes query: {exp[:20]}...")
diversity_score += max(0, echo_score)
# HYDE (0-20)
hyde_score = 0
if parsed["hyde"]:
hyde_text = parsed["hyde"][0]
hyde_score += 5
hyde_len = len(hyde_text)
if 50 <= hyde_len <= 200:
hyde_score += 5
elif hyde_len < 50:
hyde_score += 2
deductions.append(f"hyde too short ({hyde_len})")
else:
deductions.append(f"hyde too long ({hyde_len})")
if "\n" not in hyde_text:
hyde_score += 5
rep_penalty = word_repetition_penalty(hyde_text)
hyde_score += max(0, 5 - rep_penalty)
# QUALITY (0-20)
quality_score = 5
if parsed["lex"] and parsed["vec"]:
avg_lex = sum(len(l) for l in parsed["lex"]) / len(parsed["lex"])
avg_vec = sum(len(v) for v in parsed["vec"]) / len(parsed["vec"])
if avg_lex <= avg_vec:
quality_score += 5
else:
deductions.append("lex longer than vec")
if parsed["vec"]:
natural = sum(1 for v in parsed["vec"] if " " in v and len(v) > 15)
if natural == len(parsed["vec"]):
quality_score += 5
else:
quality_score += 2
if parsed["lex"]:
lex_with_terms = sum(1 for l in parsed["lex"] if lex_preserves_key_terms(l, query))
if lex_with_terms == len(parsed["lex"]):
quality_score += 5
elif lex_with_terms > 0:
quality_score += 2
else:
deductions.append("lex missing key terms")
# NAMED ENTITY PRESERVATION (0-20, can go negative)
entity_score = 0
entities = extract_named_entities(query)
if entities and parsed["lex"]:
lex_with_entities = sum(1 for l in parsed["lex"] if lex_preserves_entities(l, entities))
if lex_with_entities == len(parsed["lex"]):
entity_score += 15
elif lex_with_entities > 0:
entity_score += 5
else:
entity_score -= 30
deductions.append(f"lex missing entities: {entities}")
generic_count = sum(1 for l in parsed["lex"] if lex_is_generic(l))
if generic_count > 0:
entity_score -= generic_count * 15
deductions.append(f"{generic_count} generic lex phrases")
if parsed["vec"]:
vec_with_entities = sum(1 for v in parsed["vec"] if lex_preserves_entities(v, entities))
if vec_with_entities > 0:
entity_score += 5
elif not entities:
entity_score = 10
# TOTAL
total = format_score + diversity_score + hyde_score + quality_score + entity_score
max_possible = 120 if parsed["hyde"] else 100
percentage = max(0.0, min(100.0, total / max_possible * 100))
# Rating
if percentage >= 80:
rating = "Excellent"
elif percentage >= 60:
rating = "Good"
elif percentage >= 40:
rating = "Acceptable"
elif percentage >= 20:
rating = "Poor"
else:
rating = "Failed"
return {
"format": format_score,
"diversity": diversity_score,
"hyde": hyde_score,
"quality": quality_score,
"entity": max(0, entity_score),
"total": max(0, total),
"max_possible": max_possible,
"percentage": round(percentage, 1),
"rating": rating,
"deductions": deductions,
"parsed": parsed,
"entities_detected": list(entities) if entities else [],
}
def print_result(query: str, expansion: str, scores: dict, verbose: bool = False):
"""Print a single result."""
print(f"\n{'='*60}")
print(f"Query: {query}")
print(f"{'─'*60}")
print(expansion)
print(f"{'─'*60}")
print(f"Score: {scores['percentage']:.0f}% ({scores['rating']})")
print(f" Format: {scores['format']}/30 Diversity: {scores['diversity']}/30 "
f"Hyde: {scores['hyde']}/20 Quality: {scores['quality']}/20 Entity: {scores['entity']}/20")
if verbose and scores["deductions"]:
print(f" Deductions: {', '.join(scores['deductions'][:5])}")
if verbose and scores["entities_detected"]:
print(f" Entities: {scores['entities_detected']}")
def main():
parser = argparse.ArgumentParser(description="Score query expansion results")
parser.add_argument("input", help="Input JSONL file from run.py")
parser.add_argument("--output", help="Output JSON file with scores")
parser.add_argument("--verbose", "-v", action="store_true", help="Show detailed output")
parser.add_argument("--summary-only", action="store_true", help="Only show summary")
args = parser.parse_args()
# Load results
results = []
metadata = None
with open(args.input) as f:
for line in f:
data = json.loads(line)
if data.get("_meta"):
metadata = data
else:
results.append(data)
print(f"Scoring {len(results)} results from {args.input}", file=sys.stderr)
if metadata:
print(f"Model: {metadata.get('model', 'unknown')}", file=sys.stderr)
# Score each result
scored_results = []
for result in results:
query = result["query"]
expansion = result["expansion"]
scores = score_expansion(query, expansion)
if not args.summary_only:
print_result(query, expansion, scores, args.verbose)
scored_results.append({
"query": query,
"expansion": expansion,
"scores": {k: v for k, v in scores.items() if k not in ["parsed", "deductions", "entities_detected"]},
"deductions": scores["deductions"],
"entities_detected": scores["entities_detected"],
})
# Summary
print(f"\n{'='*60}")
print("SUMMARY")
print(f"{'='*60}")
avg_score = sum(r["scores"]["percentage"] for r in scored_results) / len(scored_results)
ratings = Counter(r["scores"]["rating"] for r in scored_results)
print(f" Total queries: {len(scored_results)}")
print(f" Average score: {avg_score:.1f}%")
print(f" Ratings:")
for rating in ["Excellent", "Good", "Acceptable", "Poor", "Failed"]:
count = ratings.get(rating, 0)
bar = "█" * count
print(f" {rating:10s}: {count:2d} {bar}")
# Save output
if args.output:
output_data = {
"metadata": metadata,
"summary": {
"total": len(scored_results),
"average_score": round(avg_score, 1),
"ratings": dict(ratings),
},
"results": scored_results,
}
with open(args.output, "w") as f:
json.dump(output_data, f, indent=2)
print(f"\nScores saved to: {args.output}")
if __name__ == "__main__":
main()

580
finetune/evaluate_model.py
View File

@ -1,580 +0,0 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "transformers>=4.45.0",
# "peft>=0.7.0",
# "torch",
# "huggingface_hub",
# "accelerate",
# ]
# ///
"""
Evaluate QMD query expansion model quality.
See SCORING.md for detailed scoring criteria.
"""
import json
import re
import torch
from collections import Counter
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
# Test queries covering different QMD use cases
TEST_QUERIES = [
# Technical documentation
"how to configure authentication",
"typescript async await",
"docker compose networking",
"git rebase vs merge",
"react useEffect cleanup",
# Short/ambiguous queries
"auth",
"config",
"setup",
"api",
# Personal notes / journals style
"meeting notes project kickoff",
"ideas for new feature",
"todo list app architecture",
# Research / learning
"what is dependency injection",
"difference between sql and nosql",
"kubernetes vs docker swarm",
# Error/debugging
"connection timeout error",
"memory leak debugging",
"cors error fix",
# Complex queries
"how to implement caching with redis in nodejs",
"best practices for api rate limiting",
"setting up ci cd pipeline with github actions",
]
# Prompt is generated via tokenizer.apply_chat_template() - see generate_expansion()
# Don't manually construct <|im_start|> tags
STOPWORDS = {'the', 'a', 'an', 'is', 'are', 'to', 'for', 'of', 'in', 'and', 'or', 'it', 'this', 'that', 'be', 'with', 'as', 'on', 'by'}
def parse_expansion(text: str) -> dict:
"""Parse expansion into structured format."""
lines = text.strip().split("\n")
result = {"lex": [], "vec": [], "hyde": [], "invalid": []}
for line in lines:
line = line.strip()
if not line:
continue
if line.startswith("lex:"):
result["lex"].append(line[4:].strip())
elif line.startswith("vec:"):
result["vec"].append(line[4:].strip())
elif line.startswith("hyde:"):
result["hyde"].append(line[5:].strip())
else:
result["invalid"].append(line)
return result
def edit_distance_simple(a: str, b: str) -> int:
"""Simple word-level edit distance."""
words_a = set(a.lower().split())
words_b = set(b.lower().split())
return len(words_a ^ words_b) # Symmetric difference
def is_diverse(a: str, b: str, min_distance: int = 2) -> bool:
"""Check if two strings are sufficiently different."""
a, b = a.lower().strip(), b.lower().strip()
if a == b:
return False
if a in b or b in a:
return False
return edit_distance_simple(a, b) >= min_distance
def echoes_query(expansion: str, query: str) -> bool:
"""Check if expansion is just echoing the query."""
exp = expansion.lower().strip()
q = query.lower().strip()
# Exact match or very close
if exp == q:
return True
# Query is contained in expansion with little else
if q in exp and len(exp) < len(q) + 10:
return True
return False
def get_key_terms(query: str) -> set:
"""Extract key terms from query (excluding stopwords)."""
stopwords = {'what', 'is', 'how', 'to', 'the', 'a', 'an', 'in', 'on', 'for', 'of',
'and', 'or', 'with', 'my', 'your', 'do', 'does', 'can', 'i', 'me', 'we'}
words = set(query.lower().split())
return words - stopwords
def lex_preserves_key_terms(lex_line: str, query: str) -> bool:
"""Check if lex line contains at least one key term from query."""
key_terms = get_key_terms(query)
if not key_terms: # Very short query
return True
lex_words = set(lex_line.lower().split())
return bool(key_terms & lex_words)
def word_repetition_penalty(text: str) -> int:
"""Count penalty for repeated words (excluding stopwords)."""
words = re.findall(r'\b\w+\b', text.lower())
counts = Counter(words)
penalty = 0
for word, count in counts.items():
if count >= 3 and word not in STOPWORDS and len(word) > 2:
penalty += (count - 2) * 2
return penalty
def is_continuation(expansion: str) -> bool:
"""
Detect if output is a continuation rather than proper expansion.
A continuation is when the model continues the query as prose
instead of outputting lex:/vec:/hyde: lines.
"""
text = expansion.strip()
if not text:
return True
# Check first non-empty line
first_line = text.split("\n")[0].strip()
# Valid outputs must start with a prefix
valid_prefixes = ("lex:", "vec:", "hyde:")
if first_line.startswith(valid_prefixes):
return False
# If first line doesn't have a valid prefix, it's a continuation
# Exception: empty first line (check second)
lines = [l.strip() for l in text.split("\n") if l.strip()]
if lines and not lines[0].startswith(valid_prefixes):
return True
return False
def score_expansion(query: str, expansion: str) -> dict:
"""
Score an expansion based on SCORING.md criteria.
Returns dict with score breakdown and total (0-100).
"""
# HARD FAIL: Continuation detection
if is_continuation(expansion):
return {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": ["CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"],
"parsed": {"lex": [], "vec": [], "hyde": [], "invalid": [expansion[:100]]},
"is_continuation": True,
}
parsed = parse_expansion(expansion)
scores = {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"deductions": [],
}
# === FORMAT (0-30) ===
format_score = 0
# Has at least one lex: line (+10)
if parsed["lex"]:
format_score += 10
else:
scores["deductions"].append("missing lex: (-10)")
# Has at least one vec: line (+10)
if parsed["vec"]:
format_score += 10
else:
scores["deductions"].append("missing vec: (-10)")
# All lines have valid prefix (+10, -5 per invalid)
if not parsed["invalid"]:
format_score += 10
else:
invalid_penalty = min(10, len(parsed["invalid"]) * 5)
format_score += (10 - invalid_penalty)
scores["deductions"].append(f"{len(parsed['invalid'])} invalid lines (-{invalid_penalty})")
scores["format"] = max(0, format_score)
# === DIVERSITY (0-30) ===
diversity_score = 0
# 2+ different types present (+10)
types_present = sum(1 for t in ["lex", "vec"] if parsed[t])
if types_present >= 2:
diversity_score += 10
else:
scores["deductions"].append("only one type present (-10)")
# 2+ total expansions (+5)
total_expansions = len(parsed["lex"]) + len(parsed["vec"])
if total_expansions >= 2:
diversity_score += 5
else:
scores["deductions"].append("fewer than 2 expansions (-5)")
# Multiple lex: lines are diverse (+5, -2 per duplicate pair)
lex_diverse_score = 5
for i, a in enumerate(parsed["lex"]):
for b in parsed["lex"][i+1:]:
if not is_diverse(a, b, min_distance=2):
lex_diverse_score -= 2
scores["deductions"].append(f"lex duplicates: '{a[:20]}...' ~ '{b[:20]}...'")
diversity_score += max(0, lex_diverse_score)
# Multiple vec: lines are diverse (+5, -2 per duplicate pair)
vec_diverse_score = 5
for i, a in enumerate(parsed["vec"]):
for b in parsed["vec"][i+1:]:
if not is_diverse(a, b, min_distance=3):
vec_diverse_score -= 2
scores["deductions"].append(f"vec duplicates: '{a[:20]}...' ~ '{b[:20]}...'")
diversity_score += max(0, vec_diverse_score)
# lex/vec not identical to original query (+5, -5 per echo)
echo_score = 5
for exp in parsed["lex"] + parsed["vec"]:
if echoes_query(exp, query):
echo_score -= 5
scores["deductions"].append(f"echoes query: '{exp[:30]}...'")
diversity_score += max(0, echo_score)
scores["diversity"] = max(0, diversity_score)
# === HYDE QUALITY (0-20, optional bonus) ===
hyde_score = 0
if parsed["hyde"]:
hyde_text = parsed["hyde"][0] # Only first hyde counts
# Hyde present and well-formed (+5)
hyde_score += 5
# Hyde is concise: 50-200 chars (+5)
hyde_len = len(hyde_text)
if 50 <= hyde_len <= 200:
hyde_score += 5
elif hyde_len < 50:
hyde_score += 2
scores["deductions"].append(f"hyde too short ({hyde_len} chars)")
else:
scores["deductions"].append(f"hyde too long ({hyde_len} chars)")
# Hyde has no newlines (+5)
if "\n" not in hyde_text:
hyde_score += 5
else:
scores["deductions"].append("hyde contains newlines")
# Hyde has no excessive repetition (+5)
rep_penalty = word_repetition_penalty(hyde_text)
if rep_penalty == 0:
hyde_score += 5
else:
hyde_score += max(0, 5 - rep_penalty)
scores["deductions"].append(f"hyde repetition penalty (-{min(5, rep_penalty)})")
scores["hyde"] = hyde_score
# === QUALITY (0-20) ===
quality_score = 10 # Base relevance (assume relevant unless obvious garbage)
# Lex lines should be keyword-focused (shorter than vec on average)
if parsed["lex"] and parsed["vec"]:
avg_lex = sum(len(l) for l in parsed["lex"]) / len(parsed["lex"])
avg_vec = sum(len(v) for v in parsed["vec"]) / len(parsed["vec"])
if avg_lex <= avg_vec:
quality_score += 5
else:
scores["deductions"].append("lex longer than vec (should be keywords)")
else:
quality_score += 2 # Partial credit
# Vec lines should be natural language (contain spaces, longer)
if parsed["vec"]:
vec_natural = sum(1 for v in parsed["vec"] if " " in v and len(v) > 15)
if vec_natural == len(parsed["vec"]):
quality_score += 5
else:
quality_score += 2
scores["deductions"].append("some vec lines too short/keyword-like")
# Lex lines must preserve key terms from query (not be generic)
if parsed["lex"]:
lex_with_terms = sum(1 for l in parsed["lex"] if lex_preserves_key_terms(l, query))
if lex_with_terms == len(parsed["lex"]):
quality_score += 5
elif lex_with_terms > 0:
quality_score += 2
else:
scores["deductions"].append("lex lines too generic - missing key terms from query")
scores["quality"] = min(20, quality_score) # Cap at 20
# === TOTAL ===
scores["total"] = scores["format"] + scores["diversity"] + scores["hyde"] + scores["quality"]
scores["max_possible"] = 100 if parsed["hyde"] else 80
scores["percentage"] = scores["total"] / scores["max_possible"] * 100
# Rating
pct = scores["percentage"]
if pct >= 80:
scores["rating"] = "Excellent"
elif pct >= 60:
scores["rating"] = "Good"
elif pct >= 40:
scores["rating"] = "Acceptable"
elif pct >= 20:
scores["rating"] = "Poor"
else:
scores["rating"] = "Failed"
scores["parsed"] = parsed
scores["is_continuation"] = False
return scores
def load_model(model_name: str, base_model: str = "Qwen/Qwen3-0.6B"):
"""Load the finetuned model."""
print(f"Loading tokenizer from {base_model}...")
tokenizer = AutoTokenizer.from_pretrained(base_model)
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
print(f"Loading base model...")
base = AutoModelForCausalLM.from_pretrained(
base_model,
torch_dtype=torch.bfloat16,
device_map="auto",
)
print(f"Loading adapter from {model_name}...")
model = PeftModel.from_pretrained(base, model_name)
model.eval()
return model, tokenizer
def generate_expansion(model, tokenizer, query: str, max_new_tokens: int = 200) -> str:
"""Generate query expansion using proper Qwen3 chat template."""
# Use tokenizer's chat template with /no_think to disable thinking mode
messages = [{"role": "user", "content": f"/no_think Expand this search query: {query}"}]
prompt = tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True
)
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
with torch.no_grad():
outputs = model.generate(
**inputs,
max_new_tokens=max_new_tokens,
temperature=0.7,
do_sample=True,
pad_token_id=tokenizer.pad_token_id,
eos_token_id=tokenizer.eos_token_id,
)
# Decode and extract expansion
# skip_special_tokens=True strips <|im_start|> etc, leaving "user\n...\nassistant\n..."
full_output = tokenizer.decode(outputs[0], skip_special_tokens=True)
# Extract assistant response
if "\nassistant\n" in full_output:
expansion = full_output.split("\nassistant\n")[-1].strip()
elif "assistant\n" in full_output:
expansion = full_output.split("assistant\n")[-1].strip()
else:
# Fallback: strip prompt length
expansion = full_output[len(prompt):].strip()
# Remove any <think> tags that might remain
if expansion.startswith("<think>"):
# Find end of thinking block
think_end = expansion.find("</think>")
if think_end != -1:
expansion = expansion[think_end + 8:].strip()
return expansion
def print_score_breakdown(scores: dict):
"""Pretty print score breakdown."""
print(f" Score: {scores['total']}/{scores['max_possible']} ({scores['percentage']:.0f}%) - {scores['rating']}")
print(f" Format: {scores['format']}/30")
print(f" Diversity: {scores['diversity']}/30")
print(f" Hyde: {scores['hyde']}/20")
print(f" Quality: {scores['quality']}/20")
if scores["deductions"]:
print(f" Deductions:")
for d in scores["deductions"][:5]: # Show top 5
print(f" - {d}")
if len(scores["deductions"]) > 5:
print(f" ... and {len(scores['deductions']) - 5} more")
def run_examples():
"""Run good and bad examples to demonstrate scoring."""
print("=" * 70)
print("SCORING EXAMPLES")
print("=" * 70)
# Good example
good_expansion = """lex: react hooks tutorial
lex: usestate useeffect
vec: how to use react hooks in functional components
vec: react hooks best practices guide
hyde: React Hooks allow you to use state and lifecycle features in functional components without writing a class."""
print("\n[GOOD EXAMPLE]")
print(f"Query: react hooks")
print(f"Output:\n{good_expansion}")
scores = score_expansion("react hooks", good_expansion)
print_score_breakdown(scores)
# Bad example
bad_expansion = """auth is an important concept that relates to authentication.
The answer should be in Chinese.
The answer should be in Chinese."""
print("\n[BAD EXAMPLE]")
print(f"Query: auth")
print(f"Output:\n{bad_expansion}")
scores = score_expansion("auth", bad_expansion)
print_score_breakdown(scores)
# Medium example - repetitive hyde
medium_expansion = """lex: docker networking
vec: docker networking
hyde: Docker networking is an important concept. Docker networking is used for container communication. Docker networking configuration is essential."""
print("\n[MEDIUM EXAMPLE - Repetitive]")
print(f"Query: docker networking")
print(f"Output:\n{medium_expansion}")
scores = score_expansion("docker networking", medium_expansion)
print_score_breakdown(scores)
# Medium example - echoes query
echo_expansion = """lex: auth
lex: authentication
vec: auth
vec: authentication configuration
hyde: Authentication is the process of verifying identity."""
print("\n[MEDIUM EXAMPLE - Echoes Query]")
print(f"Query: auth")
print(f"Output:\n{echo_expansion}")
scores = score_expansion("auth", echo_expansion)
print_score_breakdown(scores)
print("\n" + "=" * 70)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--model", default="tobil/qmd-query-expansion-0.6B",
help="Model to evaluate")
parser.add_argument("--base-model", default="Qwen/Qwen3-0.6B",
help="Base model")
parser.add_argument("--output", default="evaluation_results.json",
help="Output file for results")
parser.add_argument("--queries", type=str, help="Custom queries file (one per line)")
parser.add_argument("--examples", action="store_true", help="Run scoring examples only")
args = parser.parse_args()
# Run examples if requested
if args.examples:
run_examples()
return
# Load custom queries if provided
queries = TEST_QUERIES
if args.queries:
with open(args.queries) as f:
queries = [l.strip() for l in f if l.strip()]
# Load model
model, tokenizer = load_model(args.model, args.base_model)
# Run evaluation
results = []
print(f"\n{'='*70}")
print("EVALUATION RESULTS")
print(f"{'='*70}\n")
for i, query in enumerate(queries, 1):
print(f"[{i}/{len(queries)}] Query: {query}")
print("-" * 50)
expansion = generate_expansion(model, tokenizer, query)
scores = score_expansion(query, expansion)
print(expansion)
print()
print_score_breakdown(scores)
print()
results.append({
"query": query,
"expansion": expansion,
"scores": {k: v for k, v in scores.items() if k != "parsed"},
"parsed": scores["parsed"],
})
# Summary
print(f"\n{'='*70}")
print("SUMMARY")
print(f"{'='*70}")
avg_score = sum(r["scores"]["percentage"] for r in results) / len(results)
excellent = sum(1 for r in results if r["scores"]["rating"] == "Excellent")
good = sum(1 for r in results if r["scores"]["rating"] == "Good")
acceptable = sum(1 for r in results if r["scores"]["rating"] == "Acceptable")
poor = sum(1 for r in results if r["scores"]["rating"] == "Poor")
failed = sum(1 for r in results if r["scores"]["rating"] == "Failed")
print(f" Total queries: {len(results)}")
print(f" Average score: {avg_score:.1f}%")
print(f" Ratings:")
print(f" Excellent: {excellent}")
print(f" Good: {good}")
print(f" Acceptable: {acceptable}")
print(f" Poor: {poor}")
print(f" Failed: {failed}")
# Save results
with open(args.output, "w") as f:
json.dump(results, f, indent=2)
print(f"\n Results saved to: {args.output}")
if __name__ == "__main__":
main()

548
finetune/evaluation_grpo_failed.json
View File

@ -1,548 +0,0 @@
[
{
"query": "how to configure authentication",
"expansion": "Here is an expanded version of your search query for configuring authentication:\n\n**\"How to configure authentication settings in [specific platform or system]?\"**\n\nIf you have a specific platform or system in mind (such as web apps, mobile apps, APIs, or a database system), please clarify, and I can tailor the query further.",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Here is an expanded version of your search query for configuring authentication:\n\n**\"How to configur"
]
}
},
{
"query": "typescript async await",
"expansion": "The query \"typescript async await\" can be expanded in several ways depending on the context in which it's used:\n\n1. **Programming Language Use**: \n - \"Typescript: Async/await with Await\" \n - \"Typescript Async/await Example\" \n - \"How to use Async/await in Typescript\"\n\n2. **Learning or Documentation**: \n - \"Typescript Async/await: Explained\" \n - \"Async/await in Typescript: Learn and Example\"\n\n3. **Technical Support or Guides**: \n - \"Typescript Async/await: Learn, Use, and Example\" \n - \"Async/await in Typescript: Guide and Example\"\n\n4. **Code Examples**: \n - \"How to use async/await in Typescript\" \n - \"Async/await in Typescript: Code Example\"\n\nIf you need further assistance, feel free to let me know!",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"The query \"typescript async await\" can be expanded in several ways depending on the context in which"
]
}
},
{
"query": "docker compose networking",
"expansion": "Here's an expanded version of the search query \"docker compose networking\":\n\n**\"docker compose networking query\"**\n\nThis query can be used to find information related to Docker Compose, networking configurations, or network settings in Docker.",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Here's an expanded version of the search query \"docker compose networking\":\n\n**\"docker compose netwo"
]
}
},
{
"query": "git rebase vs merge",
"expansion": "Here's an expanded version of the search query \"git rebase vs merge\" with more context and explanation:\n\n**\"git rebase vs merge** \nThis question is commonly asked when working with Git, and it's important to understand the difference between **rebase** and **merge** in order to make informed decisions about how to proceed with your Git workflow. \n\n### What is **git rebase**?\n- **git rebase** is a command used to **fix up** and **merge** changes back into a common branch or commit history. \n- It is typically used to apply changes to the current branch, and it often **reverts** or **corrects** changes that have been made to a different branch. \n- It is useful when you want to **correct** or **update** your branch history to match the main branch or a specific commit.\n\n### What is **git merge**?\n- **git merge** is a command used to **",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Here's an expanded version of the search query \"git rebase vs merge\" with more context and explanati"
]
}
},
{
"query": "react useEffect cleanup",
"expansion": "To expand the search query **\"react useEffect cleanup\"**, you can consider the following variations:\n\n### 1. **React Hook Clean Up**\n- **\"react useEffect cleanup code\"**\n- **\"react useEffect cleanup best practices\"**\n- **\"react useEffect cleanup function\"**\n\n### 2. **React Clean Up with useEffect**\n- **\"react cleanup useEffect best\"**\n- **\"react cleanup useEffect code\"**\n- **\"react cleanup useEffect example\"**\n\n### 3. **React Clean Up and useEffect**\n- **\"react cleanup useEffect code\"**\n- **\"react cleanup useEffect code example\"**\n- **\"react cleanup useEffect best practices\"**\n\n### 4. **React Cleanup with useEffect**\n- **\"react cleanup useEffect code\"**\n- **\"react cleanup useEffect example\"**\n\n### 5. **React Clean Up Function with useEffect**\n- **\"react cleanup useEffect function\"**\n- **\"react cleanup useEffect code\"**\n\nThese variations help",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"To expand the search query **\"react useEffect cleanup\"**, you can consider the following variations:"
]
}
},
{
"query": "auth",
"expansion": "The search query \"auth\" can be expanded to include various related terms and concepts. Here are some possible expansions:\n\n1. **Authentication** - This refers to the process of verifying a user or system's identity. \n2. **User Authentication** - A specific type of authentication mechanism used to verify user access. \n3. **Secure Authentication** - A method to ensure that only authorized users can access a system or service. \n4. **Authentication and Authorization** - Combining the concepts of authentication and authorization. \n5. **Role-Based Access Control** - A system that determines access rights based on user roles. \n6. **User Verification** - The act of confirming a user's identity. \n7. **Multi-Factor Authentication (MFA)** - An additional layer of security beyond a password. \n8. **API Authentication** - Authentication required for accessing an API. \n9. **OAuth** - A protocol for authorization that allows apps to access user",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"The search query \"auth\" can be expanded to include various related terms and concepts. Here are some"
]
}
},
{
"query": "config",
"expansion": "Expanding the search query \"config\" would involve modifying the search to include more context or details. Here are some possible expansions:\n\n1. **config details** - to find information about specific configuration settings \n2. **config settings** - to find configuration parameters \n3. **configurations** - to find configuration options or settings \n4. **config file** - to find configuration files \n5. **config management** - to find configuration management tools or practices \n6. **config file structure** - to find configuration file structures \n7. **config values** - to find configuration values \n8. **config options** - to find configuration options \n9. **config parameters** - to find configuration parameters \n10. **config settings** - to find configuration settings \n\nLet me know if you meant to search for a specific type of configuration!",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Expanding the search query \"config\" would involve modifying the search to include more context or de"
]
}
},
{
"query": "setup",
"expansion": "Expanding the search query **\"setup\"** would involve adding more context or specific terms to provide a more accurate result. Here are some possibilities:\n\n1. **\"Setup a project\"** \n2. **\"Set up a system\"** \n3. **\"Configure a setup\"** \n4. **\"Configuration of a setup\"** \n5. **\"Install a setup\"** \n6. **\"Create a setup\"** \n7. **\"Install or configure a setup\"** \n8. **\"Setup a configuration\"** \n9. **\"Set up a system configuration\"** \n10. **\"Configuration setup\"** \n\nLet me know if you meant a specific task or context!",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Expanding the search query **\"setup\"** would involve adding more context or specific terms to provid"
]
}
},
{
"query": "api",
"expansion": "Certainly! Here's an expanded version of your search query:\n\n**\"api | search api | api details | api documentation | api usage | api integration | api development | api best practices | api implementation | api testing | api troubleshooting | api documentation | api examples | api reference | api installation | api configuration | api security | api optimization | api maintenance | api architecture | api deployment | api testing tools | api monitoring | api performance | api scalability | api troubleshooting | api best practices | api documentation | api examples | api reference | api documentation | api examples | api reference | api documentation | api examples | api reference | api documentation | api examples | api reference | api documentation | api examples | api reference | api documentation | api examples | api reference | api documentation | api examples | api reference | api documentation | api examples | api reference | api documentation | api examples | api reference | api documentation | api examples | api reference | api documentation | api examples | api reference |",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Certainly! Here's an expanded version of your search query:\n\n**\"api | search api | api details | api"
]
}
},
{
"query": "meeting notes project kickoff",
"expansion": "Expanding the search query **\"meeting notes project kickoff\"**, you can refine it to:\n\n- **\"meeting notes for project kickoff\"** \n- **\"project kickoff meeting notes\"** \n- **\"meeting notes project kickoff details\"** \n- **\"meeting notes on project kickoff\"** \n- **\"meeting notes for project launch\"** \n\nThese variations provide different ways to phrase or search for meeting notes related to a project's kickoff event.",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Expanding the search query **\"meeting notes project kickoff\"**, you can refine it to:\n\n- **\"meeting "
]
}
},
{
"query": "ideas for new feature",
"expansion": "Here are some **ideas** for new features you could consider:\n\n1. **User Authentication & Authorization**: Add new methods for users to log in (e.g., social media, email, phone) and manage access.\n\n2. **Real-time Data Integration**: Integrate with APIs or databases to fetch real-time data and update the app accordingly.\n\n3. **Mobile-First Design**: Focus on mobile-friendly UI and functionality, especially for a mobile app.\n\n4. **Customization & Personalization**: Allow users to customize their experience (e.g., themes, preferences).\n\n5. **Multilingual Support**: Offer features to support users from different countries or regions.\n\n6. **Gamification Elements**: Integrate game-like features (e.g., points, badges) to encourage user engagement.\n\n7. **AI & Machine Learning**: Incorporate AI to improve features (e.g., personalized recommendations, chatbots).\n\n8. **Cross-Platform Compatibility**: Ensure the app works seamlessly across different",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Here are some **ideas** for new features you could consider:\n\n1. **User Authentication & Authorizati"
]
}
},
{
"query": "todo list app architecture",
"expansion": "Here's an expanded version of the search query: \n\n**\"Todo list app architecture\"** \n\nThis query seeks information on the structure, design, and implementation of a **todo list application**. It covers topics such as:\n\n- **Architecture Overview** \n- **Key Components** (e.g., UI, backend, data storage) \n- **Design Patterns** used in todo apps \n- **Scalability and performance** \n- **User Interface and User Experience (UIUX)** \n- **Database Systems** (e.g., SQLite, MongoDB) \n- **APIs and Integration** \n- **Security considerations** \n- **Development tools** and frameworks used (e.g., React, Flutter, Node.js) \n\nLet me know if you'd like a breakdown of specific components or case studies!",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Here's an expanded version of the search query: \n\n**\"Todo list app architecture\"** \n\nThis query se"
]
}
},
{
"query": "what is dependency injection",
"expansion": "Dependency injection (DI) is a design pattern used in software development to **independently manage the dependencies** of a class or module. The idea is that the program **involves no direct dependency between classes**, but instead, the implementation of these classes is **provided by a container**\u2014a system that automatically manages the dependencies.\n\n### Key Concepts:\n- **Inversion of Control (IoC)**: This is the core principle of DI. Instead of the programmer defining the dependencies, the container handles the dependency relationships.\n- **DI Containers**: These are the systems that manage the dependencies, such as Spring, Unity, or others.\n- **DI Inverses**: These are the classes that are \"injected\" into other classes, such as a `Person` class that receives an `Email` and `Name`.\n\n### Example:\n```java\nclass Person {\n private String name;\n private Email email;\n\n public Person(String name, Email email) {",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Dependency injection (DI) is a design pattern used in software development to **independently manage"
]
}
},
{
"query": "difference between sql and nosql",
"expansion": "The difference between SQL and NoSQL is a fundamental distinction in the world of databases. Here's a breakdown:\n\n### **1. SQL (Structured Query Language)**\n- **Purpose**: SQL is designed for structured data and is used to manage and manipulate relational databases. It is built on the relational model where data is stored in tables and organized in rows and columns.\n- **Key Features**:\n - **Strong typing** and **data integrity**.\n - **Support for complex queries** and transactions.\n - **Standardized syntax** and conventions.\n- **Used in**: Relational databases like MySQL, PostgreSQL, and SQL Server.\n- **Best for**: Data that is structured, static, and requires complex queries.\n\n### **2. NoSQL**\n- **Purpose**: NoSQL is a family of non-relational databases that do not follow a fixed schema. They are designed to store and retrieve data in a flexible, scalable, and dynamic way.\n- **Key",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"The difference between SQL and NoSQL is a fundamental distinction in the world of databases. Here's "
]
}
},
{
"query": "kubernetes vs docker swarm",
"expansion": "Here is an expanded version of your search query:\n\n**\"Kubernetes vs Docker Swarm\"**\n\nThis query highlights the comparison between **Kubernetes** and **Docker Swarm**, focusing on their roles, features, and use cases in cloud computing. If you'd like to refine it further, feel free to provide more context!",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Here is an expanded version of your search query:\n\n**\"Kubernetes vs Docker Swarm\"**\n\nThis query high"
]
}
},
{
"query": "connection timeout error",
"expansion": "Expanding the search query \"connection timeout error\" to provide more context and detail, here are some possible expansions:\n\n1. **\"Connection timeout error in web services\"** \n This focuses on the issue occurring in web-based applications or services.\n\n2. **\"How to resolve a connection timeout error in programming\"** \n This expands the query to include troubleshooting steps.\n\n3. **\"Connection timeout error symptoms and solutions\"** \n This highlights the symptoms and possible solutions related to the error.\n\n4. **\"Common causes and resolutions for connection timeout errors\"** \n This provides a comprehensive overview of the problem and how to address it.\n\n5. **\"Troubleshooting connection timeout error in different technologies\"** \n This expands to include different programming languages or platforms.\n\nLet me know if you'd like help with a specific area (e.g., web development, network, programming) or need further details!",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Expanding the search query \"connection timeout error\" to provide more context and detail, here are s"
]
}
},
{
"query": "memory leak debugging",
"expansion": "\"Memory leak debugging\" is a common issue in software development, particularly in applications that use memory management. Here's an expanded version of the search query to include more context:\n\n**Expanded Search Query:**\n\n- Memory leak debugging techniques\n- How to debug memory leaks in software\n- Best practices for identifying and resolving memory leaks\n- Common causes of memory leaks in applications\n- Tools for memory leak debugging\n- Techniques to prevent memory leaks\n- Debugging memory leaks in different programming languages\n\nThis query can help you find resources, tutorials, or guides on effectively debugging memory leaks in your code.",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"\"Memory leak debugging\" is a common issue in software development, particularly in applications that"
]
}
},
{
"query": "cors error fix",
"expansion": "Here\u2019s an expanded version of the search query:\n\n**\"How to fix a CORS error in your website?\"**\n\nThis query allows you to ask about the general process of resolving a CORS (Cross-Origin Resource Sharing) error. If you have a specific browser or framework in mind, you can tailor the search further.",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Here\u2019s an expanded version of the search query:\n\n**\"How to fix a CORS error in your website?\"**\n\nThi"
]
}
},
{
"query": "how to implement caching with redis in nodejs",
"expansion": "To implement caching with Redis in Node.js, you can use the Redis module from the Node.js standard library. Here's a step-by-step guide:\n\n### 1. **Install the Redis Module**\nFirst, install the `redis` package using npm:\n```bash\nnpm install redis --save\n```\n\n### 2. **Set Up Redis Connection**\nYou need to establish a connection to Redis. You can do this by creating a Redis client instance. Below is a simple example:\n\n```javascript\nconst { Redis } = require('redis');\n\n// Connect to Redis\nconst redis = Redis.createClient({\n host: 'localhost',\n port: 6379,\n database: 0, // Default database\n});\n\n// Set a value in Redis\nredis.set('key1', 'value1', (err) => {\n if (err) {\n console.error('Error setting Redis value:', err);\n } else {\n console.log('Value",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"To implement caching with Redis in Node.js, you can use the Redis module from the Node.js standard l"
]
}
},
{
"query": "best practices for api rate limiting",
"expansion": "Here is an expanded search query for \"best practices for API rate limiting\":\n\n**\"Best practices for API rate limiting\"**\n\nThis query allows for a broader search, including results that explain the principles, implementation methods, and effective strategies for API rate limiting.",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Here is an expanded search query for \"best practices for API rate limiting\":\n\n**\"Best practices for "
]
}
},
{
"query": "setting up ci cd pipeline with github actions",
"expansion": "Setting up a CI/CD pipeline with GitHub Actions is a common practice in modern software development. Here's an expanded version of the search query to help you better understand and find solutions:\n\n**\"How to set up a CI/CD pipeline with GitHub Actions in a scalable and efficient way?\"**\n\n### Key Features and Benefits:\n1. **Scalability**: GitHub Actions can handle complex pipelines across multiple environments.\n2. **Efficiency**: It automates repetitive tasks, reducing manual intervention.\n3. **Integration**: Works seamlessly with GitHub, GitLab, and other platforms.\n4. **Version Control**: Supports multiple Git repositories.\n5. **CI/CD Best Practices**: Includes best practices for setting up and managing pipelines.\n\n### Popular Options:\n- **GitHub Actions**: A lightweight and open-source tool for CI/CD pipelines.\n- **Jenkins (or GitLab CI)**: Commercial tools with more advanced features.\n- **Azure DevOps**: For enterprise-level CI/CD",
"scores": {
"format": 0,
"diversity": 0,
"hyde": 0,
"quality": 0,
"total": 0,
"max_possible": 80,
"percentage": 0,
"rating": "Failed",
"deductions": [
"CONTINUATION DETECTED - output does not start with lex:/vec:/hyde:"
],
"is_continuation": true
},
"parsed": {
"lex": [],
"vec": [],
"hyde": [],
"invalid": [
"Setting up a CI/CD pipeline with GitHub Actions is a common practice in modern software development."
]
}
}
]

664
finetune/evaluation_sft.json
View File

@ -1,664 +0,0 @@
[
{
"query": "how to configure authentication",
"expansion": "lex: authentication settings\nlex: user login\nvec: authentication settings\nvec: user login\nhyde: To configure authentication, start by reviewing the requirements and dependencies. Configure authentication settings in your project configuration.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 19,
"deductions": [
"some vec lines too short/keyword-like"
],
"total": 99,
"max_possible": 100,
"percentage": 99.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"authentication settings",
"user login"
],
"vec": [
"authentication settings",
"user login"
],
"hyde": [
"To configure authentication, start by reviewing the requirements and dependencies. Configure authentication settings in your project configuration."
],
"invalid": []
}
},
{
"query": "typescript async await",
"expansion": "lex: asynchronous await\nlex: await typescript\nvec: asynchronous await\nvec: await typescript\nhyde: Typescript async await is an important concept that relates to asynchronous await. It provides functionality for various use cases in software development.",
"scores": {
"format": 30,
"diversity": 28,
"hyde": 20,
"quality": 20,
"deductions": [
"vec duplicates: 'asynchronous await...' ~ 'await typescript...'"
],
"total": 98,
"max_possible": 100,
"percentage": 98.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"asynchronous await",
"await typescript"
],
"vec": [
"asynchronous await",
"await typescript"
],
"hyde": [
"Typescript async await is an important concept that relates to asynchronous await. It provides functionality for various use cases in software development."
],
"invalid": []
}
},
{
"query": "docker compose networking",
"expansion": "lex: docker-compose config\nlex: docker-compose settings\nvec: docker-compose config\nvec: docker-compose settings\nhyde: Docker compose networking is an important concept that relates to docker-compose settings. It provides functionality for various use cases in software development.",
"scores": {
"format": 30,
"diversity": 28,
"hyde": 20,
"quality": 20,
"deductions": [
"vec duplicates: 'docker-compose confi...' ~ 'docker-compose setti...'",
"lex lines too generic - missing key terms from query"
],
"total": 98,
"max_possible": 100,
"percentage": 98.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"docker-compose config",
"docker-compose settings"
],
"vec": [
"docker-compose config",
"docker-compose settings"
],
"hyde": [
"Docker compose networking is an important concept that relates to docker-compose settings. It provides functionality for various use cases in software development."
],
"invalid": []
}
},
{
"query": "git rebase vs merge",
"expansion": "lex: difference between rebase\nlex: what's the\nvec: difference between rebase and merge\nvec: what's the difference between git rebase and merge\nhyde: The topic of git rebase vs merge covers what's the difference between rebase and merge.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"difference between rebase",
"what's the"
],
"vec": [
"difference between rebase and merge",
"what's the difference between git rebase and merge"
],
"hyde": [
"The topic of git rebase vs merge covers what's the difference between rebase and merge."
],
"invalid": []
}
},
{
"query": "react useEffect cleanup",
"expansion": "lex: react cleanup code\nlex: react cleanup example\nvec: code examples for react useEffect cleanup\nvec: practical useEffect cleanup implementation examples\nhyde: Here are some practical examples of useEffect cleanup in action. Each example demonstrates a common use case with working code.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"react cleanup code",
"react cleanup example"
],
"vec": [
"code examples for react useEffect cleanup",
"practical useEffect cleanup implementation examples"
],
"hyde": [
"Here are some practical examples of useEffect cleanup in action. Each example demonstrates a common use case with working code."
],
"invalid": []
}
},
{
"query": "auth",
"expansion": "lex: access\nlex: login\nvec: access\nvec: login\nhyde: The topic of auth covers sign in. Proper implementation follows established patterns and best practices.",
"scores": {
"format": 30,
"diversity": 28,
"hyde": 20,
"quality": 17,
"deductions": [
"vec duplicates: 'access...' ~ 'login...'",
"some vec lines too short/keyword-like",
"lex lines too generic - missing key terms from query"
],
"total": 95,
"max_possible": 100,
"percentage": 95.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"access",
"login"
],
"vec": [
"access",
"login"
],
"hyde": [
"The topic of auth covers sign in. Proper implementation follows established patterns and best practices."
],
"invalid": []
}
},
{
"query": "config",
"expansion": "lex: config fix\nlex: config error\nvec: troubleshooting common config problems\nvec: how to fix config errors\nhyde: If you encounter config issues, check your configuration first. Common problems include missing dependencies and incorrect settings.",
"scores": {
"format": 30,
"diversity": 25,
"hyde": 20,
"quality": 20,
"deductions": [
"echoes query: 'config fix...'",
"echoes query: 'config error...'"
],
"total": 95,
"max_possible": 100,
"percentage": 95.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"config fix",
"config error"
],
"vec": [
"troubleshooting common config problems",
"how to fix config errors"
],
"hyde": [
"If you encounter config issues, check your configuration first. Common problems include missing dependencies and incorrect settings."
],
"invalid": []
}
},
{
"query": "setup",
"expansion": "lex: install\nlex: setup guide\nvec: install setup tutorial\nvec: setup guide\nhyde: This guide covers setup configuration and usage. Follow the steps below to get started with your first implementation.",
"scores": {
"format": 30,
"diversity": 25,
"hyde": 20,
"quality": 19,
"deductions": [
"echoes query: 'setup guide...'",
"echoes query: 'setup guide...'",
"some vec lines too short/keyword-like"
],
"total": 94,
"max_possible": 100,
"percentage": 94.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"install",
"setup guide"
],
"vec": [
"install setup tutorial",
"setup guide"
],
"hyde": [
"This guide covers setup configuration and usage. Follow the steps below to get started with your first implementation."
],
"invalid": []
}
},
{
"query": "api",
"expansion": "lex: api troubleshooting\nlex: api fixes\nvec: troubleshooting common api problems\nvec: how to fix api errors\nhyde: If you encounter api issues, check your configuration first. Common problems include missing dependencies and incorrect settings.",
"scores": {
"format": 30,
"diversity": 25,
"hyde": 20,
"quality": 20,
"deductions": [
"echoes query: 'api fixes...'"
],
"total": 95,
"max_possible": 100,
"percentage": 95.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"api troubleshooting",
"api fixes"
],
"vec": [
"troubleshooting common api problems",
"how to fix api errors"
],
"hyde": [
"If you encounter api issues, check your configuration first. Common problems include missing dependencies and incorrect settings."
],
"invalid": []
}
},
{
"query": "meeting notes project kickoff",
"expansion": "lex: invite attendees to\nlex: agenda setup and\nvec: invite attendees to organize a meeting notes project\nvec: agenda setup and preparation for meeting notes\nhyde: Understanding meeting notes project kickoff is essential for modern development. Key aspects include agenda setup and preparation for.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [
"lex lines too generic - missing key terms from query"
],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"invite attendees to",
"agenda setup and"
],
"vec": [
"invite attendees to organize a meeting notes project",
"agenda setup and preparation for meeting notes"
],
"hyde": [
"Understanding meeting notes project kickoff is essential for modern development. Key aspects include agenda setup and preparation for."
],
"invalid": []
}
},
{
"query": "ideas for new feature",
"expansion": "lex: brainstorm new enhancements\nlex: how to develop\nvec: brainstorm new enhancements\nvec: how to develop a new feature idea\nhyde: Ideas for new feature is an important concept that relates to how to develop a new feature idea.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"brainstorm new enhancements",
"how to develop"
],
"vec": [
"brainstorm new enhancements",
"how to develop a new feature idea"
],
"hyde": [
"Ideas for new feature is an important concept that relates to how to develop a new feature idea."
],
"invalid": []
}
},
{
"query": "todo list app architecture",
"expansion": "lex: task manager design\nlex: list app structure\nvec: task manager design\nvec: list app structure\nhyde: Todo list app architecture is an important concept that relates to task manager design.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"task manager design",
"list app structure"
],
"vec": [
"task manager design",
"list app structure"
],
"hyde": [
"Todo list app architecture is an important concept that relates to task manager design."
],
"invalid": []
}
},
{
"query": "what is dependency injection",
"expansion": "lex: definition of dependency\nlex: how to use\nvec: definition of dependency injection and its benefits\nvec: how to use dependency injection in code\nhyde: Dependency injection is defined as examples of dependency injection in practice. This plays a crucial role in modern development practices.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"definition of dependency",
"how to use"
],
"vec": [
"definition of dependency injection and its benefits",
"how to use dependency injection in code"
],
"hyde": [
"Dependency injection is defined as examples of dependency injection in practice. This plays a crucial role in modern development practices."
],
"invalid": []
}
},
{
"query": "difference between sql and nosql",
"expansion": "lex: distinctions between sql\nlex: sql vs nosql\nvec: distinctions between sql and nosql\nvec: sql vs nosql\nhyde: The topic of difference between sql and nosql covers distinctions between sql and nosql. Proper implementation follows established patterns and best practices.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [
"some vec lines too short/keyword-like"
],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"distinctions between sql",
"sql vs nosql"
],
"vec": [
"distinctions between sql and nosql",
"sql vs nosql"
],
"hyde": [
"The topic of difference between sql and nosql covers distinctions between sql and nosql. Proper implementation follows established patterns and best practices."
],
"invalid": []
}
},
{
"query": "kubernetes vs docker swarm",
"expansion": "lex: docker vs\nlex: kubernetes and\nvec: docker vs kubernetes\nvec: kubernetes and docker comparison\nhyde: The topic of kubernetes vs docker swarm covers comparison between kubernetes and docker.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"docker vs",
"kubernetes and"
],
"vec": [
"docker vs kubernetes",
"kubernetes and docker comparison"
],
"hyde": [
"The topic of kubernetes vs docker swarm covers comparison between kubernetes and docker."
],
"invalid": []
}
},
{
"query": "connection timeout error",
"expansion": "lex: error handling for\nlex: connection timeout\nvec: error handling for connection timeouts\nvec: connection timeout handling in software\nhyde: If you encounter connection timeout error, verify that the issue is resolved by checking the configuration.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"error handling for",
"connection timeout"
],
"vec": [
"error handling for connection timeouts",
"connection timeout handling in software"
],
"hyde": [
"If you encounter connection timeout error, verify that the issue is resolved by checking the configuration."
],
"invalid": []
}
},
{
"query": "memory leak debugging",
"expansion": "lex: troubleshooting common memory\nlex: memory leak fix\nvec: troubleshooting common memory leaks\nvec: memory leak fix\nhyde: Memory leak debugging is an important concept that relates to memory leak fix. Proper implementation follows established patterns and best practices.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [
"some vec lines too short/keyword-like"
],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"troubleshooting common memory",
"memory leak fix"
],
"vec": [
"troubleshooting common memory leaks",
"memory leak fix"
],
"hyde": [
"Memory leak debugging is an important concept that relates to memory leak fix. Proper implementation follows established patterns and best practices."
],
"invalid": []
}
},
{
"query": "cors error fix",
"expansion": "lex: cors fix\nlex: cors error handling\nvec: how to fix cors errors\nvec: common cors troubleshooting problems\nhyde: If you encounter cors error fix issues, check your configuration first. Common problems include missing dependencies and incorrect settings.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"cors fix",
"cors error handling"
],
"vec": [
"how to fix cors errors",
"common cors troubleshooting problems"
],
"hyde": [
"If you encounter cors error fix issues, check your configuration first. Common problems include missing dependencies and incorrect settings."
],
"invalid": []
}
},
{
"query": "how to implement caching with redis in nodejs",
"expansion": "lex: steps for setting\nlex: guide to configuring\nvec: steps for setting up redis caching in nodejs\nvec: guide to configuring redis caching in nodejs\nhyde: To implement caching with redis in nodejs, first configure the redis settings in your project configuration.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [
"lex lines too generic - missing key terms from query"
],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"steps for setting",
"guide to configuring"
],
"vec": [
"steps for setting up redis caching in nodejs",
"guide to configuring redis caching in nodejs"
],
"hyde": [
"To implement caching with redis in nodejs, first configure the redis settings in your project configuration."
],
"invalid": []
}
},
{
"query": "best practices for api rate limiting",
"expansion": "lex: recommendations for implementing\nlex: guidelines for setting\nvec: recommendations for implementing rate limiting in api calls\nvec: guidelines for setting up api rate limiting\nhyde: The process of best practices for api rate limiting involves several steps. First, recommendations for implementing rate limiting in api calls.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [
"lex lines too generic - missing key terms from query"
],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"recommendations for implementing",
"guidelines for setting"
],
"vec": [
"recommendations for implementing rate limiting in api calls",
"guidelines for setting up api rate limiting"
],
"hyde": [
"The process of best practices for api rate limiting involves several steps. First, recommendations for implementing rate limiting in api calls."
],
"invalid": []
}
},
{
"query": "setting up ci cd pipeline with github actions",
"expansion": "lex: how to configure\nlex: guide to configuring\nvec: how to configure github actions for ci cd pipeline\nvec: guide to configuring a github actions ci cd pipeline\nhyde: Setting up ci cd pipeline with github actions requires several steps. First, setup the necessary dependencies.",
"scores": {
"format": 30,
"diversity": 30,
"hyde": 20,
"quality": 20,
"deductions": [
"lex lines too generic - missing key terms from query"
],
"total": 100,
"max_possible": 100,
"percentage": 100.0,
"rating": "Excellent",
"is_continuation": false
},
"parsed": {
"lex": [
"how to configure",
"guide to configuring"
],
"vec": [
"how to configure github actions for ci cd pipeline",
"guide to configuring a github actions ci cd pipeline"
],
"hyde": [
"Setting up ci cd pipeline with github actions requires several steps. First, setup the necessary dependencies."
],
"invalid": []
}
}
]