Merge main into finetune

Brings in:
- /only: variants for single-type expansions
- LLM session management for lifecycle safety
- skills.sh integration for AI agent discovery
- Various bug fixes for vector search and embeddings

Merge conflicts resolved by keeping hyde-first format ordering
from finetune branch while accepting expanded templates and
new features from main.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Tobi Lutke 2026-01-31 12:10:22 -05:00
commit 7de18ee066
No known key found for this signature in database
24 changed files with 9618 additions and 1388 deletions

1
.gitignore vendored
View File

@ -8,3 +8,4 @@ texts/
*.md
!README.md
!CLAUDE.md
!skills/**/*.md

View File

@ -213,14 +213,25 @@ QMD uses three local GGUF models (auto-downloaded on first use):
|-------|---------|------|
| `embeddinggemma-300M-Q8_0` | Vector embeddings | ~300MB |
| `qwen3-reranker-0.6b-q8_0` | Re-ranking | ~640MB |
| [`qmd-query-expansion-1.7B-q4_k_m`](https://huggingface.co/tobil/qmd-query-expansion-1.7B-gguf) | Query expansion (fine-tuned) | ~1.1GB |
| `qmd-query-expansion-1.7B-q4_k_m` | Query expansion (fine-tuned) | ~1.1GB |
Models are downloaded from HuggingFace and cached in `~/.cache/qmd/models/`.
## Installation
```sh
bun install -g github:tobi/qmd
```
Make sure `~/.bun/bin` is in your PATH.
### Development
```sh
git clone https://github.com/tobi/qmd
cd qmd
bun install
bun link
```
## Usage

View File

@ -0,0 +1,38 @@
# SFT Training Config - v4 with /only: support
# Usage: accelerate launch --config_file configs/accelerate_multi_gpu.yaml train.py sft --config configs/sft_v4.yaml
model:
base: "Qwen/Qwen3-1.7B"
output: "qmd-sft-v4"
dataset:
name: "data/train_v4"
text_field: "text"
split: "train"
eval_split: 0.1
training:
epochs: 3
batch_size: 2
gradient_accumulation_steps: 4
learning_rate: 0.0002
max_length: 512
warmup_ratio: 0.03
lr_scheduler: "cosine"
lora:
rank: 16
alpha: 32
dropout: 0.0
target_modules:
- "q_proj"
- "k_proj"
- "v_proj"
- "o_proj"
- "gate_proj"
- "up_proj"
- "down_proj"
tracking:
project: "qmd-query-expansion"
run_name: "sft-1.7B-v4-only-modes"

View File

@ -0,0 +1,336 @@
{"input": "ssh key authentication only: lex", "output": "lex: ssh key auth setup\nlex: ssh public private key pair\nlex: passwordless ssh login"}
{"input": "ssh key authentication only: vec", "output": "vec: how to set up ssh key-based authentication instead of passwords\nvec: step-by-step guide to generating and configuring ssh keys for secure server access"}
{"input": "ssh key authentication only: hyde", "output": "hyde: Generate an SSH key pair with ssh-keygen -t ed25519. Copy the public key to ~/.ssh/authorized_keys on the remote server using ssh-copy-id. Ensure permissions are 700 for .ssh and 600 for authorized_keys."}
{"input": "Python virtual environments only: lex", "output": "lex: python venv virtualenv\nlex: pip virtual environment setup\nlex: python isolated dependencies"}
{"input": "Python virtual environments only: vec", "output": "vec: how to create and activate a python virtual environment for project isolation\nvec: what is the difference between venv, virtualenv, and conda for managing python dependencies"}
{"input": "Python virtual environments only: hyde", "output": "hyde: Create a virtual environment with python -m venv myenv, then activate it with source myenv/bin/activate on Unix or myenv\\Scripts\\activate on Windows. Install packages with pip and they stay isolated from your system Python."}
{"input": "git merge conflicts only: lex", "output": "lex: git merge conflict resolve\nlex: git conflict markers HEAD\nlex: resolving merge conflicts"}
{"input": "git merge conflicts only: vec", "output": "vec: how to resolve merge conflicts in git when two branches modify the same lines\nvec: what do the conflict markers mean and how do you manually edit conflicted files"}
{"input": "git merge conflicts only: hyde", "output": "hyde: Git marks conflicts with <<<<<<< HEAD, =======, and >>>>>>> branch-name. Edit the file to keep the code you want, remove the markers, then git add the file and commit. Use git mergetool for a visual diff interface."}
{"input": "TCP vs UDP only: lex", "output": "lex: tcp udp protocol difference\nlex: tcp reliable udp fast\nlex: connection-oriented vs connectionless"}
{"input": "TCP vs UDP only: vec", "output": "vec: what are the key differences between TCP and UDP network protocols\nvec: when should you use TCP versus UDP for application networking"}
{"input": "TCP vs UDP only: hyde", "output": "hyde: TCP provides reliable, ordered delivery with acknowledgments and retransmission. UDP is faster but unreliable\u2014packets may arrive out of order or not at all. Use TCP for web, email, file transfer. Use UDP for video streaming, gaming, DNS where speed matters more than reliability."}
{"input": "Docker compose volumes only: lex", "output": "lex: docker compose volume mount\nlex: docker persistent storage volumes\nlex: compose yaml volumes section"}
{"input": "Docker compose volumes only: vec", "output": "vec: how to configure persistent volumes in docker compose for data that survives container restarts\nvec: what is the difference between bind mounts and named volumes in docker compose"}
{"input": "Docker compose volumes only: hyde", "output": "hyde: In docker-compose.yml, define volumes under the top-level volumes key and reference them in services. Named volumes persist data in Docker's storage. Bind mounts map host directories directly: volumes: - ./data:/app/data for development, - myvolume:/app/data for production."}
{"input": "regex lookahead lookbehind only: lex", "output": "lex: regex lookahead assertion\nlex: regex lookbehind positive negative\nlex: zero-width assertions regex"}
{"input": "regex lookahead lookbehind only: vec", "output": "vec: how do lookahead and lookbehind assertions work in regular expressions\nvec: what is the syntax for positive and negative lookahead and lookbehind in regex"}
{"input": "regex lookahead lookbehind only: hyde", "output": "hyde: Lookahead (?=pattern) matches a position followed by pattern without consuming it. Negative lookahead (?!pattern) matches where pattern doesn't follow. Lookbehind (?<=pattern) matches a position preceded by pattern. Example: \\d+(?= dollars) matches numbers followed by 'dollars'."}
{"input": "Kubernetes secrets management only: lex", "output": "lex: kubernetes secrets k8s\nlex: k8s secret yaml base64\nlex: kubectl create secret"}
{"input": "Kubernetes secrets management only: vec", "output": "vec: how to create and use secrets in kubernetes for sensitive configuration data\nvec: what are best practices for managing secrets in kubernetes clusters"}
{"input": "Kubernetes secrets management only: hyde", "output": "hyde: Create secrets with kubectl create secret generic mysecret --from-literal=password=abc123. Reference in pods via env valueFrom secretKeyRef or volume mounts. Secrets are base64 encoded, not encrypted\u2014use sealed-secrets or external secret managers like Vault for production."}
{"input": "CORS errors fix only: lex", "output": "lex: cors error fix browser\nlex: access-control-allow-origin header\nlex: cors preflight request"}
{"input": "CORS errors fix only: vec", "output": "vec: how to fix CORS errors when making API requests from a web browser\nvec: what causes cross-origin resource sharing errors and how do you configure the server to allow them"}
{"input": "CORS errors fix only: hyde", "output": "hyde: CORS errors occur when a browser blocks requests to a different origin. Fix by adding Access-Control-Allow-Origin headers on the server. For Express: app.use(cors()). For preflight requests, handle OPTIONS and return Access-Control-Allow-Methods and Access-Control-Allow-Headers."}
{"input": "PostgreSQL indexes explain only: lex", "output": "lex: postgresql index explain analyze\nlex: postgres btree index performance\nlex: create index postgresql"}
{"input": "PostgreSQL indexes explain only: vec", "output": "vec: how to use EXPLAIN ANALYZE to understand query performance and index usage in postgresql\nvec: what types of indexes does postgresql support and when should you use each"}
{"input": "PostgreSQL indexes explain only: hyde", "output": "hyde: Run EXPLAIN ANALYZE SELECT... to see the query plan and actual execution time. Look for Seq Scan on large tables\u2014add an index with CREATE INDEX idx_name ON table(column). B-tree indexes work for equality and range queries, GIN for full-text search and arrays, GiST for geometric data."}
{"input": "JWT token refresh only: lex", "output": "lex: jwt refresh token flow\nlex: access token refresh token\nlex: jwt token expiration renewal"}
{"input": "JWT token refresh only: vec", "output": "vec: how does the jwt refresh token flow work for maintaining user sessions\nvec: what is the difference between access tokens and refresh tokens in jwt authentication"}
{"input": "JWT token refresh only: hyde", "output": "hyde: Access tokens are short-lived (15 min) and sent with each request. Refresh tokens are long-lived (days/weeks) and stored securely. When the access token expires, send the refresh token to /auth/refresh to get a new access token without re-authenticating."}
{"input": "React useEffect cleanup only: lex", "output": "lex: react useeffect cleanup function\nlex: useeffect return cleanup\nlex: react unmount cleanup"}
{"input": "React useEffect cleanup only: vec", "output": "vec: how to properly clean up side effects in react useeffect to prevent memory leaks\nvec: when does the useeffect cleanup function run and what should you clean up"}
{"input": "React useEffect cleanup only: hyde", "output": "hyde: Return a cleanup function from useEffect to run before the component unmounts or before the effect re-runs. Use it to cancel subscriptions, clear timers, and abort fetch requests. Example: useEffect(() => { const id = setInterval(fn, 1000); return () => clearInterval(id); }, []);"}
{"input": "nginx reverse proxy only: lex", "output": "lex: nginx reverse proxy config\nlex: nginx proxy_pass upstream\nlex: nginx load balancer setup"}
{"input": "nginx reverse proxy only: vec", "output": "vec: how to configure nginx as a reverse proxy to forward requests to backend servers\nvec: what nginx directives do you need for a basic reverse proxy configuration"}
{"input": "nginx reverse proxy only: hyde", "output": "hyde: In nginx.conf, use proxy_pass inside a location block: location /api { proxy_pass http://localhost:3000; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; }. Add upstream blocks for load balancing across multiple backend servers."}
{"input": "systemd service file only: lex", "output": "lex: systemd service unit file\nlex: systemctl enable start service\nlex: systemd service configuration"}
{"input": "systemd service file only: vec", "output": "vec: how to create a systemd service file to run an application as a linux daemon\nvec: what are the essential sections and directives in a systemd unit file"}
{"input": "systemd service file only: hyde", "output": "hyde: Create /etc/systemd/system/myapp.service with [Unit] Description, [Service] ExecStart=/path/to/app, Restart=always, User=appuser, and [Install] WantedBy=multi-user.target. Run systemctl daemon-reload, then systemctl enable --now myapp."}
{"input": "websocket vs http only: lex", "output": "lex: websocket http difference\nlex: websocket persistent connection\nlex: http polling vs websocket"}
{"input": "websocket vs http only: vec", "output": "vec: what are the differences between websockets and http for real-time communication\nvec: when should you use websockets instead of http long polling or server-sent events"}
{"input": "websocket vs http only: hyde", "output": "hyde: HTTP is request-response: client asks, server answers, connection closes. WebSocket upgrades HTTP to a persistent bidirectional connection. Use WebSocket for chat, live updates, gaming. Use SSE for server-to-client only streaming. HTTP polling wastes bandwidth with repeated requests."}
{"input": "SQL injection prevention only: lex", "output": "lex: sql injection prevent parameterized\nlex: prepared statements sql injection\nlex: sql injection sanitize input"}
{"input": "SQL injection prevention only: vec", "output": "vec: how to prevent sql injection attacks in web applications\nvec: why are parameterized queries and prepared statements important for database security"}
{"input": "SQL injection prevention only: hyde", "output": "hyde: Never concatenate user input into SQL strings. Use parameterized queries: cursor.execute('SELECT * FROM users WHERE id = ?', (user_id,)). ORMs like SQLAlchemy handle this automatically. Validate and sanitize input, but parameterization is the primary defense."}
{"input": "TypeScript generics only: lex", "output": "lex: typescript generics type parameter\nlex: typescript generic function interface\nlex: ts generics constraints extends"}
{"input": "TypeScript generics only: vec", "output": "vec: how to use generics in typescript to write reusable type-safe functions and classes\nvec: what is the syntax for generic type parameters and constraints in typescript"}
{"input": "TypeScript generics only: hyde", "output": "hyde: Generics let you write flexible, reusable code while maintaining type safety. Declare with angle brackets: function identity<T>(arg: T): T { return arg; }. Add constraints with extends: function getLength<T extends { length: number }>(item: T): number { return item.length; }."}
{"input": "OAuth 2.0 authorization code flow only: lex", "output": "lex: oauth2 authorization code flow\nlex: oauth authorization code grant\nlex: oauth2 pkce code verifier"}
{"input": "OAuth 2.0 authorization code flow only: vec", "output": "vec: how does the oauth 2.0 authorization code flow work for secure third-party authentication\nvec: what are the steps in the oauth authorization code grant and why is pkce recommended"}
{"input": "OAuth 2.0 authorization code flow only: hyde", "output": "hyde: User clicks login, redirected to auth server with client_id and redirect_uri. User authenticates, gets authorization code. App exchanges code for tokens at token endpoint. PKCE adds code_verifier/code_challenge to prevent interception attacks\u2014required for public clients."}
{"input": "Redis caching strategies only: lex", "output": "lex: redis cache strategy pattern\nlex: redis cache aside through\nlex: redis ttl expiration caching"}
{"input": "Redis caching strategies only: vec", "output": "vec: what are the common caching strategies when using redis for application performance\nvec: how do you implement cache-aside, write-through, and write-behind patterns with redis"}
{"input": "Redis caching strategies only: hyde", "output": "hyde: Cache-aside: app checks Redis first, fetches from DB on miss, writes to cache. Write-through: writes go to cache and DB together. Write-behind: writes to cache, async sync to DB. Set TTL with EXPIRE to prevent stale data. Use SETEX for atomic set-with-expiry."}
{"input": "GraphQL vs REST only: lex", "output": "lex: graphql rest api comparison\nlex: graphql query flexibility\nlex: rest vs graphql tradeoffs"}
{"input": "GraphQL vs REST only: vec", "output": "vec: what are the main differences between graphql and rest api design approaches\nvec: when should you choose graphql over rest for your api architecture"}
{"input": "GraphQL vs REST only: hyde", "output": "hyde: REST uses fixed endpoints returning predefined data shapes. GraphQL uses one endpoint where clients specify exactly what fields they need, reducing over-fetching. REST is simpler, better cached. GraphQL excels for mobile apps, complex data requirements, and avoiding multiple round trips."}
{"input": "linux file permissions chmod only: lex", "output": "lex: linux chmod file permissions\nlex: unix rwx permission bits\nlex: chmod 755 644 meaning"}
{"input": "linux file permissions chmod only: vec", "output": "vec: how do linux file permissions work and how do you change them with chmod\nvec: what do the rwx permission bits mean for owner, group, and others"}
{"input": "linux file permissions chmod only: hyde", "output": "hyde: Permissions are rwx for read, write, execute. Three groups: owner, group, others. chmod 755 means rwxr-xr-x (owner full, others read+execute). chmod 644 means rw-r--r-- (owner read+write, others read only). Use chmod +x to add execute permission."}
{"input": "async await error handling only: lex", "output": "lex: async await try catch\nlex: javascript promise error handling\nlex: async function exception handling"}
{"input": "async await error handling only: vec", "output": "vec: how to properly handle errors in javascript async await functions\nvec: what happens when an async function throws and how do you catch those errors"}
{"input": "async await error handling only: hyde", "output": "hyde: Wrap await calls in try-catch blocks: try { const data = await fetchData(); } catch (err) { console.error(err); }. Unhandled rejections in async functions become unhandled promise rejections. For multiple awaits, catch individually or use Promise.allSettled to handle partial failures."}
{"input": "Elasticsearch query DSL only: lex", "output": "lex: elasticsearch query dsl\nlex: elasticsearch bool must should\nlex: es full text search query"}
{"input": "Elasticsearch query DSL only: vec", "output": "vec: how to write search queries using elasticsearch query dsl syntax\nvec: what are the common query types in elasticsearch like match, term, and bool queries"}
{"input": "Elasticsearch query DSL only: hyde", "output": "hyde: Elasticsearch Query DSL uses JSON. Match query for full-text: {match: {title: 'search'}}. Term for exact: {term: {status: 'published'}}. Bool combines queries: {bool: {must: [...], should: [...], filter: [...], must_not: [...]}}. Filter context skips scoring for faster filtering."}
{"input": "terraform state management only: lex", "output": "lex: terraform state file backend\nlex: terraform remote state s3\nlex: tfstate locking management"}
{"input": "terraform state management only: vec", "output": "vec: how to manage terraform state files and what are the best practices for team collaboration\nvec: why should you use remote state backends in terraform and how do you configure them"}
{"input": "terraform state management only: hyde", "output": "hyde: Store state remotely in S3, GCS, or Terraform Cloud\u2014never commit tfstate to git. Configure backend in terraform { backend \"s3\" { bucket = \"my-state\", key = \"prod.tfstate\", region = \"us-east-1\", dynamodb_table = \"tf-locks\" } }. DynamoDB provides state locking to prevent concurrent modifications."}
{"input": "monorepo vs polyrepo only: lex", "output": "lex: monorepo polyrepo comparison\nlex: monorepo benefits drawbacks\nlex: single repo multiple repos"}
{"input": "monorepo vs polyrepo only: vec", "output": "vec: what are the tradeoffs between using a monorepo versus multiple repositories\nvec: when does a monorepo make sense and what tools help manage large monorepos"}
{"input": "monorepo vs polyrepo only: hyde", "output": "hyde: Monorepos keep all code in one repository\u2014easier atomic changes across packages, shared tooling, consistent versioning. Polyrepos give teams autonomy, simpler CI, clearer ownership. Use monorepos for tightly coupled code. Tools: Nx, Turborepo, Lerna, Bazel for build orchestration."}
{"input": "prometheus alerting rules only: lex", "output": "lex: prometheus alerting rules config\nlex: prometheus alertmanager rules\nlex: promql alert expressions"}
{"input": "prometheus alerting rules only: vec", "output": "vec: how to write prometheus alerting rules to notify on metric thresholds\nvec: what is the syntax for prometheus alert rules and how do they integrate with alertmanager"}
{"input": "prometheus alerting rules only: hyde", "output": "hyde: Define rules in YAML: groups: - name: example rules: - alert: HighErrorRate expr: rate(http_errors_total[5m]) > 0.1 for: 5m labels: severity: critical annotations: summary: High error rate. Prometheus evaluates rules periodically and sends firing alerts to Alertmanager for routing and deduplication."}
{"input": "CSS flexbox centering only: lex", "output": "lex: css flexbox center align\nlex: flexbox justify-content align-items\nlex: css center div flexbox"}
{"input": "CSS flexbox centering only: vec", "output": "vec: how to center elements horizontally and vertically using css flexbox\nvec: what flexbox properties do you use to center content in a container"}
{"input": "CSS flexbox centering only: hyde", "output": "hyde: On the container, set display: flex; justify-content: center; align-items: center;. justify-content handles the main axis (horizontal by default), align-items handles the cross axis. Add height: 100vh to center within the viewport. For a single item, margin: auto also works inside flex containers."}
{"input": "database connection pooling only: lex", "output": "lex: database connection pool\nlex: connection pooling performance\nlex: db pool size configuration"}
{"input": "database connection pooling only: vec", "output": "vec: what is database connection pooling and why does it improve application performance\nvec: how do you configure connection pool size for optimal database throughput"}
{"input": "database connection pooling only: hyde", "output": "hyde: Opening database connections is expensive. Connection pools maintain reusable connections. Set pool size based on: pool_size = (core_count * 2) + effective_spindle_count. Too small starves the app, too large overwhelms the database. Popular libraries: HikariCP for Java, pgbouncer for PostgreSQL."}
{"input": "kafka consumer groups only: lex", "output": "lex: kafka consumer group offset\nlex: kafka partition consumer rebalance\nlex: kafka consumer group id"}
{"input": "kafka consumer groups only: vec", "output": "vec: how do kafka consumer groups work for parallel message processing\nvec: what happens during consumer group rebalancing and how are partitions assigned"}
{"input": "kafka consumer groups only: hyde", "output": "hyde: Consumers with the same group.id share partitions\u2014each partition is consumed by only one consumer in the group. Adding consumers triggers rebalancing. If consumers > partitions, some idle. Offsets track progress per partition. Use enable.auto.commit=false for exactly-once semantics with manual commits."}
{"input": "vim search replace only: lex", "output": "lex: vim search replace substitute\nlex: vim sed command :%s\nlex: vim find replace regex"}
{"input": "vim search replace only: vec", "output": "vec: how to search and replace text in vim using the substitute command\nvec: what is the syntax for vim search and replace with regular expressions and flags"}
{"input": "vim search replace only: hyde", "output": "hyde: Use :%s/old/new/g to replace all occurrences in the file. % means all lines, g means global (all matches per line). Add c for confirmation: :%s/old/new/gc. Use \\< and \\> for word boundaries. & in replacement refers to the matched text. Use :s for current line only."}
{"input": "http status codes meaning only: lex", "output": "lex: http status codes list\nlex: http 200 400 500 codes\nlex: rest api status codes"}
{"input": "http status codes meaning only: vec", "output": "vec: what do the common http status codes mean and when should you use each\nvec: how do you choose the right http status code for api responses"}
{"input": "http status codes meaning only: hyde", "output": "hyde: 200 OK success, 201 Created for POST, 204 No Content for DELETE. 400 Bad Request for invalid input, 401 Unauthorized for auth required, 403 Forbidden for insufficient permissions, 404 Not Found. 500 Internal Server Error for unexpected failures, 503 Service Unavailable for temporary issues."}
{"input": "binary search algorithm only: lex", "output": "lex: binary search algorithm\nlex: binary search sorted array\nlex: binary search time complexity"}
{"input": "binary search algorithm only: vec", "output": "vec: how does the binary search algorithm work and what is its time complexity\nvec: how do you implement binary search to find an element in a sorted array"}
{"input": "binary search algorithm only: hyde", "output": "hyde: Binary search halves the search space each iteration. Compare target with middle element: if smaller, search left half; if larger, search right. O(log n) time complexity. Requires sorted input. Watch for integer overflow in mid calculation: use low + (high - low) / 2 instead of (low + high) / 2."}
{"input": "git rebase interactive only: lex", "output": "lex: git rebase interactive squash\nlex: git rebase -i edit commits\nlex: git squash commits rebase"}
{"input": "git rebase interactive only: vec", "output": "vec: how to use git interactive rebase to edit, squash, and reorder commits\nvec: what are the commands available in git rebase interactive mode"}
{"input": "git rebase interactive only: hyde", "output": "hyde: Run git rebase -i HEAD~5 to edit the last 5 commits. In the editor, change 'pick' to: squash (s) to combine with previous, reword (r) to edit message, edit (e) to amend, drop (d) to remove. Save and follow prompts. Never rebase commits already pushed to shared branches."}
{"input": "environment variables docker only: lex", "output": "lex: docker environment variables\nlex: docker env file compose\nlex: docker run -e env vars"}
{"input": "environment variables docker only: vec", "output": "vec: how to pass environment variables to docker containers\nvec: what are the different ways to set environment variables in docker and docker compose"}
{"input": "environment variables docker only: hyde", "output": "hyde: Use -e flag: docker run -e DB_HOST=localhost myapp. In docker-compose.yml: environment: - DB_HOST=localhost or env_file: - .env. For secrets, prefer docker secrets or mount files. Variables in Dockerfile with ENV persist in the image; runtime -e overrides them."}
{"input": "rate limiting algorithms only: lex", "output": "lex: rate limiting algorithm api\nlex: token bucket leaky bucket\nlex: rate limit sliding window"}
{"input": "rate limiting algorithms only: vec", "output": "vec: what algorithms are used for api rate limiting and how do they differ\nvec: how do token bucket and sliding window rate limiting algorithms work"}
{"input": "rate limiting algorithms only: hyde", "output": "hyde: Token bucket: bucket fills at fixed rate, requests consume tokens, rejected when empty\u2014allows bursts. Leaky bucket: requests queue, processed at fixed rate\u2014smooths traffic. Sliding window: count requests in rolling time window. Fixed window has boundary issues; sliding window log is precise but memory-heavy."}
{"input": "blue green deployment only: lex", "output": "lex: blue green deployment strategy\nlex: zero downtime deployment\nlex: blue green kubernetes rollout"}
{"input": "blue green deployment only: vec", "output": "vec: what is blue green deployment and how does it enable zero downtime releases\nvec: how do you implement blue green deployments in kubernetes or cloud environments"}
{"input": "blue green deployment only: hyde", "output": "hyde: Blue-green runs two identical environments. Blue is live, green has the new version. Test green thoroughly, then switch the load balancer. Instant rollback by switching back to blue. In Kubernetes, use two deployments with a service selector update, or Argo Rollouts for automated blue-green."}
{"input": "memory leak debugging only: lex", "output": "lex: memory leak debug profiler\nlex: memory leak detection tools\nlex: heap dump memory analysis"}
{"input": "memory leak debugging only: vec", "output": "vec: how to find and fix memory leaks in applications\nvec: what tools and techniques help identify memory leaks in different programming languages"}
{"input": "memory leak debugging only: hyde", "output": "hyde: Use heap profilers: Chrome DevTools for JavaScript, VisualVM or MAT for Java, Valgrind for C/C++, tracemalloc for Python. Take heap snapshots before and after operations, compare retained objects. Common causes: forgotten event listeners, closures holding references, unbounded caches, circular references."}
{"input": "Stripe webhook verification only: lex", "output": "lex: stripe webhook signature verify\nlex: stripe webhook endpoint secret\nlex: stripe event verification"}
{"input": "Stripe webhook verification only: vec", "output": "vec: how to verify stripe webhook signatures to ensure events are authentic\nvec: what is the correct way to handle and validate incoming stripe webhook events"}
{"input": "Stripe webhook verification only: hyde", "output": "hyde: Stripe signs webhooks with your endpoint secret. Verify using stripe.webhooks.constructEvent(body, sig, endpointSecret). Use the raw request body, not parsed JSON. Return 200 quickly, process async. Handle event types like checkout.session.completed. Store endpoint secret securely, rotate if compromised."}
{"input": "React context vs Redux only: lex", "output": "lex: react context redux comparison\nlex: useContext vs redux state\nlex: react state management choice"}
{"input": "React context vs Redux only: vec", "output": "vec: when should you use react context versus redux for state management\nvec: what are the tradeoffs between react context api and redux for global state"}
{"input": "React context vs Redux only: hyde", "output": "hyde: Context is built-in, simple for low-frequency updates like themes and auth. Redux adds boilerplate but provides devtools, middleware, time-travel debugging, predictable updates. Context re-renders all consumers on any change; Redux allows granular subscriptions. Use Context for simple cases, Redux for complex state logic."}
{"input": "DNS records explained only: lex", "output": "lex: dns records types a cname mx\nlex: dns configuration records\nlex: domain name system records"}
{"input": "DNS records explained only: vec", "output": "vec: what are the different types of dns records and what does each one do\nvec: how do you configure dns records for a domain including a, cname, mx, and txt records"}
{"input": "DNS records explained only: hyde", "output": "hyde: A record maps domain to IPv4 address. AAAA for IPv6. CNAME aliases one domain to another (can't be on root domain). MX for mail servers with priority. TXT for verification and SPF/DKIM. NS delegates to nameservers. TTL controls caching duration. Changes propagate based on previous TTL."}
{"input": "tmux session management only: lex", "output": "lex: tmux session window pane\nlex: tmux attach detach session\nlex: tmux commands shortcuts"}
{"input": "tmux session management only: vec", "output": "vec: how to create and manage tmux sessions for persistent terminal workflows\nvec: what are the essential tmux commands for session, window, and pane management"}
{"input": "tmux session management only: hyde", "output": "hyde: Start session: tmux new -s name. Detach: Ctrl-b d. Reattach: tmux attach -t name. New window: Ctrl-b c. Split pane: Ctrl-b % (vertical), Ctrl-b \" (horizontal). Navigate panes: Ctrl-b arrow. List sessions: tmux ls. Kill session: tmux kill-session -t name. Sessions persist after disconnect."}
{"input": "utf-8 encoding explained only: lex", "output": "lex: utf-8 unicode encoding\nlex: utf8 character encoding bytes\nlex: unicode utf-8 ascii difference"}
{"input": "utf-8 encoding explained only: vec", "output": "vec: how does utf-8 encoding work and why is it the standard for text\nvec: what is the relationship between unicode and utf-8 and how are characters encoded as bytes"}
{"input": "utf-8 encoding explained only: hyde", "output": "hyde: UTF-8 encodes Unicode code points as 1-4 bytes. ASCII characters (0-127) use 1 byte, compatible with ASCII. Higher code points use more bytes with leading bits indicating length. UTF-8 is self-synchronizing and space-efficient for Latin text. Always specify encoding explicitly when reading/writing files."}
{"input": "microservices communication patterns only: lex", "output": "lex: microservices communication patterns\nlex: sync async microservice calls\nlex: event driven microservices"}
{"input": "microservices communication patterns only: vec", "output": "vec: what are the common communication patterns between microservices\nvec: when should microservices use synchronous rest calls versus asynchronous messaging"}
{"input": "microservices communication patterns only: hyde", "output": "hyde: Sync (REST/gRPC): simple, immediate response, but creates coupling and cascade failures. Async (message queues, events): decoupled, resilient, eventual consistency. Use sync for queries needing immediate response. Use async for commands, notifications, cross-service workflows. Event sourcing and CQRS for complex domains."}
{"input": "shell script best practices only: lex", "output": "lex: bash script best practices\nlex: shell script error handling\nlex: bash scripting guidelines"}
{"input": "shell script best practices only: vec", "output": "vec: what are the best practices for writing reliable and maintainable shell scripts\nvec: how do you handle errors and edge cases properly in bash scripts"}
{"input": "shell script best practices only: hyde", "output": "hyde: Start with #!/usr/bin/env bash and set -euo pipefail. Use shellcheck for linting. Quote variables: \"$var\". Use [[ ]] for tests. Handle errors with trap. Use functions for reusability. Avoid parsing ls output\u2014use globs. Prefer printf over echo. Use local variables in functions. Add -- before filenames from user input."}
{"input": "load balancer health checks only: lex", "output": "lex: load balancer health check\nlex: health check endpoint liveness\nlex: lb health probe configuration"}
{"input": "load balancer health checks only: vec", "output": "vec: how do load balancer health checks work and why are they important\nvec: what should a health check endpoint return and how do you configure health check intervals"}
{"input": "load balancer health checks only: hyde", "output": "hyde: Load balancers probe backend instances to route traffic only to healthy ones. Health endpoint should check critical dependencies (database, cache) and return 200 if healthy, 503 if not. Configure interval (10-30s), timeout (5s), and threshold (2-3 failures). Include /health and /ready endpoints for Kubernetes liveness and readiness."}
{"input": "certificate ssl tls renewal only: lex", "output": "lex: ssl tls certificate renewal\nlex: lets encrypt certbot renew\nlex: https certificate expiration"}
{"input": "certificate ssl tls renewal only: vec", "output": "vec: how to renew ssl tls certificates before they expire\nvec: what is the process for automated certificate renewal with lets encrypt and certbot"}
{"input": "certificate ssl tls renewal only: hyde", "output": "hyde: Let's Encrypt certificates expire in 90 days. Certbot auto-renews via cron or systemd timer: certbot renew runs twice daily, renews within 30 days of expiry. Test with --dry-run. For other CAs, set calendar reminders. Check expiration: openssl s_client -connect domain:443 | openssl x509 -noout -dates."}
{"input": "python decorators explained only: lex", "output": "lex: python decorator function\nlex: python @ decorator syntax\nlex: python wrapper decorator"}
{"input": "python decorators explained only: vec", "output": "vec: how do python decorators work and what is the syntax for creating them\nvec: what are common use cases for decorators in python like logging, caching, and authentication"}
{"input": "python decorators explained only: hyde", "output": "hyde: Decorators wrap functions to extend behavior. @decorator before def is syntactic sugar for func = decorator(func). A decorator is a function taking a function and returning a new function. Use functools.wraps to preserve metadata. Common uses: @lru_cache for memoization, @login_required for auth, timing/logging wrappers."}
{"input": "cap theorem database only: lex", "output": "lex: cap theorem distributed database\nlex: consistency availability partition tolerance\nlex: cap theorem tradeoffs"}
{"input": "cap theorem database only: vec", "output": "vec: what is the cap theorem and how does it apply to distributed database design\nvec: how do different databases choose between consistency and availability during network partitions"}
{"input": "cap theorem database only: hyde", "output": "hyde: CAP theorem: distributed systems can guarantee only 2 of 3\u2014Consistency (all nodes see same data), Availability (requests get responses), Partition tolerance (survives network splits). During partitions, choose CP (reject requests for consistency, like MongoDB) or AP (serve potentially stale data, like Cassandra). PACELC extends CAP for normal operation tradeoffs."}
{"input": "garbage collection tuning only: lex", "output": "lex: garbage collection gc tuning\nlex: jvm gc heap memory\nlex: gc pause time optimization"}
{"input": "garbage collection tuning only: vec", "output": "vec: how to tune garbage collection for better application performance\nvec: what gc algorithms are available and how do you choose gc settings for low latency"}
{"input": "garbage collection tuning only: hyde", "output": "hyde: For JVM, G1GC is default, good balance of throughput and pause times. ZGC and Shenandoah offer sub-millisecond pauses for low-latency needs. Tune heap size: -Xms and -Xmx same to avoid resizing. Monitor with gc logs: -Xlog:gc*. Reduce allocation rate by reusing objects and avoiding unnecessary autoboxing."}
{"input": "feature flags implementation only: lex", "output": "lex: feature flags toggles\nlex: feature flag implementation\nlex: gradual rollout feature flags"}
{"input": "feature flags implementation only: vec", "output": "vec: how to implement feature flags for gradual rollouts and a/b testing\nvec: what are the best practices for managing feature flags in production"}
{"input": "feature flags implementation only: hyde", "output": "hyde: Feature flags decouple deployment from release. Simple: if (featureEnabled('new-checkout')) { ... }. Store flags in config, database, or services like LaunchDarkly. Use for gradual rollout (1% -> 10% -> 100%), A/B tests, kill switches. Clean up old flags to prevent technical debt. Log flag evaluations for debugging."}
{"input": "apache kafka partitions only: lex", "output": "lex: kafka partitions topics\nlex: kafka partition key ordering\nlex: kafka partition count scaling"}
{"input": "apache kafka partitions only: vec", "output": "vec: how do kafka partitions work and how do they affect scalability and message ordering\nvec: how do you choose the right number of partitions for a kafka topic"}
{"input": "apache kafka partitions only: hyde", "output": "hyde: Partitions enable parallelism\u2014each partition is consumed by one consumer in a group. Messages with same key go to same partition, preserving order per key. More partitions = more throughput but more overhead. Start with partitions = max(expected throughput / partition throughput, consumer count). Can't reduce partitions, only increase."}
{"input": "cron job syntax only: lex", "output": "lex: cron job syntax schedule\nlex: crontab expression format\nlex: cron schedule examples"}
{"input": "cron job syntax only: vec", "output": "vec: how to write cron expressions to schedule jobs at specific times\nvec: what does each field in a crontab entry mean and what are common scheduling patterns"}
{"input": "cron job syntax only: hyde", "output": "hyde: Cron format: minute hour day-of-month month day-of-week command. */5 * * * * runs every 5 minutes. 0 2 * * * runs daily at 2 AM. 0 0 * * 0 runs weekly on Sunday. Use crontab -e to edit. Tools like crontab.guru help build expressions. Consider timezone\u2014cron uses system time."}
{"input": "GPG key signing only: lex", "output": "lex: gpg key sign verify\nlex: gpg signature git commits\nlex: pgp key signing encryption"}
{"input": "GPG key signing only: vec", "output": "vec: how to use gpg keys for signing and verifying files and git commits\nvec: what is the process for creating gpg keys and configuring git to sign commits"}
{"input": "GPG key signing only: hyde", "output": "hyde: Generate key: gpg --full-generate-key. List keys: gpg --list-keys. Sign file: gpg --sign file.txt. Verify: gpg --verify file.txt.gpg. For git: git config --global user.signingkey KEYID, git config --global commit.gpgsign true. Export public key for GitHub: gpg --armor --export KEYID."}
{"input": "api versioning strategies only: lex", "output": "lex: api versioning strategy\nlex: rest api version url header\nlex: api backward compatibility"}
{"input": "api versioning strategies only: vec", "output": "vec: what are the different strategies for versioning rest apis\nvec: how do you maintain backward compatibility when evolving an api"}
{"input": "api versioning strategies only: hyde", "output": "hyde: URL versioning (/v1/users) is explicit, easy to route. Header versioning (Accept: application/vnd.api+json;version=1) keeps URLs clean. Query param (?version=1) is simple but pollutes URLs. Prefer additive changes\u2014new fields don't break clients. Deprecate gracefully with sunset headers and migration guides."}
{"input": "mutex vs semaphore only: lex", "output": "lex: mutex semaphore difference\nlex: mutex lock synchronization\nlex: semaphore counting binary"}
{"input": "mutex vs semaphore only: vec", "output": "vec: what is the difference between a mutex and a semaphore in concurrent programming\nvec: when should you use a mutex versus a semaphore for thread synchronization"}
{"input": "mutex vs semaphore only: hyde", "output": "hyde: Mutex is a binary lock owned by one thread\u2014used for mutual exclusion protecting shared resources. Semaphore is a counter allowing N concurrent accesses\u2014used for limiting concurrency (connection pools, rate limiting). Mutex has ownership (same thread must unlock), semaphore doesn't. Use mutex for critical sections, semaphore for resource counting."}
{"input": "json schema validation only: lex", "output": "lex: json schema validation\nlex: jsonschema validator python\nlex: json schema types required"}
{"input": "json schema validation only: vec", "output": "vec: how to use json schema to validate the structure of json data\nvec: what are the common json schema keywords for defining types, required fields, and constraints"}
{"input": "json schema validation only: hyde", "output": "hyde: JSON Schema defines expected structure. Key properties: type (string, number, object, array), properties for object fields, required array for mandatory fields, items for array elements. Validators: ajv (JS), jsonschema (Python). Use for API request validation, config file validation, documentation generation."}
{"input": "CI CD pipeline stages only: lex", "output": "lex: ci cd pipeline stages\nlex: continuous integration deployment\nlex: build test deploy pipeline"}
{"input": "CI CD pipeline stages only: vec", "output": "vec: what are the typical stages in a ci cd pipeline\nvec: how do you design a continuous integration and deployment pipeline for reliable releases"}
{"input": "CI CD pipeline stages only: hyde", "output": "hyde: Typical stages: 1) Source\u2014trigger on commit, 2) Build\u2014compile, bundle, create artifacts, 3) Test\u2014unit, integration, e2e tests, 4) Security scan\u2014SAST, dependency audit, 5) Deploy to staging, 6) Acceptance tests, 7) Deploy to production. Use parallelization for speed. Gate deployments on test pass. Implement rollback mechanisms."}
{"input": "event sourcing pattern only: lex", "output": "lex: event sourcing pattern\nlex: event store append only log\nlex: cqrs event sourcing"}
{"input": "event sourcing pattern only: vec", "output": "vec: what is event sourcing and how does it differ from traditional crud data storage\nvec: how do you implement event sourcing and what are its benefits and challenges"}
{"input": "event sourcing pattern only: hyde", "output": "hyde: Event sourcing stores state changes as immutable events rather than current state. Account balance is sum of all Deposit and Withdrawal events. Benefits: full audit trail, time travel, replay for debugging. Challenges: eventual consistency, event schema evolution, increased complexity. Often paired with CQRS\u2014separate read models built from event stream."}
{"input": "IPv4 vs IPv6 only: lex", "output": "lex: ipv4 ipv6 difference\nlex: ipv6 address format\nlex: ipv4 exhaustion ipv6 transition"}
{"input": "IPv4 vs IPv6 only: vec", "output": "vec: what are the key differences between ipv4 and ipv6 addressing\nvec: why is ipv6 necessary and how does the transition from ipv4 work"}
{"input": "IPv4 vs IPv6 only: hyde", "output": "hyde: IPv4 uses 32-bit addresses (4 billion), exhausted in 2011. IPv6 uses 128-bit addresses (340 undecillion), formatted as eight hex groups: 2001:0db8::1. IPv6 eliminates NAT need, has built-in IPsec. Transition via dual-stack (both protocols) or tunneling. Check IPv6 support: curl -6 ipv6.google.com."}
{"input": "dependency injection benefits only: lex", "output": "lex: dependency injection di pattern\nlex: di inversion of control ioc\nlex: dependency injection testing"}
{"input": "dependency injection benefits only: vec", "output": "vec: what is dependency injection and why does it improve code maintainability\nvec: how does dependency injection make unit testing easier"}
{"input": "dependency injection benefits only: hyde", "output": "hyde: Dependency injection provides dependencies from outside rather than creating them internally. Class receives DatabaseService via constructor instead of instantiating it. Benefits: loose coupling, easy testing with mocks, flexible configuration. Instead of new EmailService(), inject interface IEmailService\u2014swap implementations without changing consumer code."}
{"input": "S3 bucket policy only: lex", "output": "lex: s3 bucket policy permissions\nlex: aws s3 iam policy json\nlex: s3 bucket access control"}
{"input": "S3 bucket policy only: vec", "output": "vec: how to write an s3 bucket policy to control access permissions\nvec: what is the difference between s3 bucket policies and iam policies for access control"}
{"input": "S3 bucket policy only: hyde", "output": "hyde: S3 bucket policies are resource-based JSON policies attached to buckets. Grant public read: {\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":\"*\",\"Action\":\"s3:GetObject\",\"Resource\":\"arn:aws:s3:::bucket/*\"}]}. IAM policies attach to users/roles. Use bucket policies for cross-account access, IAM for user-specific permissions. Block public access settings override policies."}
{"input": "idempotency api design only: lex", "output": "lex: idempotency api design\nlex: idempotent request key\nlex: api retry safety idempotency"}
{"input": "idempotency api design only: vec", "output": "vec: what is idempotency in api design and why is it important for reliability\nvec: how do you implement idempotent endpoints to handle duplicate requests safely"}
{"input": "idempotency api design only: hyde", "output": "hyde: Idempotent operations produce the same result regardless of how many times called. GET, PUT, DELETE are naturally idempotent. POST needs idempotency keys: client sends unique key, server stores result, returns cached result on retry. Store keys with TTL (24h). Critical for payment APIs\u2014prevents double charges on network retry."}
{"input": "awk command examples only: lex", "output": "lex: awk command examples\nlex: awk print column field\nlex: awk text processing"}
{"input": "awk command examples only: vec", "output": "vec: how to use awk for text processing and extracting columns from files\nvec: what are common awk patterns and commands for parsing structured text"}
{"input": "awk command examples only: hyde", "output": "hyde: awk processes text line by line, splitting into fields. Print second column: awk '{print $2}' file. Custom delimiter: awk -F',' '{print $1}'. Pattern match: awk '/error/ {print}'. Sum column: awk '{sum+=$3} END {print sum}'. Variables: awk -v threshold=100 '$3 > threshold'. Built-in vars: NF (fields), NR (line number)."}
{"input": "database sharding strategies only: lex", "output": "lex: database sharding horizontal\nlex: shard key partition strategy\nlex: database horizontal scaling"}
{"input": "database sharding strategies only: vec", "output": "vec: what is database sharding and what strategies exist for partitioning data\nvec: how do you choose a shard key and what are the tradeoffs of different sharding approaches"}
{"input": "database sharding strategies only: hyde", "output": "hyde: Sharding distributes data across multiple databases. Strategies: range-based (user IDs 1-1M on shard 1), hash-based (consistent hashing), directory-based (lookup table). Choose shard key with high cardinality, even distribution, query locality. Avoid hot spots\u2014don't shard by timestamp. Cross-shard queries are expensive. Consider sharding only after vertical scaling exhausted."}
{"input": "jq json parsing only: lex", "output": "lex: jq json parsing command\nlex: jq filter select query\nlex: jq command line json"}
{"input": "jq json parsing only: vec", "output": "vec: how to use jq to parse and transform json data from the command line\nvec: what are the common jq filters for extracting and manipulating json fields"}
{"input": "jq json parsing only: hyde", "output": "hyde: jq is a command-line JSON processor. Extract field: jq '.name' file.json. Array element: jq '.[0]'. Nested: jq '.users[].email'. Filter: jq '.items[] | select(.price > 100)'. Transform: jq '{name: .title, count: .items | length}'. Raw output: jq -r. Pipe curl output: curl api | jq '.data'."}
{"input": "compile time vs runtime errors only: lex", "output": "lex: compile time runtime error difference\nlex: static dynamic type checking\nlex: compilation errors vs exceptions"}
{"input": "compile time vs runtime errors only: vec", "output": "vec: what is the difference between compile time and runtime errors in programming\nvec: why are compile time errors generally preferable to runtime errors for code reliability"}
{"input": "compile time vs runtime errors only: hyde", "output": "hyde: Compile time errors occur during compilation before code runs\u2014syntax errors, type mismatches in statically typed languages. Runtime errors occur during execution\u2014null pointer, division by zero, file not found. Compile time errors are caught early, cheaper to fix. Static typing and linters catch more at compile time. TypeScript catches errors that JavaScript defers to runtime."}
{"input": "content delivery network cdn only: lex", "output": "lex: cdn content delivery network\nlex: cdn caching edge servers\nlex: cloudflare cdn setup"}
{"input": "content delivery network cdn only: vec", "output": "vec: how does a content delivery network cdn improve website performance\nvec: what content should you serve through a cdn and how do you configure cache headers"}
{"input": "content delivery network cdn only: hyde", "output": "hyde: CDN caches content at edge servers geographically close to users, reducing latency. Serve static assets (images, CSS, JS) through CDN. Set Cache-Control headers: max-age=31536000 for versioned assets, shorter for dynamic content. Configure origin pulls, purge cache on deploys. Popular CDNs: Cloudflare, CloudFront, Fastly, Akamai."}
{"input": "circuit breaker pattern only: lex", "output": "lex: circuit breaker pattern\nlex: circuit breaker resilience\nlex: hystrix resilience4j circuit"}
{"input": "circuit breaker pattern only: vec", "output": "vec: what is the circuit breaker pattern and how does it improve system resilience\nvec: how do you implement circuit breakers to prevent cascade failures in distributed systems"}
{"input": "circuit breaker pattern only: hyde", "output": "hyde: Circuit breaker prevents repeated calls to failing services. States: Closed (normal), Open (failing, reject calls immediately), Half-Open (test recovery). After N failures, opens circuit. After timeout, allows test request. If succeeds, closes. Prevents cascade failures, provides fallbacks. Libraries: resilience4j (Java), polly (.NET), opossum (Node.js)."}
{"input": "mac address vs ip address only: lex", "output": "lex: mac address ip address difference\nlex: mac address layer 2 hardware\nlex: ip vs mac network address"}
{"input": "mac address vs ip address only: vec", "output": "vec: what is the difference between a mac address and an ip address in networking\nvec: how do mac addresses and ip addresses work together for network communication"}
{"input": "mac address vs ip address only: hyde", "output": "hyde: MAC address is hardware identifier burned into NIC, 48 bits (AA:BB:CC:DD:EE:FF), used in Layer 2 (local network). IP address is logical, assigned by network, used in Layer 3 (routing). ARP maps IP to MAC on local network. IP gets packets between networks, MAC delivers within a network segment. MAC is permanent, IP changes with network."}
{"input": "unit test vs integration test only: lex", "output": "lex: unit test integration test difference\nlex: testing pyramid unit integration e2e\nlex: unit test isolation mocking"}
{"input": "unit test vs integration test only: vec", "output": "vec: what is the difference between unit tests and integration tests\nvec: how should you balance unit tests and integration tests in the testing pyramid"}
{"input": "unit test vs integration test only: hyde", "output": "hyde: Unit tests verify single functions or classes in isolation using mocks for dependencies. Fast, many of them. Integration tests verify components working together with real dependencies. Slower, fewer of them. Testing pyramid: many unit tests at base, fewer integration tests in middle, few e2e tests at top. Unit tests catch logic bugs, integration tests catch interface mismatches."}
{"input": "base64 encoding decoding only: lex", "output": "lex: base64 encoding decoding\nlex: base64 encode decode string\nlex: base64 binary to text"}
{"input": "base64 encoding decoding only: vec", "output": "vec: what is base64 encoding and when should you use it\nvec: how do you encode and decode base64 strings in different programming languages"}
{"input": "base64 encoding decoding only: hyde", "output": "hyde: Base64 encodes binary data as ASCII text using 64 characters (A-Z, a-z, 0-9, +, /). Increases size by ~33%. Use for embedding binary in JSON/XML, data URLs, email attachments. Not encryption\u2014easily decoded. In shell: echo -n 'text' | base64. Decode: echo 'dGV4dA==' | base64 -d. In JS: btoa('text'), atob('dGV4dA==')."}
{"input": "tail recursion optimization only: lex", "output": "lex: tail recursion optimization\nlex: tail call optimization tco\nlex: recursive function stack overflow"}
{"input": "tail recursion optimization only: vec", "output": "vec: what is tail recursion and how does tail call optimization prevent stack overflow\nvec: how do you convert a recursive function to tail recursive form"}
{"input": "tail recursion optimization only: hyde", "output": "hyde: Tail recursion: recursive call is the last operation, no work after it returns. TCO reuses stack frame instead of adding new one\u2014prevents stack overflow. Convert by passing accumulated result as parameter: factorial(n, acc=1) { return n <= 1 ? acc : factorial(n-1, n*acc); }. Not all languages implement TCO\u2014JavaScript in strict mode, Scheme yes, Python no."}
{"input": "nginx location block only: lex", "output": "lex: nginx location block config\nlex: nginx location regex prefix\nlex: nginx location matching order"}
{"input": "nginx location block only: vec", "output": "vec: how do nginx location blocks work and in what order are they matched\nvec: what is the syntax for nginx location directives including prefix and regex matching"}
{"input": "nginx location block only: hyde", "output": "hyde: Location matching order: 1) Exact match (= /path), 2) Preferential prefix (^~ /path), 3) Regex in config order (~* case-insensitive, ~ case-sensitive), 4) Longest prefix match. Example: location /api { proxy_pass http://backend; }. Regex: location ~ \\.php$ { fastcgi_pass; }. Use = for exact matches to skip regex evaluation."}
{"input": "oop encapsulation abstraction only: lex", "output": "lex: oop encapsulation abstraction\nlex: object oriented principles\nlex: encapsulation data hiding"}
{"input": "oop encapsulation abstraction only: vec", "output": "vec: what are encapsulation and abstraction in object oriented programming\nvec: how do encapsulation and abstraction differ and why are they important for software design"}
{"input": "oop encapsulation abstraction only: hyde", "output": "hyde: Encapsulation bundles data and methods, restricting direct access via private fields and public getters/setters. Protects internal state, enables validation. Abstraction hides implementation complexity, exposing only essential interface. Car has accelerate() method\u2014you don't need to know engine internals. Encapsulation is how you hide, abstraction is what you hide."}
{"input": "webhook vs api polling only: lex", "output": "lex: webhook vs polling api\nlex: push vs pull api pattern\nlex: webhook callback http"}
{"input": "webhook vs api polling only: vec", "output": "vec: what are the differences between webhooks and api polling for receiving updates\nvec: when should you use webhooks instead of polling an api for changes"}
{"input": "webhook vs api polling only: hyde", "output": "hyde: Polling: client repeatedly asks server for updates. Simple but wastes bandwidth if nothing changed, may miss events between polls. Webhooks: server pushes updates to client endpoint when events occur. Real-time, efficient, but requires public endpoint and handling failures. Use webhooks when available (Stripe, GitHub), fall back to polling for systems without webhook support."}
{"input": "database transaction isolation levels only: lex", "output": "lex: database transaction isolation levels\nlex: read committed serializable\nlex: sql isolation dirty read phantom"}
{"input": "database transaction isolation levels only: vec", "output": "vec: what are the different database transaction isolation levels and their tradeoffs\nvec: how do isolation levels prevent anomalies like dirty reads and phantom reads"}
{"input": "database transaction isolation levels only: hyde", "output": "hyde: Isolation levels from weakest to strongest: Read Uncommitted (dirty reads possible), Read Committed (sees only committed data, default in PostgreSQL), Repeatable Read (no non-repeatable reads), Serializable (no phantom reads, full isolation). Higher isolation = more locking = lower concurrency. Choose based on consistency needs vs performance."}
{"input": "hash table collision resolution only: lex", "output": "lex: hash table collision resolution\nlex: hash map chaining open addressing\nlex: hash collision handling"}
{"input": "hash table collision resolution only: vec", "output": "vec: how do hash tables handle collisions when multiple keys hash to the same bucket\nvec: what are the differences between chaining and open addressing for collision resolution"}
{"input": "hash table collision resolution only: hyde", "output": "hyde: Chaining: each bucket holds a linked list of entries with same hash. Simple, handles high load well. Open addressing: on collision, probe for next empty slot. Linear probing (check next slot), quadratic probing, double hashing. Better cache locality but degrades at high load factors. Most implementations use chaining (Java HashMap) or open addressing with good probing (Python dict)."}
{"input": "yaml vs json config only: lex", "output": "lex: yaml json config comparison\nlex: yaml vs json syntax\nlex: configuration file format"}
{"input": "yaml vs json config only: vec", "output": "vec: what are the differences between yaml and json for configuration files\nvec: when should you choose yaml over json for application configuration"}
{"input": "yaml vs json config only: hyde", "output": "hyde: JSON: strict syntax, no comments, explicit quotes, universal parsing. YAML: superset of JSON, allows comments, cleaner for humans, indentation-based. Use JSON for data interchange, APIs, when strict parsing needed. Use YAML for configs (Docker Compose, Kubernetes, CI/CD) where human editing is common. YAML gotchas: Norway problem (NO parsed as false), inconsistent indentation."}
{"input": "Kubernetes ingress controller only: lex", "output": "lex: kubernetes ingress controller\nlex: k8s ingress nginx traefik\nlex: ingress rules path host"}
{"input": "Kubernetes ingress controller only: vec", "output": "vec: what is a kubernetes ingress controller and how does it route external traffic to services\nvec: how do you configure ingress rules for path-based and host-based routing in kubernetes"}
{"input": "Kubernetes ingress controller only: hyde", "output": "hyde: Ingress controller implements Ingress resources, routing external HTTP/HTTPS to services. Popular controllers: nginx-ingress, Traefik, HAProxy. Ingress resource defines rules: host (foo.com), paths (/api -> api-service, / -> frontend). Annotations configure TLS, rate limiting, auth. Install controller first, then create Ingress resources."}
{"input": "docker layer caching only: lex", "output": "lex: docker layer caching build\nlex: dockerfile cache optimization\nlex: docker build cache layers"}
{"input": "docker layer caching only: vec", "output": "vec: how does docker layer caching work and how do you optimize dockerfiles for faster builds\nvec: what dockerfile practices maximize cache hits when building docker images"}
{"input": "docker layer caching only: hyde", "output": "hyde: Docker caches each instruction as a layer. Cache invalidates when instruction or context changes, invalidating all subsequent layers. Optimization: order from least to most frequently changing. Copy package.json and install deps before copying source code. Use .dockerignore. Multi-stage builds discard intermediate layers. COPY --from for selective extraction."}
{"input": "ssh tunnel port forwarding only: lex", "output": "lex: ssh tunnel port forwarding\nlex: ssh local remote forward\nlex: ssh -L -R tunnel"}
{"input": "ssh tunnel port forwarding only: vec", "output": "vec: how to set up ssh tunnels for local and remote port forwarding\nvec: what is the difference between ssh local port forwarding and remote port forwarding"}
{"input": "ssh tunnel port forwarding only: hyde", "output": "hyde: Local forwarding (-L): access remote service through local port. ssh -L 8080:localhost:3000 server\u2014localhost:8080 reaches server's port 3000. Remote forwarding (-R): expose local service through remote port. ssh -R 8080:localhost:3000 server\u2014server:8080 reaches your port 3000. Use for accessing databases behind firewalls, exposing dev servers temporarily."}
{"input": "rest api pagination only: lex", "output": "lex: rest api pagination\nlex: api pagination offset cursor\nlex: paginated response next page"}
{"input": "rest api pagination only: vec", "output": "vec: what are the different approaches to implementing pagination in rest apis\nvec: how do offset-based and cursor-based pagination compare for api design"}
{"input": "rest api pagination only: hyde", "output": "hyde: Offset pagination: ?page=2&limit=20 or ?offset=20&limit=20. Simple but slow for deep pages, inconsistent with real-time inserts. Cursor pagination: ?cursor=abc123&limit=20, cursor encodes position. Consistent, efficient, better for infinite scroll. Return next_cursor in response. Use Link headers or response body for pagination URLs."}
{"input": "solid principles explained only: lex", "output": "lex: solid principles oop\nlex: single responsibility open closed\nlex: solid design principles"}
{"input": "solid principles explained only: vec", "output": "vec: what are the solid principles in object oriented design\nvec: how do the solid principles improve code maintainability and flexibility"}
{"input": "solid principles explained only: hyde", "output": "hyde: SOLID: Single Responsibility (one reason to change), Open/Closed (open for extension, closed for modification), Liskov Substitution (subtypes substitutable for base types), Interface Segregation (many specific interfaces over one general), Dependency Inversion (depend on abstractions not concretions). Following SOLID produces loosely coupled, testable, maintainable code."}
{"input": "protobuf vs json only: lex", "output": "lex: protobuf json comparison\nlex: protocol buffers serialization\nlex: grpc protobuf format"}
{"input": "protobuf vs json only: vec", "output": "vec: what are the differences between protocol buffers and json for data serialization\nvec: when should you use protobuf instead of json for api communication"}
{"input": "protobuf vs json only: hyde", "output": "hyde: JSON: human-readable, self-describing, universal support, larger payload. Protobuf: binary format, 3-10x smaller, faster serialization, requires schema (.proto files), strong typing. Use JSON for public APIs, debugging, human interaction. Use Protobuf for internal microservices, high-throughput systems, gRPC. Schema evolution with field numbers enables backward compatibility."}
{"input": "linux namespaces containers only: lex", "output": "lex: linux namespaces containers\nlex: container isolation namespace cgroup\nlex: docker linux namespaces"}
{"input": "linux namespaces containers only: vec", "output": "vec: how do linux namespaces enable container isolation\nvec: what kernel features do docker and containers use for process isolation"}
{"input": "linux namespaces containers only: hyde", "output": "hyde: Containers use Linux namespaces for isolation: PID (process tree), NET (network stack), MNT (filesystem mounts), UTS (hostname), IPC (inter-process communication), USER (user IDs). Cgroups limit resource usage (CPU, memory). Together they isolate processes without full VM overhead. Containers share host kernel but see isolated views of system resources."}
{"input": "GraphQL subscriptions websocket only: lex", "output": "lex: graphql subscriptions websocket\nlex: graphql realtime subscriptions\nlex: graphql subscription server"}
{"input": "GraphQL subscriptions websocket only: vec", "output": "vec: how do graphql subscriptions work for real-time data updates\nvec: what is the underlying protocol for graphql subscriptions and how do you implement them"}
{"input": "GraphQL subscriptions websocket only: hyde", "output": "hyde: GraphQL subscriptions enable real-time updates via persistent connections. Client subscribes: subscription { messageAdded { text } }. Server pushes when events occur. Typically uses WebSocket with graphql-ws protocol. Server maintains subscription registry, publishes events through PubSub. Apollo Server and Relay support subscriptions natively."}
{"input": "stateless vs stateful services only: lex", "output": "lex: stateless stateful service\nlex: stateless api design\nlex: session state storage"}
{"input": "stateless vs stateful services only: vec", "output": "vec: what is the difference between stateless and stateful services in application architecture\nvec: why are stateless services easier to scale and how do you handle state when needed"}
{"input": "stateless vs stateful services only: hyde", "output": "hyde: Stateless services don't store client state between requests\u2014any instance can handle any request. Scale by adding instances, no session affinity needed. Stateful services maintain client state, requiring sticky sessions or shared storage. Make services stateless by storing session in JWT tokens, Redis, or databases. Stateless is preferred for horizontal scaling and resilience."}
{"input": "git bisect debugging only: lex", "output": "lex: git bisect bug finding\nlex: git bisect good bad\nlex: binary search git commit"}
{"input": "git bisect debugging only: vec", "output": "vec: how to use git bisect to find the commit that introduced a bug\nvec: what is the git bisect workflow for binary search debugging through commit history"}
{"input": "git bisect debugging only: hyde", "output": "hyde: git bisect does binary search through commits to find where bug was introduced. Start: git bisect start, git bisect bad (current has bug), git bisect good v1.0 (known good commit). Git checks out middle commit\u2014test and mark git bisect good or git bisect bad. Repeat until found. Automate with git bisect run ./test.sh. End with git bisect reset."}
{"input": "dns propagation time only: lex", "output": "lex: dns propagation time\nlex: dns ttl propagation delay\nlex: dns changes not working"}
{"input": "dns propagation time only: vec", "output": "vec: why do dns changes take time to propagate and how can you speed it up\nvec: what is dns propagation and how does ttl affect how quickly changes are visible"}
{"input": "dns propagation time only: hyde", "output": "hyde: DNS propagation is time for changes to spread through cached resolvers worldwide. TTL (Time To Live) controls cache duration. High TTL (86400s) means up to 24h wait. Before changes, lower TTL to 300s, wait for old TTL, make change, then restore TTL. Use dig @8.8.8.8 domain.com to check Google's view. Full propagation can take 24-48h for high-TTL records."}
{"input": "fall of the Roman Empire only: lex", "output": "lex: roman empire fall causes\nlex: decline of rome 476 AD\nlex: western roman empire collapse"}
{"input": "fall of the Roman Empire only: vec", "output": "vec: what were the main causes of the fall of the western roman empire\nvec: how did economic, military, and political factors contribute to rome's collapse"}
{"input": "fall of the Roman Empire only: hyde", "output": "hyde: The Western Roman Empire fell in 476 AD when Odoacer deposed Romulus Augustulus. Contributing factors included economic troubles, military overextension, political instability with rapid emperor turnover, pressure from Germanic tribes, and the division of the empire. The Eastern Roman Empire (Byzantine) survived until 1453."}
{"input": "causes of World War I only: lex", "output": "lex: world war 1 causes\nlex: ww1 assassination archduke franz ferdinand\nlex: causes great war 1914"}
{"input": "causes of World War I only: vec", "output": "vec: what were the main causes and triggers of world war one\nvec: how did the assassination of archduke franz ferdinand lead to a global war"}
{"input": "causes of World War I only: hyde", "output": "hyde: WWI was caused by MAIN: Militarism, Alliances, Imperialism, Nationalism. The assassination of Archduke Franz Ferdinand on June 28, 1914 in Sarajevo triggered a chain reaction through alliance systems. Austria-Hungary declared war on Serbia, pulling in Russia, Germany, France, and Britain within weeks."}
{"input": "ancient Egypt pyramids construction only: lex", "output": "lex: egyptian pyramids how built\nlex: pyramid construction ancient egypt\nlex: great pyramid giza building"}
{"input": "ancient Egypt pyramids construction only: vec", "output": "vec: how were the ancient egyptian pyramids constructed without modern technology\nvec: what techniques and labor did ancient egyptians use to build the pyramids at giza"}
{"input": "ancient Egypt pyramids construction only: hyde", "output": "hyde: The pyramids were built using ramps, levers, and organized labor forces of tens of thousands of workers. Limestone blocks weighing 2.5 tons average were quarried nearby and transported on sledges. Workers were not slaves but paid laborers housed in nearby villages. The Great Pyramid took approximately 20 years to complete around 2560 BC."}
{"input": "French Revolution timeline only: lex", "output": "lex: french revolution timeline events\nlex: french revolution 1789 bastille\nlex: reign of terror robespierre"}
{"input": "French Revolution timeline only: vec", "output": "vec: what were the major events of the french revolution in chronological order\nvec: how did the french revolution progress from the storming of the bastille to napoleon"}
{"input": "French Revolution timeline only: hyde", "output": "hyde: 1789: Estates-General convenes, Bastille stormed July 14. 1791: Constitutional monarchy established. 1792: Republic declared, king executed. 1793-94: Reign of Terror under Robespierre, 17,000 guillotined. 1794: Thermidorian Reaction ends Terror. 1799: Napoleon's coup establishes Consulate."}
{"input": "Ottoman Empire history only: lex", "output": "lex: ottoman empire history\nlex: ottoman sultanate 1299 1922\nlex: turkish ottoman empire rise fall"}
{"input": "Ottoman Empire history only: vec", "output": "vec: what was the history of the ottoman empire from its founding to its dissolution\nvec: how did the ottoman empire rise to become a major world power and eventually decline"}
{"input": "Ottoman Empire history only: hyde", "output": "hyde: Founded by Osman I around 1299, the Ottoman Empire conquered Constantinople in 1453, ending the Byzantine Empire. At its peak under Suleiman the Magnificent (1520-1566), it controlled Southeast Europe, Western Asia, and North Africa. Gradual decline through the 18th-19th centuries culminated in dissolution after WWI in 1922."}
{"input": "American Civil War battles only: lex", "output": "lex: american civil war battles\nlex: civil war gettysburg antietam\nlex: union confederate battles 1861"}
{"input": "American Civil War battles only: vec", "output": "vec: what were the major battles of the american civil war\nvec: which battles were turning points in the civil war between union and confederate forces"}
{"input": "American Civil War battles only: hyde", "output": "hyde: Major battles: Fort Sumter (1861, war begins), Bull Run (Confederate victory), Antietam (1862, bloodiest single day, led to Emancipation Proclamation), Gettysburg (1863, Union turning point), Vicksburg (Union controls Mississippi), Sherman's March (1864), Appomattox (1865, Lee surrenders). Total casualties exceeded 600,000."}
{"input": "Ming Dynasty China only: lex", "output": "lex: ming dynasty china history\nlex: ming dynasty 1368 1644\nlex: chinese ming emperors"}
{"input": "Ming Dynasty China only: vec", "output": "vec: what were the major achievements and characteristics of the ming dynasty in china\nvec: how did the ming dynasty rise to power and what led to its eventual fall"}
{"input": "Ming Dynasty China only: hyde", "output": "hyde: The Ming Dynasty (1368-1644) was founded by Zhu Yuanzhang after overthrowing Mongol Yuan rule. Notable achievements: construction of the Forbidden City, voyages of Zheng He, restoration of the Great Wall, and flourishing arts and porcelain. Fell to the Manchu Qing after peasant rebellions weakened central authority."}
{"input": "Viking Age exploration only: lex", "output": "lex: viking age exploration\nlex: vikings norse exploration america\nlex: viking raids settlements"}
{"input": "Viking Age exploration only: vec", "output": "vec: where did the vikings explore and settle during the viking age\nvec: what routes did norse explorers take and what lands did they discover"}
{"input": "Viking Age exploration only: hyde", "output": "hyde: The Viking Age (793-1066 AD) saw Norse expansion across Europe and beyond. Vikings raided British Isles and France, settled Iceland (874), Greenland (985), and reached North America (Vinland, c.1000) under Leif Erikson. They also traveled east through Russia to Constantinople and served as Varangian Guard."}
{"input": "Industrial Revolution inventions only: lex", "output": "lex: industrial revolution inventions\nlex: industrial revolution steam engine\nlex: 18th century industrial innovations"}
{"input": "Industrial Revolution inventions only: vec", "output": "vec: what were the key inventions that drove the industrial revolution\nvec: how did the steam engine and textile machinery transform manufacturing in the 18th century"}
{"input": "Industrial Revolution inventions only: hyde", "output": "hyde: Key inventions: Spinning Jenny (1764), Water Frame (1769), Steam Engine improved by James Watt (1769), Power Loom (1785), Cotton Gin (1793), Steam Locomotive (1804). These enabled factory production, mass manufacturing, and transformed society from agricultural to industrial. Britain led the revolution starting around 1760."}
{"input": "Byzantine Empire Constantinople only: lex", "output": "lex: byzantine empire constantinople\nlex: eastern roman empire byzantium\nlex: fall of constantinople 1453"}
{"input": "Byzantine Empire Constantinople only: vec", "output": "vec: what was the byzantine empire and how long did it last after rome fell\nvec: how did constantinople serve as the capital of the byzantine empire until 1453"}
{"input": "Byzantine Empire Constantinople only: hyde", "output": "hyde: The Byzantine Empire was the continuation of the Eastern Roman Empire, lasting from 330 AD (Constantinople founded) to 1453. At its peak under Justinian I, it reconquered much of the western Mediterranean. Constantinople was the largest and wealthiest European city for centuries until falling to Ottoman Turks under Mehmed II on May 29, 1453."}
{"input": "Aztec Empire civilization only: lex", "output": "lex: aztec empire civilization\nlex: aztec tenochtitlan mexico\nlex: aztec history mesoamerica"}
{"input": "Aztec Empire civilization only: vec", "output": "vec: what was the aztec empire and how did their civilization develop in mesoamerica\nvec: how did the aztecs build tenochtitlan and what led to the fall of their empire"}
{"input": "Aztec Empire civilization only: hyde", "output": "hyde: The Aztec Empire (1428-1521) dominated central Mexico from their capital Tenochtitlan, built on an island in Lake Texcoco (modern Mexico City). Population reached 200,000+. Known for pyramids, human sacrifice, chinampas (floating gardens), and tribute system. Conquered by Hern\u00e1n Cort\u00e9s in 1521 with help from rival indigenous groups and smallpox."}
{"input": "Renaissance Italy Florence only: lex", "output": "lex: renaissance italy florence\nlex: italian renaissance medici\nlex: florence renaissance art"}
{"input": "Renaissance Italy Florence only: vec", "output": "vec: why did the renaissance begin in italy particularly in florence\nvec: how did the medici family and florence become the center of the italian renaissance"}
{"input": "Renaissance Italy Florence only: hyde", "output": "hyde: The Renaissance began in Florence around 1400 due to wealth from banking and trade, political stability, and classical heritage. The Medici family, especially Lorenzo the Magnificent, patronized artists like Leonardo, Michelangelo, and Botticelli. Florence's guilds, humanism from rediscovered Greek texts, and competition among city-states drove cultural innovation."}
{"input": "Cold War Berlin Wall only: lex", "output": "lex: cold war berlin wall\nlex: berlin wall 1961 1989\nlex: east west germany division"}
{"input": "Cold War Berlin Wall only: vec", "output": "vec: what was the significance of the berlin wall during the cold war\nvec: why was the berlin wall built and what led to its fall in 1989"}
{"input": "Cold War Berlin Wall only: hyde", "output": "hyde: The Berlin Wall was built overnight on August 13, 1961 by East Germany to stop emigration to the West\u20143.5 million had fled since 1945. It divided Berlin for 28 years, symbolizing the Iron Curtain. Fell November 9, 1989 after Hungary opened its border and East German protests grew. Germany reunified October 3, 1990."}
{"input": "Mongol Empire Genghis Khan only: lex", "output": "lex: mongol empire genghis khan\nlex: mongol conquests 13th century\nlex: genghis khan mongol history"}
{"input": "Mongol Empire Genghis Khan only: vec", "output": "vec: how did genghis khan build the mongol empire into the largest contiguous land empire\nvec: what territories did the mongol empire conquer and how did they administer such vast lands"}
{"input": "Mongol Empire Genghis Khan only: hyde", "output": "hyde: Genghis Khan united Mongol tribes by 1206 and conquered from Korea to Poland by his death in 1227. The empire peaked under his grandsons, spanning 24 million km\u00b2\u2014largest contiguous empire ever. Success came from cavalry tactics, meritocracy, religious tolerance, and the Yam relay system. Divided into khanates after 1260."}
{"input": "ancient Greece democracy Athens only: lex", "output": "lex: ancient greece democracy athens\nlex: athenian democracy 5th century bc\nlex: greek democracy origins"}
{"input": "ancient Greece democracy Athens only: vec", "output": "vec: how did democracy develop in ancient athens and how did it function\nvec: what were the key institutions and practices of athenian democracy"}
{"input": "ancient Greece democracy Athens only: hyde", "output": "hyde: Athenian democracy emerged under Cleisthenes (508 BC) and peaked under Pericles (461-429 BC). Citizens (adult male non-slaves) voted directly in the Assembly (Ekklesia) on laws and policy. The Council of 500, chosen by lot, set the agenda. Jury courts had hundreds of jurors. About 30,000 of 300,000 residents were citizens."}
{"input": "Protestant Reformation Martin Luther only: lex", "output": "lex: protestant reformation luther\nlex: martin luther 95 theses\nlex: reformation 1517 catholic church"}
{"input": "Protestant Reformation Martin Luther only: vec", "output": "vec: what started the protestant reformation and what were its main ideas\nvec: how did martin luther's 95 theses challenge the catholic church and spread across europe"}
{"input": "Protestant Reformation Martin Luther only: hyde", "output": "hyde: Martin Luther posted his 95 Theses on October 31, 1517 in Wittenberg, criticizing indulgences and papal authority. Key ideas: salvation by faith alone, scripture as sole authority, priesthood of all believers. The printing press spread his ideas rapidly. Luther was excommunicated in 1521. The Reformation split Western Christianity and sparked religious wars across Europe."}
{"input": "Silk Road trade routes only: lex", "output": "lex: silk road trade route\nlex: silk road ancient trade china\nlex: silk road history commerce"}
{"input": "Silk Road trade routes only: vec", "output": "vec: what was the silk road and how did it connect east and west\nvec: what goods and ideas were exchanged along the ancient silk road trade routes"}
{"input": "Silk Road trade routes only: hyde", "output": "hyde: The Silk Road was a network of trade routes connecting China to the Mediterranean from around 130 BC to 1450s AD. Goods traded: silk, spices, porcelain from East; gold, glass, horses from West. Also spread Buddhism, Islam, technologies like paper and gunpowder, and unfortunately, the Black Death. Named by German geographer Ferdinand von Richthofen in 1877."}
{"input": "Napoleonic Wars Europe only: lex", "output": "lex: napoleonic wars europe\nlex: napoleon bonaparte campaigns\nlex: napoleonic era 1803 1815"}
{"input": "Napoleonic Wars Europe only: vec", "output": "vec: what were the major campaigns and outcomes of the napoleonic wars\nvec: how did napoleon's military conquests reshape europe and lead to his downfall"}
{"input": "Napoleonic Wars Europe only: hyde", "output": "hyde: The Napoleonic Wars (1803-1815) saw France under Napoleon dominate continental Europe through brilliant campaigns at Austerlitz, Jena, and Wagram. His empire stretched from Spain to Poland. The failed 1812 Russian invasion (600,000 troops, 100,000 returned) began his decline. Exiled to Elba 1814, returned for Hundred Days, finally defeated at Waterloo June 18, 1815."}
{"input": "ancient Mesopotamia civilizations only: lex", "output": "lex: ancient mesopotamia civilizations\nlex: mesopotamia sumer babylon\nlex: cradle of civilization tigris euphrates"}
{"input": "ancient Mesopotamia civilizations only: vec", "output": "vec: what civilizations arose in ancient mesopotamia and what were their achievements\nvec: why is mesopotamia called the cradle of civilization and what did sumerians invent"}
{"input": "ancient Mesopotamia civilizations only: hyde", "output": "hyde: Mesopotamia (modern Iraq) between Tigris and Euphrates rivers hosted the world's first civilizations. Sumerians (4500-1900 BC) invented writing (cuneiform), the wheel, sailboat, and plow. Akkadian Empire under Sargon was first empire. Babylon produced Hammurabi's Code. Assyrians and Persians followed. Agriculture surplus enabled cities, specialization, and complex society."}
{"input": "Meiji Restoration Japan only: lex", "output": "lex: meiji restoration japan\nlex: meiji era modernization 1868\nlex: japan meiji emperor reform"}
{"input": "Meiji Restoration Japan only: vec", "output": "vec: what was the meiji restoration and how did it transform japan\nvec: how did japan modernize so rapidly during the meiji period from 1868 to 1912"}
{"input": "Meiji Restoration Japan only: hyde", "output": "hyde: The Meiji Restoration (1868) ended 250 years of Tokugawa shogunate rule, restoring imperial power under Emperor Meiji. Japan rapidly industrialized and westernized: abolished feudalism, created national army, built railways, established constitution (1889). Slogan: 'Rich country, strong army.' Japan defeated China (1895) and Russia (1905), becoming a world power within 50 years."}
{"input": "Black Death plague Europe only: lex", "output": "lex: black death plague europe\nlex: bubonic plague 1347 medieval\nlex: black death medieval europe"}
{"input": "Black Death plague Europe only: vec", "output": "vec: what was the black death and how did it impact medieval europe\nvec: how did the bubonic plague spread across europe and what were its consequences"}
{"input": "Black Death plague Europe only: hyde", "output": "hyde: The Black Death (1347-1351) killed 75-200 million people, 30-60% of Europe's population. Caused by Yersinia pestis bacteria spread by fleas on rats, it arrived via Genoese ships from Crimea. Symptoms: buboes, fever, death within days. Consequences: labor shortages raised wages, weakened feudalism, sparked religious movements and persecution of Jews."}
{"input": "Spanish Conquest Americas only: lex", "output": "lex: spanish conquest americas\nlex: conquistadors cortez pizarro\nlex: spanish colonization new world"}
{"input": "Spanish Conquest Americas only: vec", "output": "vec: how did spanish conquistadors conquer the aztec and inca empires\nvec: what factors enabled spain to colonize the americas so rapidly in the 16th century"}
{"input": "Spanish Conquest Americas only: hyde", "output": "hyde: Hern\u00e1n Cort\u00e9s conquered the Aztec Empire (1519-1521) with 500 soldiers, allying with Tlaxcalans and exploiting Montezuma's hesitation. Francisco Pizarro conquered the Inca Empire (1532-1533) capturing Atahualpa during civil war. Spanish advantages: steel weapons, horses, gunpowder, and crucially, Old World diseases like smallpox that killed 90% of indigenous populations."}
{"input": "World War II D-Day only: lex", "output": "lex: world war 2 d-day normandy\nlex: d-day june 6 1944 invasion\nlex: operation overlord ww2"}
{"input": "World War II D-Day only: vec", "output": "vec: what happened on d-day and why was the normandy invasion a turning point in world war two\nvec: how was the d-day invasion of normandy planned and executed by allied forces"}
{"input": "World War II D-Day only: hyde", "output": "hyde: D-Day, June 6, 1944, was the largest amphibious invasion in history. Operation Overlord landed 156,000 Allied troops on five Normandy beaches (Utah, Omaha, Gold, Juno, Sword). Despite 10,000+ casualties, it established a Western Front, leading to Paris liberation (August 1944) and Germany's surrender (May 1945). Supreme Commander: Dwight D. Eisenhower."}
{"input": "Han Dynasty China achievements only: lex", "output": "lex: han dynasty china achievements\nlex: han dynasty 206 bc history\nlex: ancient china han empire"}
{"input": "Han Dynasty China achievements only: vec", "output": "vec: what were the major achievements and contributions of the han dynasty in china\nvec: why is the han dynasty considered a golden age in chinese history"}
{"input": "Han Dynasty China achievements only: hyde", "output": "hyde: The Han Dynasty (206 BC - 220 AD) is considered China's golden age. Achievements: Silk Road trade established, paper invented (105 AD), civil service exams introduced, Confucianism became state ideology. Population reached 60 million. So influential that ethnic Chinese still call themselves 'Han people.' Collapsed due to court intrigue, eunuch power, and Yellow Turban Rebellion."}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,399 @@
{"input": "what is base jumping? /only:hyde", "output": "hyde: BASE jumping involves parachuting from fixed objects: Buildings, Antennas, Spans (bridges), and Earth (cliffs). Unlike skydiving from aircraft, BASE jumps occur at much lower altitudes, giving jumpers only seconds to deploy their parachute."}
{"input": "knife skills basics /only:vec", "output": "vec: how to knife skills basics\nvec: complete knife skills basics reference"}
{"input": "React changelog 2025 /only:vec", "output": "vec: complete React changelog 2025 reference\nvec: how to React changelog 2025"}
{"input": "cold war key events /only:vec", "output": "vec: understanding cold war key events\nvec: learn about cold war key events"}
{"input": "how does culture influence ethics /only:vec", "output": "vec: how does culture shape people's ethical beliefs and moral values\nvec: what is the relationship between cultural norms and ethical decision-making"}
{"input": "what is the difference between ethics and morals /only:lex", "output": "lex: ethics vs morals difference\nlex: ethics morals philosophy distinction\nlex: moral principles ethical systems comparison"}
{"input": "GitHub recent news November /only:vec", "output": "vec: complete GitHub recent news November reference\nvec: learn about GitHub recent news November"}
{"input": "how to learn about native american culture /only:vec", "output": "vec: how can I respectfully learn about Native American culture and history\nvec: what are good resources for understanding Indigenous peoples' traditions and heritage"}
{"input": "decision making psychology /only:vec", "output": "vec: learn about decision making psychology\nvec: how to decision making psychology"}
{"input": "grilling temperature chart /only:vec", "output": "vec: guide for grilling temperature chart\nvec: understanding grilling temperature chart"}
{"input": "how do different cultures commemorate death? /only:hyde", "output": "hyde: In Mexico, D\u00eda de los Muertos celebrates deceased loved ones with altars, marigolds, and sugar skulls. Hindu cremation ceremonies release the soul for reincarnation. In Ghana, elaborate fantasy coffins reflect the deceased's life. Japanese Obon festivals welcome ancestral spirits home."}
{"input": "bronze age collapse /only:vec", "output": "vec: guide for bronze age collapse\nvec: understanding bronze age collapse"}
{"input": "time zones map /only:vec", "output": "vec: how to time zones map\nvec: complete time zones map reference"}
{"input": "battery test /only:hyde", "output": "hyde: Test a 12V car battery with a multimeter set to DC volts. A fully charged battery reads 12.6V or higher. Between 12.0-12.4V indicates partial charge. Below 12.0V means the battery is discharged. For a load test, apply a load equal to half the CCA rating for 15 seconds\u2014voltage should stay above 9.6V."}
{"input": "Han Dynasty China achievements /only:vec", "output": "vec: what were the major achievements and contributions of the han dynasty in china\nvec: why is the han dynasty considered a golden age in chinese history"}
{"input": "how to brew the perfect cup of tea /only:hyde", "output": "hyde: Water temperature and steep time vary by tea type. Black tea: 200-212\u00b0F for 3-5 minutes. Green tea: 160-180\u00b0F for 2-3 minutes. White tea: 160-185\u00b0F for 4-5 minutes. Oolong: 185-205\u00b0F for 3-5 minutes. Use 1 teaspoon of loose leaf per 8 oz cup. Pre-warm the teapot with hot water for consistent extraction."}
{"input": "Docker new features 2025 /only:lex", "output": "lex: Docker new features 2025 guide\nlex: Docker new features 2025 best practices\nlex: Docker new features 2025 tutorial"}
{"input": "weightlifting proper form /only:vec", "output": "vec: guide for weightlifting proper form\nvec: how to weightlifting proper form"}
{"input": "Byzantine Empire Constantinople /only:vec", "output": "vec: what was the byzantine empire and how long did it last after rome fell\nvec: how did constantinople serve as the capital of the byzantine empire until 1453"}
{"input": "how does philosophy explore the nature of truth? /only:hyde", "output": "hyde: Philosophy examines truth through several theories. The correspondence theory holds that truth is agreement between a proposition and reality. The coherence theory says a statement is true if it fits consistently within a system of beliefs. The pragmatic theory (James, Dewey) defines truth as what works in practice. Deflationary theories argue that \"true\" adds nothing beyond the assertion itself."}
{"input": "ancient Egypt pyramids construction /only:vec", "output": "vec: how were the ancient egyptian pyramids constructed without modern technology\nvec: what techniques and labor did ancient egyptians use to build the pyramids at giza"}
{"input": "what was the enlightenment /only:lex", "output": "lex: Enlightenment 18th century intellectual movement\nlex: Age of Enlightenment reason philosophy\nlex: Enlightenment thinkers Voltaire Locke Kant"}
{"input": "Vue changelog 2025 /only:vec", "output": "vec: guide for Vue changelog 2025\nvec: understanding Vue changelog 2025"}
{"input": "TCP vs UDP /only:hyde", "output": "hyde: TCP provides reliable, ordered delivery with acknowledgments and retransmission. UDP is faster but unreliable\u2014packets may arrive out of order or not at all. Use TCP for web, email, file transfer. Use UDP for video streaming, gaming, DNS where speed matters more than reliability."}
{"input": "how to analyze experimental data /only:hyde", "output": "hyde: Start by cleaning the data: remove outliers using predefined criteria and check for missing values. Calculate descriptive statistics (mean, median, standard deviation). Visualize distributions with histograms or box plots. Apply appropriate statistical tests to evaluate hypotheses. Interpret results in context of your research question and note limitations."}
{"input": "how to find a reliable realtor /only:vec", "output": "vec: how do you find and vet a trustworthy real estate agent for buying or selling a home\nvec: what qualities and credentials should you look for in a reliable realtor"}
{"input": "how to build a capsule wardrobe /only:hyde", "output": "hyde: A capsule wardrobe consists of 30-40 versatile pieces that mix and match. Start by choosing a neutral color palette (black, navy, white, beige). Include 2-3 pairs of pants, 5-7 tops, 2 jackets, 2 pairs of shoes, and 1-2 dresses or suits. Remove items you haven't worn in a year. Invest in quality basics over trendy pieces."}
{"input": "how to sell a car privately? /only:hyde", "output": "hyde: To sell a car privately, first determine a fair price using Kelley Blue Book or Edmunds. Gather the title, maintenance records, and smog certificate. List the car on Craigslist, Facebook Marketplace, or AutoTrader. When meeting buyers, accept cashier's checks or cash. Sign the title over and file a release of liability with your DMV."}
{"input": "sail boat /only:hyde", "output": "hyde: Sailboats are propelled by wind acting on sails. Common types include dinghies (small, single-hull), keelboats (weighted keel for stability), catamarans (twin hulls), and sloops (single mast, fore-and-aft rigged). Key parts include the hull, mast, boom, jib, mainsail, rudder, and keel."}
{"input": "grammar punctuation rules /only:lex", "output": "lex: grammar punctuation rules best practices\nlex: grammar punctuation rules documentation\nlex: grammar punctuation rules tutorial"}
{"input": "tail recursion optimization /only:vec", "output": "vec: what is tail recursion and how does tail call optimization prevent stack overflow\nvec: how do you convert a recursive function to tail recursive form"}
{"input": "binary search algorithm /only:vec", "output": "vec: how does the binary search algorithm work and what is its time complexity\nvec: how do you implement binary search to find an element in a sorted array"}
{"input": "how to shoot video in low light /only:hyde", "output": "hyde: For low light video, open your aperture to f/1.4\u2013f/2.8 and lower your shutter speed to 1/50 for 24fps footage. Raise ISO gradually \u2014 modern cameras handle ISO 3200\u20136400 with acceptable noise. Use a fast prime lens and add practical lights in the scene when possible."}
{"input": "how to boil an egg perfectly /only:hyde", "output": "hyde: Place eggs in a single layer in a pot and cover with cold water by 1 inch. Bring to a rolling boil, then remove from heat and cover. For soft-boiled: 6-7 minutes. For medium: 9-10 minutes. For hard-boiled: 12-13 minutes. Transfer immediately to an ice bath for 5 minutes. Older eggs (7-10 days) peel more easily than fresh ones."}
{"input": "what is the great wall of china? /only:hyde", "output": "hyde: The Great Wall of China is a series of fortifications built over centuries to protect Chinese states and empires from northern invasions. The most well-known sections were built during the Ming Dynasty (1368-1644). The total length, including all branches and sections across dynasties, is approximately 21,196 kilometers (13,171 miles)."}
{"input": "stock market basics beginners /only:vec", "output": "vec: guide for stock market basics beginners\nvec: learn about stock market basics beginners"}
{"input": "how do thought experiments aid philosophical reasoning /only:lex", "output": "lex: thought experiments philosophy reasoning\nlex: philosophical thought experiment trolley problem examples"}
{"input": "causes of World War I /only:lex", "output": "lex: world war 1 causes\nlex: ww1 assassination archduke franz ferdinand\nlex: causes great war 1914"}
{"input": "ocean currents patterns /only:lex", "output": "lex: ocean currents patterns tutorial\nlex: ocean currents patterns examples\nlex: ocean currents patterns documentation"}
{"input": "what are the main sects of islam? /only:lex", "output": "lex: sects of Islam Sunni Shia Sufi\nlex: Islamic denominations branches\nlex: Sunni Shia differences beliefs"}
{"input": "Kubernetes changelog 2025 /only:lex", "output": "lex: Kubernetes changelog 2025 best practices\nlex: Kubernetes changelog 2025 documentation\nlex: Kubernetes changelog 2025 examples"}
{"input": "what is the significance of song in worship? /only:lex", "output": "lex: song worship significance religious singing\nlex: worship music congregational singing hymns praise"}
{"input": "how to create a moon garden? /only:vec", "output": "vec: how to plan and plant a garden designed to be enjoyed at night\nvec: what plants and flowers work best in a moon garden"}
{"input": "how do religions interpret the concept of sacredness? /only:hyde", "output": "hyde: In Christianity, sacredness is conferred by God's presence\u2014churches, sacraments, and scripture are holy. In Hinduism, sacred rivers like the Ganges and temples house divine energy. Indigenous traditions see sacredness in natural features\u2014mountains, groves, and animals. Islam treats the Quran and Mecca as inviolably sacred."}
{"input": "what is the significance of the ten commandments /only:hyde", "output": "hyde: The Ten Commandments (Decalogue) were given by God to Moses on Mount Sinai, as recorded in Exodus 20 and Deuteronomy 5. They form the foundational moral code of Judaism and Christianity, covering duties to God (no other gods, no idols, keep the Sabbath) and duties to others (honor parents, do not murder, steal, or lie)."}
{"input": "how to remove oil stains from clothes /only:lex", "output": "lex: remove oil stains clothing\nlex: grease stain removal fabric\nlex: oil stain laundry treatment"}
{"input": "how does culture influence identity? /only:hyde", "output": "hyde: Culture shapes identity through language, traditions, values, and social norms internalized from childhood. Family, community, religion, and media all transmit cultural frameworks. Identity is constructed through negotiation between personal experiences and cultural expectations, creating a sense of belonging and self-understanding."}
{"input": "what is the philosophy of mind /only:lex", "output": "lex: philosophy of mind consciousness mental states\nlex: philosophy of mind problem qualia dualism physicalism"}
{"input": "how to volunteer for civic initiatives /only:vec", "output": "vec: how can someone find and volunteer for civic engagement and community initiatives\nvec: what are ways to get involved in local civic volunteer opportunities"}
{"input": "latest findings in climate science /only:lex", "output": "lex: climate science research findings 2025 2026\nlex: climate change latest studies temperature emissions"}
{"input": "bug fix /only:hyde", "output": "hyde: To fix a bug, first reproduce it reliably and identify the exact conditions that trigger it. Use a debugger or add logging to narrow down the faulty code path. Write a regression test that captures the bug, then modify the code until the test passes."}
{"input": "where to find heirloom seed suppliers? /only:lex", "output": "lex: heirloom seed suppliers catalog\nlex: buy heirloom seeds online non-GMO\nlex: heirloom vegetable seed company"}
{"input": "what are smart cities? /only:vec", "output": "vec: what defines a smart city and what technologies do they use\nvec: how do smart cities use IoT sensors and data analytics to improve urban infrastructure"}
{"input": "what is sacred geometry? /only:hyde", "output": "hyde: Sacred geometry assigns symbolic and spiritual meaning to geometric shapes and proportions found in nature. Key patterns include the Flower of Life (overlapping circles), Metatron's Cube, the golden ratio (1.618), and the Fibonacci spiral. These patterns appear in sunflower seeds, nautilus shells, and ancient temple architecture."}
{"input": "where to buy raised garden beds? /only:hyde", "output": "hyde: Raised garden beds are available at Home Depot, Lowe's, and garden centers. Online retailers like Gardener's Supply, Amazon, and Birdies offer metal and cedar kits. Cedar is rot-resistant and long-lasting; galvanized steel beds are durable and modern-looking."}
{"input": "how to start oil painting? /only:lex", "output": "lex: oil painting beginner supplies techniques\nlex: oil painting start canvas brushes paints medium"}
{"input": "how to do a flip on a trampoline /only:vec", "output": "vec: how do I safely learn to do a backflip on a trampoline\nvec: what is the proper technique for doing flips on a trampoline"}
{"input": "what is philosophy of mind /only:hyde", "output": "hyde: Philosophy of mind examines the nature of mental states, consciousness, and their relationship to the physical brain. Central questions include the mind-body problem: how do subjective experiences (qualia) arise from neural processes? Key positions include dualism, physicalism, functionalism, and property dualism."}
{"input": "how to lose weight fast? /only:vec", "output": "vec: what are safe and effective methods to lose weight quickly\nvec: how can I create a calorie deficit to lose weight without harming my health"}
{"input": "what caused the fall of the roman empire /only:hyde", "output": "hyde: The fall of the Western Roman Empire in 476 AD resulted from multiple factors: military overextension, barbarian invasions (Visigoths, Vandals, Ostrogoths), economic decline from debasement of currency, political instability with rapid emperor turnover, and the shift of power to Constantinople."}
{"input": "what is the capital of japan /only:hyde", "output": "hyde: Tokyo is the capital city of Japan. It became the capital in 1868 when Emperor Meiji moved the imperial seat from Kyoto. Tokyo, located on the eastern coast of Honshu, is the most populous metropolitan area in the world with over 37 million residents."}
{"input": "what to pack in a hospital bag for labor? /only:vec", "output": "vec: what items should I pack in my hospital bag before going into labor\nvec: what is a complete packing checklist for the hospital for giving birth"}
{"input": "what is an anthology? /only:vec", "output": "vec: what is an anthology and how are literary anthologies compiled and organized\nvec: what types of works are typically collected in an anthology such as short stories, poems, or essays"}
{"input": "what is the role of e-commerce in modern business /only:hyde", "output": "hyde: E-commerce enables businesses to sell products globally without physical storefronts. Companies use platforms like Shopify, Amazon Marketplace, and WooCommerce to reach customers online. In 2024, global e-commerce sales exceeded $6 trillion. Direct-to-consumer (DTC) brands cut out middlemen, while marketplaces aggregate sellers for one-stop shopping."}
{"input": "how to replace windshield wipers? /only:lex", "output": "lex: replace windshield wipers installation\nlex: change wiper blades car DIY\nlex: windshield wiper replacement size"}
{"input": "how artificial intelligence is used in healthcare /only:vec", "output": "vec: how is artificial intelligence being applied in healthcare for diagnosis and treatment\nvec: what are the main uses of AI and machine learning in the medical field"}
{"input": "best techniques for street photography /only:hyde", "output": "hyde: Shoot at f/8 for deep depth of field and zone focus at 3 meters for quick candid shots. Use a 28mm or 35mm lens. Anticipate moments\u2014find good light or backgrounds and wait for subjects to enter the frame. Shoot from the hip to stay inconspicuous."}
{"input": "what is lean manufacturing /only:hyde", "output": "hyde: Lean manufacturing, derived from the Toyota Production System, aims to minimize waste (muda) while maximizing value. Its five principles: define value from the customer's perspective, map the value stream, create flow, establish pull, and pursue perfection through continuous improvement (kaizen)."}
{"input": "what is deconstruction /only:hyde", "output": "hyde: Deconstruction, associated with Jacques Derrida, is a method of critical analysis that examines how meaning in texts is constructed through binary oppositions (speech/writing, presence/absence). Derrida argued that meaning is never fixed; it is always deferred through a chain of signifiers. Deconstruction reveals the internal contradictions and assumptions hidden within texts."}
{"input": "io file /only:vec", "output": "vec: how do you perform file input and output operations in programming languages\nvec: what are the common methods for reading from and writing to files in Python, Java, or C"}
{"input": "how to choose car speakers? /only:vec", "output": "vec: how do you choose aftermarket car speakers that fit your vehicle and sound preferences\nvec: what is the difference between coaxial and component car speakers and which should you buy"}
{"input": "how to improve sleep quality /only:lex", "output": "lex: improve sleep quality tips habits\nlex: better sleep hygiene insomnia remedies"}
{"input": "how do philosophers define happiness /only:lex", "output": "lex: philosophers define happiness philosophy\nlex: happiness eudaimonia Aristotle hedonism\nlex: philosophical theories happiness well-being"}
{"input": "how to make slime at home /only:vec", "output": "vec: what ingredients and steps do you need to make slime at home\nvec: how to make homemade slime using glue and borax or contact lens solution"}
{"input": "how to build a writing routine /only:lex", "output": "lex: writing routine daily habit\nlex: build writing practice discipline\nlex: writing schedule productivity"}
{"input": "how to choose the right camera /only:vec", "output": "vec: how do you choose the right camera for your photography needs and budget?\nvec: what factors should you consider when deciding between DSLR and mirrorless cameras?"}
{"input": "sail boat /only:lex", "output": "lex: sailboat sailing types rigging\nlex: sailboat buy beginner learn to sail\nlex: sailboat parts hull keel mast"}
{"input": "Kubernetes ingress controller /only:vec", "output": "vec: what is a kubernetes ingress controller and how does it route external traffic to services\nvec: how do you configure ingress rules for path-based and host-based routing in kubernetes"}
{"input": "what are the sacred texts of judaism /only:lex", "output": "lex: sacred texts Judaism Torah Talmud\nlex: Jewish scripture Hebrew Bible Tanakh\nlex: Judaism holy books Mishnah"}
{"input": "how climate change affects farming /only:lex", "output": "lex: climate change agriculture crop yields\nlex: global warming farming drought impact\nlex: climate change food production"}
{"input": "GraphQL vs REST /only:hyde", "output": "hyde: REST uses fixed endpoints returning predefined data shapes. GraphQL uses one endpoint where clients specify exactly what fields they need, reducing over-fetching. REST is simpler, better cached. GraphQL excels for mobile apps, complex data requirements, and avoiding multiple round trips."}
{"input": "how to obtain information on federal legislation /only:hyde", "output": "hyde: Congress.gov is the official source for federal legislation. Search by bill number, keyword, or sponsor. Each bill page shows full text, status, cosponsors, committee actions, and vote records. GovTrack.us and ProPublica's Congress API provide additional analysis and tracking tools."}
{"input": "how to replace car air filter? /only:hyde", "output": "hyde: Open the hood and locate the air filter housing\u2014usually a black plastic box near the engine. Unclip the latches, remove the old filter, and note its orientation. Insert the new filter with the rubber rim facing up, close the housing, and secure the clips. Replace every 12,000-15,000 miles."}
{"input": "what is a primary election /only:lex", "output": "lex: primary election definition process\nlex: primary election presidential nomination\nlex: open closed primary voting"}
{"input": "how to support climbing roses? /only:hyde", "output": "hyde: Install a sturdy trellis, arbor, or wire system at least 3 inches from the wall to allow air circulation. Tie canes horizontally with soft plant ties to encourage lateral growth and more blooms. Prune in late winter, removing dead wood and shortening side shoots to 2-3 buds."}
{"input": "what is a primary election /only:hyde", "output": "hyde: A primary election is a vote held by a political party to choose its candidates for the general election. In a closed primary, only registered party members can vote. In an open primary, any registered voter may participate regardless of party affiliation."}
{"input": "how does blockchain technology work /only:lex", "output": "lex: blockchain technology distributed ledger\nlex: blockchain cryptography decentralized consensus"}
{"input": "how to improve car gas mileage? /only:vec", "output": "vec: what are the best ways to improve a car's gas mileage and fuel efficiency\nvec: what driving habits and car maintenance steps help reduce fuel consumption"}
{"input": "Elasticsearch query DSL /only:lex", "output": "lex: elasticsearch query dsl\nlex: elasticsearch bool must should\nlex: es full text search query"}
{"input": "what are the themes of to kill a mockingbird? /only:hyde", "output": "hyde: The central themes of To Kill a Mockingbird include racial injustice in the American South, as shown through Tom Robinson's trial. Moral courage is embodied by Atticus Finch, who defends Robinson despite social pressure. The loss of innocence is traced through Scout's growing awareness of prejudice and cruelty in Maycomb, Alabama."}
{"input": "how to assess a neighborhood safety /only:hyde", "output": "hyde: Check crime maps on sites like CrimeMapping.com or SpotCrime using the ZIP code. Walk the neighborhood at different times of day and night. Look for signs of community investment: maintained properties, street lighting, and active businesses. Talk to residents and visit the local police precinct for crime statistics."}
{"input": "how to report scientific findings /only:lex", "output": "lex: scientific findings report writing\nlex: research results publication format\nlex: academic paper methodology results"}
{"input": "how to prepare for a triathlon /only:vec", "output": "vec: what training plan should a beginner follow to prepare for their first triathlon\nvec: how to balance swimming cycling and running workouts when training for a triathlon"}
{"input": "how to create a moon garden? /only:hyde", "output": "hyde: A moon garden features white and pale-colored flowers, silver foliage, and night-blooming plants that glow under moonlight. Include moonflower (Ipomoea alba), white nicotiana, night-blooming jasmine, dusty miller, and lamb's ear. Add light-colored gravel paths for reflection."}
{"input": "what is the significance of beauty in philosophy /only:hyde", "output": "hyde: In Plato's Symposium, beauty is a ladder ascending from physical attraction to the Form of Beauty itself. Kant distinguished between the beautiful (harmonious, universal pleasure) and the sublime (overwhelming grandeur). For Hegel, beauty in art reveals truth through sensory form. Contemporary aesthetics debates whether beauty is objective or culturally constructed."}
{"input": "what is influencer marketing /only:lex", "output": "lex: influencer marketing social media brand promotion\nlex: influencer campaigns Instagram TikTok sponsorship"}
{"input": "CSS flexbox centering /only:hyde", "output": "hyde: On the container, set display: flex; justify-content: center; align-items: center;. justify-content handles the main axis (horizontal by default), align-items handles the cross axis. Add height: 100vh to center within the viewport. For a single item, margin: auto also works inside flex containers."}
{"input": "what is the significance of the torah? /only:vec", "output": "vec: what is the Torah and why is it significant in Judaism\nvec: what role does the Torah play in Jewish religious life and law"}
{"input": "git rebase interactive /only:lex", "output": "lex: git rebase interactive squash\nlex: git rebase -i edit commits\nlex: git squash commits rebase"}
{"input": "how to hang artwork without nails /only:lex", "output": "lex: hang artwork without nails wall\nlex: picture hanging command strips adhesive hooks"}
{"input": "fix teeth /only:vec", "output": "vec: what are the options for fixing damaged, chipped, or broken teeth?\nvec: how do dentists repair teeth using crowns, veneers, bonding, and other dental treatments?"}
{"input": "how to stay motivated daily? /only:lex", "output": "lex: daily motivation habits discipline routine\nlex: stay motivated goals productivity tips"}
{"input": "how the scientific community addresses research bias /only:lex", "output": "lex: research bias scientific community peer review\nlex: scientific bias mitigation replication reproducibility"}
{"input": "what is cliffhanger? /only:vec", "output": "vec: what is a cliffhanger in storytelling and how does it create suspense\nvec: how do writers use cliffhangers to keep readers or viewers engaged"}
{"input": "what is stream of consciousness /only:hyde", "output": "hyde: Stream of consciousness is a narrative technique that presents a character's continuous flow of thoughts, feelings, and sensory impressions as they occur. Pioneered by writers like Virginia Woolf and James Joyce, it mimics the unstructured way the human mind processes experience."}
{"input": "shell script best practices /only:hyde", "output": "hyde: Start with #!/usr/bin/env bash and set -euo pipefail. Use shellcheck for linting. Quote variables: \"$var\". Use [[ ]] for tests. Handle errors with trap. Use functions for reusability. Avoid parsing ls output\u2014use globs. Prefer printf over echo. Use local variables in functions. Add -- before filenames from user input."}
{"input": "what is the function of dna /only:lex", "output": "lex: DNA function genetic information\nlex: deoxyribonucleic acid protein synthesis\nlex: DNA replication transcription translation"}
{"input": "web mail /only:hyde", "output": "hyde: Webmail allows you to access your email through a web browser without installing a desktop client. Popular services include Gmail (mail.google.com), Outlook.com, Yahoo Mail, and ProtonMail. Log in with your credentials to read, compose, and manage messages from any device."}
{"input": "terraform state management /only:hyde", "output": "hyde: Store state remotely in S3, GCS, or Terraform Cloud\u2014never commit tfstate to git. Configure backend in terraform { backend \"s3\" { bucket = \"my-state\", key = \"prod.tfstate\", region = \"us-east-1\", dynamodb_table = \"tf-locks\" } }. DynamoDB provides state locking to prevent concurrent modifications."}
{"input": "latest Vue updates /only:vec", "output": "vec: understanding latest Vue updates\nvec: learn about latest Vue updates"}
{"input": "carbon footprint reduction /only:vec", "output": "vec: guide for carbon footprint reduction\nvec: learn about carbon footprint reduction"}
{"input": "how does the body maintain homeostasis /only:lex", "output": "lex: homeostasis regulation human body\nlex: negative feedback loop physiology\nlex: body temperature pH blood glucose regulation"}
{"input": "PostgreSQL indexes explain /only:hyde", "output": "hyde: Run EXPLAIN ANALYZE SELECT... to see the query plan and actual execution time. Look for Seq Scan on large tables\u2014add an index with CREATE INDEX idx_name ON table(column). B-tree indexes work for equality and range queries, GIN for full-text search and arrays, GiST for geometric data."}
{"input": "what changed in AWS 2025 /only:lex", "output": "lex: what changed in AWS 2025 examples\nlex: what changed in AWS 2025 guide\nlex: what changed in AWS 2025 best practices"}
{"input": "how do scientists study animal behavior /only:hyde", "output": "hyde: Ethologists use direct observation, video tracking, and GPS telemetry to study animal behavior in natural habitats. Lab experiments control variables to test hypotheses about cognition and social behavior. Focal sampling follows one individual; scan sampling records group behavior at intervals."}
{"input": "tech fix /only:lex", "output": "lex: tech troubleshooting fix repair computer\nlex: technology fix common problems software hardware\nlex: tech support fix device issue"}
{"input": "http status codes meaning /only:vec", "output": "vec: what do the common http status codes mean and when should you use each\nvec: how do you choose the right http status code for api responses"}
{"input": "recent Shopify changes 2025 /only:lex", "output": "lex: recent Shopify changes 2025 examples\nlex: recent Shopify changes 2025 tutorial\nlex: recent Shopify changes 2025 best practices"}
{"input": "PostgreSQL indexes explain /only:lex", "output": "lex: postgresql index explain analyze\nlex: postgres btree index performance\nlex: create index postgresql"}
{"input": "http status codes meaning /only:hyde", "output": "hyde: 200 OK success, 201 Created for POST, 204 No Content for DELETE. 400 Bad Request for invalid input, 401 Unauthorized for auth required, 403 Forbidden for insufficient permissions, 404 Not Found. 500 Internal Server Error for unexpected failures, 503 Service Unavailable for temporary issues."}
{"input": "kindle library /only:lex", "output": "lex: kindle library ebook collection\nlex: Amazon Kindle digital library management\nlex: kindle book organization archive"}
{"input": "what is the role of family in society /only:lex", "output": "lex: family role society function socialization\nlex: family structure social institution support"}
{"input": "webhook vs api polling /only:lex", "output": "lex: webhook vs polling api\nlex: push vs pull api pattern\nlex: webhook callback http"}
{"input": "space exploration changelog 2026 /only:lex", "output": "lex: space exploration changelog 2026 documentation\nlex: space exploration changelog 2026 best practices\nlex: space exploration changelog 2026 guide"}
{"input": "resilience training programs /only:lex", "output": "lex: resilience training programs mental toughness\nlex: resilience building workplace employee training"}
{"input": "how to write a scientific research proposal /only:hyde", "output": "hyde: A scientific research proposal typically includes: title, abstract, specific aims, background and significance, preliminary data, research design and methods, timeline, budget and justification, and references. The specific aims page is the most critical \u2014 state the problem, your hypothesis, and 2-3 measurable objectives clearly in one page."}
{"input": "how to choose farm equipment /only:hyde", "output": "hyde: Match tractor horsepower to your acreage: 25-45 HP for under 50 acres, 45-85 HP for 50-200 acres, and 100+ HP for large operations. Consider PTO power for running implements like mowers and tillers. Evaluate whether two-wheel or four-wheel drive suits your terrain. Used equipment can save 40-60% over new."}
{"input": "how to enhance creativity? /only:hyde", "output": "hyde: To enhance creativity, practice divergent thinking by generating many ideas without judgment. Keep a daily journal, expose yourself to new experiences, and set aside unstructured time for daydreaming. Research shows that walking, adequate sleep, and constraints can all stimulate creative problem-solving."}
{"input": "how does culture influence ethics /only:hyde", "output": "hyde: Culture shapes ethics by defining what a society considers right or wrong. Collectivist cultures may prioritize group harmony and duty to family, while individualist cultures emphasize personal autonomy and rights. Cultural relativism argues that moral standards are culturally defined, while universalists hold that some ethical principles transcend culture."}
{"input": "what are writing prompts? /only:vec", "output": "vec: what are writing prompts and how do writers use them for inspiration\nvec: how do writing prompts help overcome writer's block and spark creativity"}
{"input": "what is moral philosophy /only:hyde", "output": "hyde: Moral philosophy, or ethics, is the branch of philosophy concerned with questions of right and wrong conduct. It includes three main branches: metaethics (the nature of moral judgments), normative ethics (frameworks like utilitarianism, deontology, and virtue ethics), and applied ethics (specific issues like abortion or euthanasia)."}
{"input": "how to participate in public policy discussions /only:lex", "output": "lex: participate public policy discussion civic\nlex: public policy engagement town hall\nlex: citizen participation policy advocacy"}
{"input": "recent React changes 2025 /only:lex", "output": "lex: recent React changes 2025 guide\nlex: recent React changes 2025 best practices\nlex: recent React changes 2025 examples"}
{"input": "rate limiting algorithms /only:vec", "output": "vec: what algorithms are used for api rate limiting and how do they differ\nvec: how do token bucket and sliding window rate limiting algorithms work"}
{"input": "how to volunteer for civic initiatives /only:lex", "output": "lex: volunteer civic initiatives community service\nlex: volunteering local government community projects"}
{"input": "dependency injection benefits /only:lex", "output": "lex: dependency injection di pattern\nlex: di inversion of control ioc\nlex: dependency injection testing"}
{"input": "what is the significance of algae in ecosystems /only:hyde", "output": "hyde: Algae produce approximately 50% of the world's oxygen through photosynthesis and form the base of aquatic food chains. Phytoplankton, a type of microalgae, supports marine ecosystems by providing energy to zooplankton, fish, and larger organisms."}
{"input": "json serial /only:lex", "output": "lex: JSON serialization deserialization\nlex: JSON serialize object string\nlex: JSON stringify parse encoding"}
{"input": "Kubernetes recent news November /only:vec", "output": "vec: how to Kubernetes recent news November\nvec: guide for Kubernetes recent news November"}
{"input": "how to grow blueberries at home? /only:lex", "output": "lex: grow blueberries home garden\nlex: blueberry bush planting acidic soil\nlex: container blueberry growing care"}
{"input": "database sharding strategies /only:vec", "output": "vec: what is database sharding and what strategies exist for partitioning data\nvec: how do you choose a shard key and what are the tradeoffs of different sharding approaches"}
{"input": "craigslist ads /only:vec", "output": "vec: how to post and browse classified ads on Craigslist\nvec: how does Craigslist work for buying, selling, and listing items locally"}
{"input": "how to protect business data /only:lex", "output": "lex: protect business data security cybersecurity\nlex: data protection encryption backup strategy\nlex: business data security firewall access control"}
{"input": "what is an elevator pitch /only:vec", "output": "vec: what is an elevator pitch and how do you structure an effective one\nvec: how do you deliver a compelling 30-second pitch for a business idea or job opportunity"}
{"input": "regex match /only:hyde", "output": "hyde: A regex (regular expression) matches text patterns. Common syntax: `.` matches any character, `*` means zero or more, `+` means one or more, `?` means optional. `[a-z]` matches lowercase letters. `\\d` matches digits. Capture groups use parentheses: `(\\d{3})-(\\d{4})` matches and captures phone number parts. Use `^` for start and `$` for end of line."}
{"input": "how do i vote in person /only:lex", "output": "lex: vote in person polling place Election Day\nlex: in-person voting process ID requirements"}
{"input": "what is the significance of logic in philosophy /only:lex", "output": "lex: logic philosophy significance role\nlex: formal logic philosophical argument validity"}
{"input": "how climate change affects farming /only:vec", "output": "vec: how does rising global temperature affect crop yields and food production\nvec: what effects does climate change have on soil quality and growing seasons for farmers"}
{"input": "machine learning recent news November /only:vec", "output": "vec: how to machine learning recent news November\nvec: understanding machine learning recent news November"}
{"input": "kafka consumer groups /only:hyde", "output": "hyde: Consumers with the same group.id share partitions\u2014each partition is consumed by only one consumer in the group. Adding consumers triggers rebalancing. If consumers > partitions, some idle. Offsets track progress per partition. Use enable.auto.commit=false for exactly-once semantics with manual commits."}
{"input": "what is yoga and its benefits /only:hyde", "output": "hyde: Yoga is an ancient practice combining physical postures (asanas), breathing techniques (pranayama), and meditation. Regular practice improves flexibility, builds strength, reduces stress and anxiety, lowers blood pressure, and enhances sleep quality. Styles range from gentle Hatha to vigorous Vinyasa and Ashtanga."}
{"input": "how to approach ethical decision-making /only:vec", "output": "vec: what frameworks or steps help with making ethical decisions in difficult situations\nvec: how do you systematically evaluate moral choices when facing an ethical dilemma"}
{"input": "what are exchange-traded funds (etfs) /only:lex", "output": "lex: exchange-traded funds ETFs investing\nlex: ETF index fund stock market\nlex: ETF vs mutual fund comparison"}
{"input": "what is the history of the jazz age /only:lex", "output": "lex: Jazz Age history 1920s\nlex: Jazz Age Harlem Renaissance Roaring Twenties\nlex: jazz music history Louis Armstrong"}
{"input": "what was the impact of the industrial revolution on society? /only:vec", "output": "vec: how did the Industrial Revolution transform society, economy, and daily life?\nvec: what were the major social and economic impacts of the Industrial Revolution on workers and cities?"}
{"input": "what are the benefits of yoga /only:lex", "output": "lex: yoga benefits health flexibility stress\nlex: yoga physical mental health advantages"}
{"input": "how to write a business plan /only:lex", "output": "lex: business plan writing template sections\nlex: business plan executive summary financial projections"}
{"input": "what was the role of the catholic church in the middle ages? /only:lex", "output": "lex: Catholic Church Middle Ages role\nlex: medieval church political power papacy\nlex: Catholic Church feudalism education medieval"}
{"input": "Vue recent news October /only:lex", "output": "lex: Vue recent news October documentation\nlex: Vue recent news October guide\nlex: Vue recent news October tutorial"}
{"input": "what is a protagonist? /only:hyde", "output": "hyde: The protagonist is the central character of a narrative, the one whose goals and conflicts drive the plot. The story is told from their perspective or follows their journey. Protagonists are not always heroes\u2014they can be antiheroes or morally ambiguous characters. The antagonist opposes the protagonist, creating the central conflict of the story."}
{"input": "what is mixed media art? /only:lex", "output": "lex: mixed media art techniques materials\nlex: mixed media collage painting assemblage"}
{"input": "what are the foundations of feminist ethics /only:hyde", "output": "hyde: Feminist ethics emerged from Carol Gilligan's critique of Kohlberg's moral development theory, arguing that women's moral reasoning emphasizes care and relationships rather than abstract principles of justice. Nel Noddings developed the ethics of care, centering moral life on attentiveness, responsibility, and responsiveness to the needs of particular others."}
{"input": "renaissance literature /only:lex", "output": "lex: Renaissance literature authors works\nlex: Renaissance literary period Shakespeare Petrarch humanism"}
{"input": "how to build self-confidence /only:hyde", "output": "hyde: Start by setting small, achievable goals and completing them\u2014each success builds evidence of competence. Practice self-compassion: replace harsh self-criticism with the tone you'd use with a friend. Keep a \"wins\" journal and review it weekly. Gradually expand your comfort zone by doing one slightly uncomfortable thing each day. Confidence grows from accumulated experience, not positive thinking alone."}
{"input": "how do philosophers define happiness /only:vec", "output": "vec: how have major philosophers throughout history defined happiness and well-being?\nvec: what is the difference between Aristotle's eudaimonia and hedonistic views of happiness?"}
{"input": "how to manage personal finances /only:lex", "output": "lex: personal finance management\nlex: manage money budgeting saving investing\nlex: personal financial planning"}
{"input": "yaml vs json config /only:hyde", "output": "hyde: JSON: strict syntax, no comments, explicit quotes, universal parsing. YAML: superset of JSON, allows comments, cleaner for humans, indentation-based. Use JSON for data interchange, APIs, when strict parsing needed. Use YAML for configs (Docker Compose, Kubernetes, CI/CD) where human editing is common. YAML gotchas: Norway problem (NO parsed as false), inconsistent indentation."}
{"input": "Kubernetes changelog 2026 /only:lex", "output": "lex: Kubernetes changelog 2026 documentation\nlex: Kubernetes changelog 2026 examples\nlex: Kubernetes changelog 2026 tutorial"}
{"input": "retirement planning strategies /only:lex", "output": "lex: retirement planning strategies guide\nlex: retirement planning strategies documentation\nlex: retirement planning strategies examples"}
{"input": "how to participate in public policy discussions /only:vec", "output": "vec: how can citizens effectively participate in public policy discussions and influence government decisions?\nvec: what are the ways individuals can engage in public policy debates at the local, state, and federal level?"}
{"input": "how to repair a leaky faucet /only:vec", "output": "vec: how to fix a dripping faucet by replacing the washer or cartridge\nvec: what are the step-by-step instructions for repairing a leaky kitchen or bathroom faucet"}
{"input": "Meiji Restoration Japan /only:hyde", "output": "hyde: The Meiji Restoration (1868) ended 250 years of Tokugawa shogunate rule, restoring imperial power under Emperor Meiji. Japan rapidly industrialized and westernized: abolished feudalism, created national army, built railways, established constitution (1889). Slogan: 'Rich country, strong army.' Japan defeated China (1895) and Russia (1905), becoming a world power within 50 years."}
{"input": "AI recent news December /only:lex", "output": "lex: AI recent news December documentation\nlex: AI recent news December guide\nlex: AI recent news December best practices"}
{"input": "ocean plastic pollution /only:lex", "output": "lex: ocean plastic pollution examples\nlex: ocean plastic pollution guide\nlex: ocean plastic pollution documentation"}
{"input": "React context vs Redux /only:hyde", "output": "hyde: Context is built-in, simple for low-frequency updates like themes and auth. Redux adds boilerplate but provides devtools, middleware, time-travel debugging, predictable updates. Context re-renders all consumers on any change; Redux allows granular subscriptions. Use Context for simple cases, Redux for complex state logic."}
{"input": "IPv4 vs IPv6 /only:lex", "output": "lex: ipv4 ipv6 difference\nlex: ipv6 address format\nlex: ipv4 exhaustion ipv6 transition"}
{"input": "how to repair a leaky faucet /only:hyde", "output": "hyde: Turn off the water supply valves under the sink. Remove the faucet handle by unscrewing the decorative cap and handle screw. Pull out the stem or cartridge. For compression faucets, replace the rubber washer and O-ring. For cartridge faucets, replace the entire cartridge. Reassemble, turn the water back on, and test for leaks."}
{"input": "what is the role of faith in spirituality /only:hyde", "output": "hyde: Faith in spirituality serves as the foundation for trust in a reality beyond the material world. It enables surrender to uncertainty and provides a framework for interpreting suffering and purpose. Unlike dogmatic belief, spiritual faith often involves personal experience\u2014a felt sense of connection to something greater that sustains practice through doubt and difficulty."}
{"input": "what causes market volatility /only:vec", "output": "vec: what economic and geopolitical factors cause stock market volatility\nvec: why do financial markets experience sudden price swings and instability"}
{"input": "calculus derivatives explained /only:lex", "output": "lex: calculus derivatives explained best practices\nlex: calculus derivatives explained examples\nlex: calculus derivatives explained documentation"}
{"input": "machine learning changelog 2025 /only:lex", "output": "lex: machine learning changelog 2025 examples\nlex: machine learning changelog 2025 guide\nlex: machine learning changelog 2025 best practices"}
{"input": "how to kayak for the first time /only:vec", "output": "vec: what should a beginner know before going kayaking for the first time\nvec: how do I paddle and balance a kayak as a first-time kayaker"}
{"input": "how to participate in earth hour? /only:lex", "output": "lex: Earth Hour participation lights off event\nlex: Earth Hour date 2026 how to join"}
{"input": "TypeScript changelog 2025 /only:lex", "output": "lex: TypeScript changelog 2025 documentation\nlex: TypeScript changelog 2025 examples\nlex: TypeScript changelog 2025 guide"}
{"input": "what is the importance of spiritual leadership? /only:vec", "output": "vec: how does spiritual leadership influence organizations and their members\nvec: what role does spiritual leadership play in providing meaning and purpose at work"}
{"input": "rivers that cross multiple countries /only:lex", "output": "lex: rivers that cross multiple countries documentation\nlex: rivers that cross multiple countries tutorial\nlex: rivers that cross multiple countries guide"}
{"input": "async web /only:vec", "output": "vec: how do asynchronous programming patterns work in web development and API requests?\nvec: what are the best async web frameworks for building non-blocking HTTP servers?"}
{"input": "where to find eco-friendly furniture /only:vec", "output": "vec: where can I buy eco-friendly and sustainably made furniture\nvec: what brands and stores sell furniture made from sustainable or recycled materials"}
{"input": "when to replace windshield wipers? /only:vec", "output": "vec: how often should windshield wipers be replaced and what are signs they need changing\nvec: what are the signs that windshield wiper blades are worn out and need replacement"}
{"input": "what are the characteristics of a just society /only:lex", "output": "lex: just society characteristics principles fairness\nlex: social justice equality Rawls distributive justice"}
{"input": "how to manage sibling rivalry? /only:hyde", "output": "hyde: Avoid comparing siblings to each other. Give each child individual attention and acknowledge their unique strengths. Teach conflict resolution skills rather than always intervening. Set clear family rules about respectful behavior and let children solve minor disputes themselves."}
{"input": "how to build self-confidence /only:vec", "output": "vec: what are practical strategies for building self-confidence and overcoming self-doubt\nvec: how can someone develop greater self-confidence through daily habits and mindset shifts"}
{"input": "React changelog 2026 /only:lex", "output": "lex: React changelog 2026 tutorial\nlex: React changelog 2026 examples\nlex: React changelog 2026 documentation"}
{"input": "what are the best soil types for roses /only:vec", "output": "vec: what type of soil do roses grow best in and how should it be prepared\nvec: what soil pH and composition are ideal for growing healthy rose bushes"}
{"input": "how does compound interest work /only:hyde", "output": "hyde: Compound interest is calculated on both the principal and accumulated interest. The formula is A = P(1 + r/n)^(nt), where P is principal, r is annual rate, n is compounding frequency, and t is time in years. Monthly compounding on $10,000 at 5% yields $16,470 after 10 years."}
{"input": "tail recursion optimization /only:hyde", "output": "hyde: Tail recursion: recursive call is the last operation, no work after it returns. TCO reuses stack frame instead of adding new one\u2014prevents stack overflow. Convert by passing accumulated result as parameter: factorial(n, acc=1) { return n <= 1 ? acc : factorial(n-1, n*acc); }. Not all languages implement TCO\u2014JavaScript in strict mode, Scheme yes, Python no."}
{"input": "TypeScript generics /only:hyde", "output": "hyde: Generics let you write flexible, reusable code while maintaining type safety. Declare with angle brackets: function identity<T>(arg: T): T { return arg; }. Add constraints with extends: function getLength<T extends { length: number }>(item: T): number { return item.length; }."}
{"input": "what is the importance of spiritual leadership? /only:hyde", "output": "hyde: Spiritual leadership theory proposes that leaders who foster a sense of calling, meaning, and membership create more engaged and productive organizations. It emphasizes vision, altruistic love, and hope as core values that transcend traditional management."}
{"input": "how does culture influence identity? /only:vec", "output": "vec: how does culture shape a person's sense of identity\nvec: in what ways do cultural values and traditions influence who we become"}
{"input": "digital transformation strategies /only:vec", "output": "vec: what strategies do organizations use to drive successful digital transformation\nvec: how do enterprises plan and execute a digital transformation initiative"}
{"input": "how to build confidence in social situations? /only:hyde", "output": "hyde: Start small: make eye contact and greet one new person at each event. Prepare a few open-ended questions in advance. Focus on listening rather than performing. After each interaction, note what went well. Gradual exposure reduces anxiety over time\u2014the more you practice, the more natural conversations become."}
{"input": "how to analyze a political candidate's stance /only:lex", "output": "lex: analyze political candidate stance positions\nlex: candidate policy positions voting record\nlex: compare political candidates issues"}
{"input": "JWT token refresh /only:vec", "output": "vec: how does the jwt refresh token flow work for maintaining user sessions\nvec: what is the difference between access tokens and refresh tokens in jwt authentication"}
{"input": "where to buy used cars online /only:lex", "output": "lex: buy used cars online marketplace\nlex: certified pre-owned cars website\nlex: online used car dealers Carvana AutoTrader"}
{"input": "GitHub Actions workflow /only:lex", "output": "lex: GitHub Actions workflow guide\nlex: GitHub Actions workflow examples\nlex: GitHub Actions workflow documentation"}
{"input": "git push /only:hyde", "output": "hyde: Use `git push origin main` to push your local main branch to the remote. For a new branch, use `git push -u origin feature-branch` to set the upstream tracking reference. If the push is rejected because the remote has new commits, run `git pull --rebase` first, then push again."}
{"input": "what is the categorical imperative /only:vec", "output": "vec: what is Kant's categorical imperative and how does it function as a moral principle\nvec: how does the categorical imperative test whether an action is morally permissible"}
{"input": "how to build a personal brand /only:hyde", "output": "hyde: Build your personal brand by defining your niche and unique value proposition. Create consistent profiles across LinkedIn, Twitter, and a personal website. Publish content regularly\u2014blog posts, videos, or podcasts\u2014that demonstrates your expertise. Engage authentically with your audience and network at industry events."}
{"input": "how augmented reality is applied in different fields /only:hyde", "output": "hyde: Augmented reality overlays digital content onto the real world and is applied across many fields. In healthcare, surgeons use AR to visualize anatomy during procedures. In education, AR apps bring textbook content to life in 3D. Retailers like IKEA use AR to let customers preview furniture in their homes. In manufacturing, AR guides workers through assembly with step-by-step overlays."}
{"input": "memory leak debugging /only:lex", "output": "lex: memory leak debug profiler\nlex: memory leak detection tools\nlex: heap dump memory analysis"}
{"input": "what is zero waste? /only:hyde", "output": "hyde: Zero waste is a philosophy and lifestyle aiming to send nothing to landfills by reducing consumption, reusing items, recycling, and composting. Practical steps include using reusable bags, bottles, and containers, buying in bulk, composting food scraps, and choosing products with minimal or recyclable packaging."}
{"input": "how to enhance concentration /only:hyde", "output": "hyde: Improve concentration by eliminating distractions: silence notifications, use website blockers, and work in a quiet environment. The Pomodoro Technique\u201425 minutes of focused work followed by a 5-minute break\u2014builds sustained attention. Regular exercise, adequate sleep (7-9 hours), and mindfulness meditation physically strengthen the brain's prefrontal cortex."}
{"input": "how to improve workplace productivity /only:vec", "output": "vec: what strategies and techniques can improve productivity in the workplace\nvec: how can employees and managers increase work output and reduce wasted time"}
{"input": "how to fix car key fob? /only:vec", "output": "vec: how do I fix a car key fob that stopped working\nvec: how to replace the battery or reprogram a car key fob"}
{"input": "what are the key features of taoist philosophy? /only:vec", "output": "vec: what are the central concepts and key features of Taoist philosophy?\nvec: how does Taoism emphasize living in harmony with the Tao and the concept of wu wei?"}
{"input": "how to write a standout personal statement /only:hyde", "output": "hyde: Open with a vivid, specific anecdote\u2014not a generic quote. Show rather than tell by describing experiences that shaped your goals. Connect your past to your intended field of study. Be authentic; admissions officers read thousands of essays and recognize genuine voice immediately."}
{"input": "what is outdoor survival training? /only:hyde", "output": "hyde: Outdoor survival training teaches skills needed to stay alive in wilderness emergencies. Core topics include building emergency shelters from natural materials, finding and purifying water, starting fire without matches using a ferro rod or bow drill, signaling for rescue, and basic navigation without GPS. Courses range from weekend workshops to multi-week immersive programs."}
{"input": "how do philosophers approach the meaning of life /only:hyde", "output": "hyde: Existentialists like Sartre argued life has no inherent meaning\u2014we must create it through our choices. Aristotle proposed eudaimonia (flourishing) as life's purpose. Camus explored the absurd, suggesting we must find meaning despite an indifferent universe. Eastern philosophy often points to liberation from suffering."}
{"input": "japanese hiragana katakana /only:vec", "output": "vec: complete japanese hiragana katakana reference\nvec: guide for japanese hiragana katakana"}
{"input": "what changed in machine learning 2025 /only:lex", "output": "lex: what changed in machine learning 2025 guide\nlex: what changed in machine learning 2025 best practices\nlex: what changed in machine learning 2025 documentation"}
{"input": "how to create a home office space /only:hyde", "output": "hyde: Set up your home office in a quiet room with natural light. Invest in an ergonomic chair with lumbar support and a desk at elbow height (28-30 inches). Position your monitor at arm's length with the top at eye level. Use a desk lamp with 4000-5000K color temperature. Keep cables organized and add a plant\u2014studies show greenery reduces stress and improves focus."}
{"input": "latest Shopify updates /only:vec", "output": "vec: complete latest Shopify updates reference\nvec: guide for latest Shopify updates"}
{"input": "Next.js new features 2026 /only:vec", "output": "vec: learn about Next.js new features 2026\nvec: how to Next.js new features 2026"}
{"input": "how did the roman empire impact culture? /only:hyde", "output": "hyde: The Roman Empire's cultural legacy includes Latin (the root of Romance languages), Roman law (the basis of civil law systems worldwide), architectural innovations like arches, aqueducts, and concrete, republican government concepts, road networks, and the spread of Christianity. Roman art, literature, and engineering influenced Western civilization for centuries."}
{"input": "how to participate in a town hall meeting /only:hyde", "output": "hyde: Check your local government website or social media for upcoming town hall schedules. Arrive early and sign up to speak if required. Prepare a concise statement (usually 2-3 minutes). Stay respectful and on-topic. Bring supporting data or personal stories to strengthen your point."}
{"input": "how to be a good listener /only:vec", "output": "vec: how can I become a better and more active listener in conversations\nvec: what techniques improve listening skills and show empathy"}
{"input": "web socket /only:hyde", "output": "hyde: WebSocket provides full-duplex communication over a single TCP connection. After an HTTP upgrade handshake, client and server can send messages in both directions without polling. Use `new WebSocket('ws://host/path')` on the client and a library like ws on the server."}
{"input": "apache kafka partitions /only:hyde", "output": "hyde: Partitions enable parallelism\u2014each partition is consumed by one consumer in a group. Messages with same key go to same partition, preserving order per key. More partitions = more throughput but more overhead. Start with partitions = max(expected throughput / partition throughput, consumer count). Can't reduce partitions, only increase."}
{"input": "inflation effects on savings /only:vec", "output": "vec: guide for inflation effects on savings\nvec: complete inflation effects on savings reference"}
{"input": "nginx location block /only:vec", "output": "vec: how do nginx location blocks work and in what order are they matched\nvec: what is the syntax for nginx location directives including prefix and regex matching"}
{"input": "IPv4 vs IPv6 /only:vec", "output": "vec: what are the key differences between ipv4 and ipv6 addressing\nvec: why is ipv6 necessary and how does the transition from ipv4 work"}
{"input": "how to raise startup capital /only:hyde", "output": "hyde: Startup capital can come from bootstrapping, friends and family, angel investors, venture capital firms, crowdfunding platforms like Kickstarter, or government grants. Prepare a pitch deck with your business model, market size, traction metrics, and financial projections before approaching investors."}
{"input": "tectonic plate boundaries /only:lex", "output": "lex: tectonic plate boundaries examples\nlex: tectonic plate boundaries documentation\nlex: tectonic plate boundaries best practices"}
{"input": "surfing wave types /only:lex", "output": "lex: surfing wave types tutorial\nlex: surfing wave types examples\nlex: surfing wave types documentation"}
{"input": "how to use a light meter /only:lex", "output": "lex: light meter photography exposure reading\nlex: incident reflected light meter settings"}
{"input": "what is literary parody? /only:hyde", "output": "hyde: Literary parody imitates the style, conventions, or content of a specific work or genre for comedic or critical effect. It exaggerates distinctive features to expose flaws or absurdities. Examples include Don Quixote (parodying chivalric romances), Northanger Abbey (Gothic novels), and The Hitchhiker's Guide to the Galaxy (science fiction tropes)."}
{"input": "how to install peel and stick wallpaper /only:lex", "output": "lex: peel and stick wallpaper installation\nlex: self-adhesive wallpaper apply walls"}
{"input": "how to meditate for beginners /only:vec", "output": "vec: how do beginners start a daily meditation practice from scratch\nvec: what are simple meditation techniques for people who have never meditated before"}
{"input": "renaissance sculpture techniques /only:lex", "output": "lex: renaissance sculpture techniques documentation\nlex: renaissance sculpture techniques examples\nlex: renaissance sculpture techniques best practices"}
{"input": "how to save money effectively /only:vec", "output": "vec: what are effective strategies and habits for saving money consistently\nvec: how can I create a budget and save more money each month"}
{"input": "GraphQL subscriptions websocket /only:vec", "output": "vec: how do graphql subscriptions work for real-time data updates\nvec: what is the underlying protocol for graphql subscriptions and how do you implement them"}
{"input": "what is the significance of algae in ecosystems /only:lex", "output": "lex: algae ecosystem role food chain\nlex: algae oxygen production aquatic ecosystems\nlex: algae photosynthesis carbon cycle"}
{"input": "React recent news November /only:lex", "output": "lex: React recent news November best practices\nlex: React recent news November examples\nlex: React recent news November guide"}
{"input": "sailing adventures /only:lex", "output": "lex: sailing adventure trips voyages\nlex: sailing vacation destinations cruises\nlex: ocean sailing expedition"}
{"input": "Elasticsearch query DSL /only:lex", "output": "lex: elasticsearch query dsl\nlex: elasticsearch bool must should\nlex: es full text search query"}
{"input": "what are the elements of short stories? /only:hyde", "output": "hyde: The essential elements of a short story are plot (the sequence of events), character (the people involved), setting (time and place), conflict (the central struggle), theme (the underlying message), and point of view (the narrative perspective). Short stories typically focus on a single incident."}
{"input": "Docker changelog 2026 /only:vec", "output": "vec: understanding Docker changelog 2026\nvec: complete Docker changelog 2026 reference"}
{"input": "what is the role of sacred music in worship? /only:vec", "output": "vec: what role does sacred music play in religious worship services across different faiths\nvec: how do hymns, chants, and liturgical music enhance the experience of communal worship"}
{"input": "hair cut /only:vec", "output": "vec: what are the popular haircut styles and how to choose the right one\nvec: how to communicate what haircut you want to a stylist or barber"}
{"input": "Han Dynasty China achievements /only:lex", "output": "lex: han dynasty china achievements\nlex: han dynasty 206 bc history\nlex: ancient china han empire"}
{"input": "how to increase productivity at work? /only:lex", "output": "lex: productivity work increase tips\nlex: workplace productivity time management techniques"}
{"input": "what is the significance of the alhambra? /only:vec", "output": "vec: why is the Alhambra in Granada, Spain considered a masterpiece of Islamic architecture?\nvec: what is the cultural and historical significance of the Alhambra palace?"}
{"input": "what is bioethics /only:lex", "output": "lex: bioethics definition medical ethics biology\nlex: bioethics issues euthanasia cloning genetic engineering"}
{"input": "how to build a successful brand /only:lex", "output": "lex: brand building strategy identity positioning\nlex: brand identity logo messaging target audience"}
{"input": "stellar cartography /only:vec", "output": "vec: what is stellar cartography and how do astronomers map the positions and movements of stars?\nvec: what tools and surveys are used to create detailed maps of stars in the galaxy?"}
{"input": "what are the key principles of confucianism? /only:hyde", "output": "hyde: The key principles of Confucianism include Ren (benevolence/humaneness), Li (ritual propriety), Xiao (filial piety), Yi (righteousness), and Zhi (wisdom). The Five Relationships define social bonds: ruler-subject, parent-child, husband-wife, elder-younger sibling, and friend-friend. Each relationship carries reciprocal obligations."}
{"input": "what changed in GitHub 2026 /only:lex", "output": "lex: what changed in GitHub 2026 tutorial\nlex: what changed in GitHub 2026 guide\nlex: what changed in GitHub 2026 documentation"}
{"input": "when to introduce solid foods to a baby? /only:hyde", "output": "hyde: Most pediatricians recommend introducing solid foods around 6 months of age. Signs of readiness include sitting up with support, showing interest in food, and loss of the tongue-thrust reflex. Start with single-ingredient purees like sweet potato, avocado, or iron-fortified cereal, one new food every 3-5 days."}
{"input": "how to mix modern and vintage decor /only:vec", "output": "vec: how do you blend vintage furniture and antique pieces with modern interior design elements\nvec: what are effective ways to combine mid-century or antique decor with contemporary minimalist style"}
{"input": "latest climate tech updates /only:vec", "output": "vec: understanding latest climate tech updates\nvec: complete latest climate tech updates reference"}
{"input": "SQL injection prevention /only:vec", "output": "vec: how to prevent sql injection attacks in web applications\nvec: why are parameterized queries and prepared statements important for database security"}
{"input": "how to use a ring light /only:hyde", "output": "hyde: Place the ring light directly in front of your face at eye level, with the camera positioned in the center of the ring. Keep the light 12-24 inches from your face for an even, shadow-free glow. Adjust brightness to avoid overexposure. The circular catchlights in the eyes are a signature look."}
{"input": "how to increase home resale value /only:lex", "output": "lex: increase home resale value renovations\nlex: home improvement ROI property value"}
{"input": "how to set financial goals /only:lex", "output": "lex: set financial goals planning budget\nlex: financial goal setting SMART savings\nlex: personal finance goals short long term"}
{"input": "GitHub new features 2025 /only:lex", "output": "lex: GitHub new features 2025 guide\nlex: GitHub new features 2025 tutorial\nlex: GitHub new features 2025 examples"}
{"input": "how to handle inflation impact /only:vec", "output": "vec: how can individuals protect their finances and manage the impact of high inflation\nvec: what financial strategies help people cope with rising prices and reduced purchasing power"}
{"input": "Python new features 2026 /only:vec", "output": "vec: guide for Python new features 2026\nvec: complete Python new features 2026 reference"}
{"input": "how to manage anxiety naturally /only:hyde", "output": "hyde: Natural anxiety management includes regular aerobic exercise (30 minutes, 5 days a week), diaphragmatic breathing, progressive muscle relaxation, and limiting caffeine and alcohol. Cognitive behavioral techniques like thought journaling help identify and challenge anxious thinking patterns. Herbal supplements such as chamomile and ashwagandha show some evidence of benefit."}
{"input": "how to talk to kids about bullying? /only:lex", "output": "lex: talk children bullying conversation advice\nlex: kids bullying prevention parent discussion"}
{"input": "what is impact investing? /only:hyde", "output": "hyde: Impact investing directs capital toward companies and projects that generate measurable social or environmental benefits alongside financial returns. Unlike ESG screening, which excludes harmful sectors, impact investing actively targets positive outcomes \u2014 such as affordable housing, renewable energy, or microfinance. The Global Impact Investing Network (GIIN) estimates the market at over $1 trillion."}
{"input": "what is competitive analysis /only:vec", "output": "vec: what is competitive analysis in business and how do companies use it to inform strategy\nvec: what frameworks and methods are used to conduct a competitive analysis of rival companies"}
{"input": "latest updates on the ukraine conflict /only:vec", "output": "vec: what are the most recent developments in the Russia-Ukraine war as of 2025-2026?\nvec: what is the current status of the Ukraine conflict including ceasefire talks and territorial changes?"}
{"input": "what is the role of research institutions /only:lex", "output": "lex: research institutions universities role science\nlex: research institutions funding labs innovation"}
{"input": "how to enhance positive social impact? /only:lex", "output": "lex: enhance social impact community\nlex: positive social impact strategies nonprofit\nlex: social change community engagement"}
{"input": "what is the significance of pilgrimage in religion? /only:hyde", "output": "hyde: Pilgrimage holds deep significance across religions. Muslims perform Hajj to Mecca as one of the Five Pillars. Christians journey to Jerusalem, Rome, and Santiago de Compostela. Hindus bathe in the Ganges at Varanasi. The physical journey symbolizes an inner spiritual transformation\u2014leaving ordinary life, enduring hardship, and arriving at a sacred place of renewal and encounter with the divine."}
{"input": "how to stay updated on global affairs /only:lex", "output": "lex: global affairs news sources current events\nlex: world news reliable sources daily updates"}
{"input": "where to buy affordable art prints /only:hyde", "output": "hyde: Affordable art prints are available on Society6, Redbubble, and Etsy, where independent artists sell prints starting at $15\u2013$30. IKEA offers framed prints under $20. For museum-quality reproductions, check Artsy or Saatchi Art's prints section. King & McGaw specializes in licensed fine art reproductions at mid-range prices."}
{"input": "building resilience /only:hyde", "output": "hyde: Building resilience involves developing a growth mindset, maintaining social connections, and practicing self-care. Reframe setbacks as learning opportunities. Cultivate problem-solving skills rather than ruminating on what went wrong. Regular exercise, adequate sleep, and mindfulness strengthen your capacity to recover from stress. Resilient people accept what they cannot control and focus energy on what they can."}
{"input": "what is the significance of community in spirituality? /only:vec", "output": "vec: why is community considered important in spiritual and religious practice?\nvec: how does belonging to a spiritual community enhance personal faith and practice?"}
{"input": "what is the significance of the hajj in islam? /only:lex", "output": "lex: Hajj Islam pilgrimage Mecca significance\nlex: Hajj pillar Islam Kaaba rituals"}
{"input": "what is the significance of community in ethics /only:vec", "output": "vec: what role does community play in ethical theory and moral life\nvec: how does communitarian philosophy view the relationship between community and ethics"}
{"input": "how to reduce stress /only:hyde", "output": "hyde: Regular physical activity releases endorphins that naturally reduce stress. Practice deep breathing: inhale for 4 counts, hold for 4, exhale for 6. Other effective strategies include progressive muscle relaxation, journaling, limiting caffeine, and maintaining a consistent sleep schedule of 7-9 hours."}
{"input": "how to improve sleep quality /only:hyde", "output": "hyde: Maintain a consistent sleep schedule, even on weekends. Keep your bedroom cool (65-68\u00b0F), dark, and quiet. Avoid screens for 30 minutes before bed. Limit caffeine after noon. Regular exercise improves sleep, but finish workouts at least 3 hours before bedtime."}
{"input": "ancient Greece democracy Athens /only:lex", "output": "lex: ancient greece democracy athens\nlex: athenian democracy 5th century bc\nlex: greek democracy origins"}
{"input": "solid principles explained /only:hyde", "output": "hyde: SOLID: Single Responsibility (one reason to change), Open/Closed (open for extension, closed for modification), Liskov Substitution (subtypes substitutable for base types), Interface Segregation (many specific interfaces over one general), Dependency Inversion (depend on abstractions not concretions). Following SOLID produces loosely coupled, testable, maintainable code."}
{"input": "TypeScript changelog 2026 /only:vec", "output": "vec: understanding TypeScript changelog 2026\nvec: how to TypeScript changelog 2026"}
{"input": "mac address vs ip address /only:vec", "output": "vec: what is the difference between a mac address and an ip address in networking\nvec: how do mac addresses and ip addresses work together for network communication"}
{"input": "protobuf vs json /only:hyde", "output": "hyde: JSON: human-readable, self-describing, universal support, larger payload. Protobuf: binary format, 3-10x smaller, faster serialization, requires schema (.proto files), strong typing. Use JSON for public APIs, debugging, human interaction. Use Protobuf for internal microservices, high-throughput systems, gRPC. Schema evolution with field numbers enables backward compatibility."}
{"input": "how to find emotional support /only:hyde", "output": "hyde: Find emotional support through multiple channels: talk to a trusted friend or family member. Contact a therapist through Psychology Today's directory or your insurance provider. Call the 988 Suicide and Crisis Lifeline (dial 988) for immediate help. Join support groups through NAMI or local community centers. Online therapy platforms like BetterHelp and Talkspace offer accessible counseling."}
{"input": "how to plant a wildflower meadow? /only:vec", "output": "vec: how do I plant and establish a wildflower meadow in my yard\nvec: what steps are needed to create a wildflower meadow from seed"}
{"input": "how to succeed in a digital marketing career? /only:vec", "output": "vec: what skills and experience do you need to build a successful digital marketing career\nvec: how to get started in digital marketing and advance to senior roles"}
{"input": "what is the difference between realism and idealism /only:lex", "output": "lex: realism idealism philosophy difference\nlex: realism vs idealism metaphysics epistemology\nlex: philosophical realism idealism comparison"}
{"input": "ai-driven analytics /only:hyde", "output": "hyde: AI-driven analytics uses machine learning algorithms to automatically detect patterns, anomalies, and trends in large datasets. Unlike traditional BI tools, AI analytics can generate predictive forecasts, perform natural language queries, and surface insights without manual configuration."}
{"input": "how does the social contract theory explain governance /only:vec", "output": "vec: how does social contract theory explain the legitimacy of government\nvec: what did Hobbes, Locke, and Rousseau argue about the social contract and governance"}
{"input": "how to grow tomatoes at home? /only:lex", "output": "lex: grow tomatoes home garden\nlex: tomato plant care watering sunlight\nlex: container tomatoes growing tips"}
{"input": "what is the significance of archetypes? /only:lex", "output": "lex: archetypes Carl Jung collective unconscious\nlex: archetypes significance literature psychology"}
{"input": "machine learning recent news November /only:lex", "output": "lex: machine learning recent news November tutorial\nlex: machine learning recent news November guide\nlex: machine learning recent news November examples"}
{"input": "Shopify recent news October /only:vec", "output": "vec: learn about Shopify recent news October\nvec: complete Shopify recent news October reference"}
{"input": "AWS latest version release /only:vec", "output": "vec: complete AWS latest version release reference\nvec: understanding AWS latest version release"}
{"input": "budget backpacking europe /only:lex", "output": "lex: budget backpacking europe documentation\nlex: budget backpacking europe guide\nlex: budget backpacking europe examples"}
{"input": "what are the elements of classical music? /only:hyde", "output": "hyde: Classical music is built on melody (a sequence of notes forming a theme), harmony (chords supporting the melody), rhythm (the timing and pattern of notes), dynamics (volume changes), and form (the structure, such as sonata, rondo, or theme and variations)."}
{"input": "electronics soldering guide /only:vec", "output": "vec: learn about electronics soldering guide\nvec: guide for electronics soldering guide"}
{"input": "balance sheet basics /only:lex", "output": "lex: balance sheet basics guide\nlex: balance sheet basics tutorial\nlex: balance sheet basics examples"}
{"input": "soccer formations tactics /only:vec", "output": "vec: complete soccer formations tactics reference\nvec: understanding soccer formations tactics"}
{"input": "how to improve interpersonal skills /only:vec", "output": "vec: what are effective ways to improve interpersonal and communication skills?\nvec: how can someone develop better listening, empathy, and social skills in personal and professional settings?"}
{"input": "enum class /only:lex", "output": "lex: enum class C++ Java strongly typed\nlex: enum class Python enumeration members\nlex: enum class scoped enumeration"}
{"input": "how to prepare for a long hike /only:hyde", "output": "hyde: Train by walking with a loaded pack for progressively longer distances over 4-6 weeks. Pack the ten essentials: navigation, sun protection, insulation, illumination, first aid, fire, tools, nutrition, hydration, and shelter. Check the weather forecast and file a trip plan with someone you trust."}
{"input": "latest findings in climate science /only:vec", "output": "vec: what are the most recent scientific findings about climate change in 2025-2026\nvec: what do the latest climate science studies reveal about global warming trends"}
{"input": "python decorators explained /only:lex", "output": "lex: python decorator function\nlex: python @ decorator syntax\nlex: python wrapper decorator"}
{"input": "awk command examples /only:vec", "output": "vec: how to use awk for text processing and extracting columns from files\nvec: what are common awk patterns and commands for parsing structured text"}
{"input": "what are the building blocks of life /only:hyde", "output": "hyde: The building blocks of life are four types of organic molecules: proteins (made from amino acids), nucleic acids (DNA and RNA from nucleotides), carbohydrates (sugars and polysaccharides), and lipids (fats and phospholipids). These molecules self-assemble into cells, the basic unit of all living organisms."}
{"input": "largest countries by area /only:vec", "output": "vec: understanding largest countries by area\nvec: complete largest countries by area reference"}
{"input": "where to find budget travel tips /only:hyde", "output": "hyde: To travel on a budget, book flights midweek, use fare comparison tools like Google Flights or Skyscanner, stay in hostels or use house-sitting platforms, and eat at local markets instead of tourist restaurants."}
{"input": "how to choose a daycare? /only:lex", "output": "lex: daycare choose selection criteria childcare\nlex: daycare center evaluation safety ratio"}
{"input": "how to make homemade pizza /only:lex", "output": "lex: homemade pizza dough recipe\nlex: pizza from scratch oven toppings\nlex: make pizza dough sauce crust"}
{"input": "mandarin tones guide /only:lex", "output": "lex: mandarin tones guide guide\nlex: mandarin tones guide best practices\nlex: mandarin tones guide examples"}
{"input": "what is the concept of original sin /only:hyde", "output": "hyde: Original sin is the Christian doctrine that humanity inherited a sinful nature from Adam and Eve's disobedience in the Garden of Eden. Augustine of Hippo formalized the teaching, arguing that all humans are born in a state of sin, redeemable only through divine grace."}
{"input": "art class /only:vec", "output": "vec: where can I find art classes for beginners to learn painting or drawing\nvec: what types of art classes are available online and in person for adults"}
{"input": "what is the veil of ignorance /only:hyde", "output": "hyde: The veil of ignorance is a thought experiment by John Rawls in A Theory of Justice (1971). It asks people to choose principles of justice from an \"original position\" where they don't know their own race, gender, wealth, or abilities. Rawls argues this produces fair, impartial rules."}
{"input": "thailand /only:hyde", "output": "hyde: Thailand is a Southeast Asian country known for tropical beaches, ornate temples, and rich cuisine. Bangkok is the capital. Popular destinations include Chiang Mai, Phuket, and the islands of Koh Samui and Phi Phi. Thai food staples include pad thai, green curry, and tom yum soup."}
{"input": "how to make scientific presentations engaging /only:vec", "output": "vec: how can scientists make their research presentations more engaging and accessible\nvec: what techniques improve the delivery and visual design of scientific talks"}
{"input": "spice combinations guide /only:lex", "output": "lex: spice combinations guide documentation\nlex: spice combinations guide tutorial\nlex: spice combinations guide examples"}
{"input": "what is the function of dialogue? /only:vec", "output": "vec: what purpose does dialogue serve in communication and storytelling\nvec: how does dialogue function in literature and everyday interaction"}
{"input": "how do philosophical arguments work /only:lex", "output": "lex: philosophical arguments logic premises conclusion\nlex: philosophical reasoning deductive inductive"}
{"input": "what is the ethics of war /only:hyde", "output": "hyde: Just war theory establishes criteria for morally permissible warfare. Jus ad bellum (right to go to war) requires just cause, legitimate authority, right intention, last resort, proportionality, and reasonable chance of success. Jus in bello (right conduct in war) requires distinction between combatants and civilians and proportional use of force."}
{"input": "what are creative portrait ideas? /only:lex", "output": "lex: creative portrait photography ideas techniques\nlex: portrait photo ideas poses lighting creative"}
{"input": "how to lose weight fast? /only:hyde", "output": "hyde: Safe weight loss is 1-2 pounds per week through a calorie deficit of 500-1000 calories daily. Combine a protein-rich diet with strength training and cardio. Avoid crash diets\u2014they cause muscle loss and metabolic slowdown. Drink water, sleep 7-9 hours, and track food intake for accountability."}
{"input": "how to choose the right camera /only:hyde", "output": "hyde: Decide what you'll shoot most: landscapes, portraits, video, or street photography. Mirrorless cameras are lighter with faster autofocus, while DSLRs offer longer battery life and more lens options. Key specs to compare: sensor size (full-frame vs APS-C), megapixels, autofocus points, and video capabilities. Budget $500-1000 for a capable starter body."}
{"input": "how to encourage siblings to get along? /only:hyde", "output": "hyde: Give each child one-on-one time to reduce competition for attention. Avoid comparing siblings or labeling them (\"the smart one\"). Teach conflict resolution: help them express feelings with \"I\" statements and find compromises. Praise cooperation when you see it. Set clear family rules about physical aggression and name-calling."}
{"input": "Renaissance Italy Florence /only:hyde", "output": "hyde: The Renaissance began in Florence around 1400 due to wealth from banking and trade, political stability, and classical heritage. The Medici family, especially Lorenzo the Magnificent, patronized artists like Leonardo, Michelangelo, and Botticelli. Florence's guilds, humanism from rediscovered Greek texts, and competition among city-states drove cultural innovation."}
{"input": "how to evaluate scientific claims critically /only:lex", "output": "lex: evaluate scientific claims critical thinking\nlex: scientific literacy evidence evaluation peer review"}
{"input": "tech fix /only:vec", "output": "vec: how to troubleshoot and fix common technology problems with computers and devices\nvec: what are basic tech fixes for common software and hardware issues"}
{"input": "latest uses of bioinformatics in research /only:lex", "output": "lex: bioinformatics research applications 2025 2026\nlex: bioinformatics genomics proteomics computational biology"}
{"input": "yoga poses beginners /only:vec", "output": "vec: learn about yoga poses beginners\nvec: guide for yoga poses beginners"}
{"input": "where to find heirloom seed suppliers? /only:hyde", "output": "hyde: Top heirloom seed suppliers include Baker Creek Heirloom Seeds, Seed Savers Exchange, and Johnny's Selected Seeds. Baker Creek offers over 1,800 open-pollinated varieties with free shipping. Seed Savers Exchange is a nonprofit dedicated to preserving rare heirloom varieties through their seed bank and catalog."}
{"input": "what is the composition of the earth's atmosphere /only:hyde", "output": "hyde: Earth's atmosphere is composed of 78.09% nitrogen (N\u2082), 20.95% oxygen (O\u2082), 0.93% argon (Ar), and 0.04% carbon dioxide (CO\u2082). Trace gases include neon, helium, methane, krypton, and water vapor (0-4% depending on humidity). The atmosphere extends roughly 480 km above the surface and is divided into five layers: troposphere, stratosphere, mesosphere, thermosphere, and exosphere."}
{"input": "event sourcing pattern /only:vec", "output": "vec: what is event sourcing and how does it differ from traditional crud data storage\nvec: how do you implement event sourcing and what are its benefits and challenges"}
{"input": "Spanish Conquest Americas /only:vec", "output": "vec: how did spanish conquistadors conquer the aztec and inca empires\nvec: what factors enabled spain to colonize the americas so rapidly in the 16th century"}
{"input": "http client /only:vec", "output": "vec: how to make HTTP requests using an HTTP client library\nvec: which HTTP client libraries are available for making API calls in different languages"}
{"input": "what are the sacred texts of judaism /only:vec", "output": "vec: what are the main sacred texts and scriptures in the Jewish religious tradition\nvec: what is the Torah and what other texts are considered holy in Judaism"}
{"input": "endangered species list /only:vec", "output": "vec: learn about endangered species list\nvec: guide for endangered species list"}
{"input": "how are glaciers formed /only:lex", "output": "lex: glacier formation process ice\nlex: glaciers formed snow compaction accumulation"}
{"input": "what is content marketing /only:vec", "output": "vec: what is content marketing and how does it attract customers\nvec: how do businesses use content marketing to drive traffic and build trust"}
{"input": "what is cycling commute? /only:hyde", "output": "hyde: Cycling commute refers to using a bicycle as your primary transportation to and from work. Bike commuters typically ride 3-15 miles each way, saving on fuel costs while getting daily exercise. Many cities now have protected bike lanes and bike-share programs."}
{"input": "what is the importance of spiritual leadership? /only:lex", "output": "lex: spiritual leadership organizations values\nlex: spiritual leadership workplace meaning purpose"}
{"input": "what is interfaith dialogue? /only:hyde", "output": "hyde: Interfaith dialogue is the cooperative interaction between people of different religious traditions, aimed at mutual understanding rather than conversion. Organizations like the Parliament of the World's Religions bring together leaders from Christianity, Islam, Judaism, Hinduism, Buddhism, and others to discuss shared values and address social issues."}
{"input": "what is the role of enzymes in digestion /only:vec", "output": "vec: how do enzymes help break down food during the digestive process\nvec: what role do specific enzymes like amylase and protease play in digestion"}
{"input": "Aztec Empire civilization /only:lex", "output": "lex: aztec empire civilization\nlex: aztec tenochtitlan mexico\nlex: aztec history mesoamerica"}
{"input": "where to find landscaping stones? /only:hyde", "output": "hyde: Landscaping stones can be purchased from home improvement stores like Home Depot and Lowe's, local stone yards, and quarries. For bulk orders, landscape supply companies deliver directly. River rock, flagstone, and pea gravel are popular choices for garden paths and borders."}
{"input": "how to stay informed about politics /only:hyde", "output": "hyde: Read multiple news sources across the political spectrum: AP News and Reuters for wire reporting, then compare coverage from different outlets. Subscribe to newsletters like The Morning (NYT) or Axios AM. Follow legislative trackers like Congress.gov. Attend local government meetings and candidate forums."}
{"input": "latest GitHub updates /only:lex", "output": "lex: latest GitHub updates guide\nlex: latest GitHub updates documentation\nlex: latest GitHub updates best practices"}
{"input": "how tourism affects local cultures /only:vec", "output": "vec: what are the positive and negative effects of tourism on local cultural traditions and communities\nvec: how does mass tourism change the customs, language, and daily life of host communities"}
{"input": "how to find a reliable realtor /only:lex", "output": "lex: find reliable realtor real estate agent\nlex: choosing trustworthy real estate agent"}
{"input": "monorepo vs polyrepo /only:vec", "output": "vec: what are the tradeoffs between using a monorepo versus multiple repositories\nvec: when does a monorepo make sense and what tools help manage large monorepos"}
{"input": "habit formation science /only:lex", "output": "lex: habit formation science examples\nlex: habit formation science tutorial\nlex: habit formation science documentation"}
{"input": "how to conserve energy in the office? /only:hyde", "output": "hyde: Switch to LED lighting and install occupancy sensors in conference rooms and restrooms. Set computers to sleep mode after 10 minutes of inactivity. Use smart power strips to eliminate phantom loads. Set thermostats to 68\u00b0F in winter and 76\u00b0F in summer. These measures typically reduce office energy use by 20-30%."}
{"input": "what is a mathematical model /only:hyde", "output": "hyde: A mathematical model uses equations and variables to represent a real-world system. For example, the SIR model uses differential equations to predict infectious disease spread: dS/dt = -\u03b2SI, dI/dt = \u03b2SI - \u03b3I, dR/dt = \u03b3I. Models are validated by comparing predictions against observed data and refined iteratively."}
{"input": "unit test vs integration test /only:lex", "output": "lex: unit test integration test difference\nlex: testing pyramid unit integration e2e\nlex: unit test isolation mocking"}
{"input": "what changed in Python 2026 /only:vec", "output": "vec: guide for what changed in Python 2026\nvec: learn about what changed in Python 2026"}
{"input": "how do philosophers explore the nature of reality /only:vec", "output": "vec: how have philosophers historically explored and debated the nature of reality and existence?\nvec: what are the main metaphysical positions on whether reality is fundamentally material, mental, or something else?"}
{"input": "what are the teachings of the baha'i faith? /only:vec", "output": "vec: what are the core beliefs and teachings of the Baha'i faith\nvec: what did Baha'u'llah teach about unity, equality, and world peace"}
{"input": "what is compositional balance? /only:hyde", "output": "hyde: Compositional balance refers to the distribution of visual weight within an image or artwork. Symmetrical balance places equal elements on both sides of a central axis, while asymmetrical balance uses contrasting elements \u2014 such as a large shape offset by a smaller, brighter one \u2014 to create dynamic equilibrium."}
{"input": "how does the body maintain homeostasis /only:vec", "output": "vec: what mechanisms does the human body use to maintain internal stability\nvec: how do feedback loops help regulate body temperature and blood sugar levels"}
{"input": "database transaction isolation levels /only:hyde", "output": "hyde: Isolation levels from weakest to strongest: Read Uncommitted (dirty reads possible), Read Committed (sees only committed data, default in PostgreSQL), Repeatable Read (no non-repeatable reads), Serializable (no phantom reads, full isolation). Higher isolation = more locking = lower concurrency. Choose based on consistency needs vs performance."}
{"input": "what to pack in a hospital bag for labor? /only:hyde", "output": "hyde: Hospital bag essentials for labor: ID and insurance card, birth plan, comfortable robe or gown, slippers, toiletries, phone charger, going-home outfit for you and baby, car seat, nursing bra, newborn diapers, snacks, and a pillow from home."}
{"input": "how to learn about native american culture /only:hyde", "output": "hyde: Visit the National Museum of the American Indian (Smithsonian) or local tribal cultural centers. Read works by Native authors like Joy Harjo, Tommy Orange, and Robin Wall Kimmerer. Attend powwows and cultural events when open to the public. Learn which tribal nations are indigenous to your area."}
{"input": "what is burnout? /only:lex", "output": "lex: burnout syndrome workplace exhaustion\nlex: burnout symptoms causes recovery"}
{"input": "dependency injection benefits /only:hyde", "output": "hyde: Dependency injection provides dependencies from outside rather than creating them internally. Class receives DatabaseService via constructor instead of instantiating it. Benefits: loose coupling, easy testing with mocks, flexible configuration. Instead of new EmailService(), inject interface IEmailService\u2014swap implementations without changing consumer code."}
{"input": "what is the role of media in politics /only:vec", "output": "vec: what role does the media play in shaping political discourse and public opinion\nvec: how does news coverage and media bias influence political outcomes and democracy"}
{"input": "what are plasmids /only:lex", "output": "lex: plasmid DNA circular extrachromosomal\nlex: plasmid bacteria gene transfer cloning\nlex: plasmid vector molecular biology"}
{"input": "how to engage in civil political discussions /only:hyde", "output": "hyde: Start by listening to understand, not to rebut. Ask questions like \"What experiences led you to that view?\" Avoid personal attacks and generalizations. Find common ground before addressing differences. Use \"I\" statements instead of \"you always\" accusations. Accept that changing minds takes time and repeated respectful engagement."}
{"input": "regex lookahead lookbehind /only:hyde", "output": "hyde: Lookahead (?=pattern) matches a position followed by pattern without consuming it. Negative lookahead (?!pattern) matches where pattern doesn't follow. Lookbehind (?<=pattern) matches a position preceded by pattern. Example: \\d+(?= dollars) matches numbers followed by 'dollars'."}
{"input": "how to diversify investment portfolio /only:hyde", "output": "hyde: Diversify across asset classes: stocks, bonds, real estate, and commodities. Within stocks, spread across sectors (tech, healthcare, energy) and geographies (US, international, emerging markets). Use index funds or ETFs for broad exposure. A common allocation is 60% stocks, 30% bonds, 10% alternatives, adjusted by age and risk tolerance."}
{"input": "recent GitHub changes 2026 /only:lex", "output": "lex: recent GitHub changes 2026 tutorial\nlex: recent GitHub changes 2026 examples\nlex: recent GitHub changes 2026 guide"}
{"input": "Sentry error tracking /only:vec", "output": "vec: understanding Sentry error tracking\nvec: learn about Sentry error tracking"}
{"input": "what is the principle of utility? /only:vec", "output": "vec: what is the principle of utility in utilitarian ethics as defined by Bentham and Mill\nvec: how does the utilitarian principle of utility evaluate actions based on their consequences for overall happiness"}
{"input": "portrait photography tips /only:vec", "output": "vec: what are the best tips for taking professional-quality portrait photographs?\nvec: how should you set up lighting, posing, and camera settings for portrait photography?"}
{"input": "what are the core practices of the bah\u00e1'\u00ed faith? /only:lex", "output": "lex: Bah\u00e1'\u00ed faith core practices worship\nlex: Bah\u00e1'\u00ed religion prayer fasting principles"}
{"input": "how to find emotional support /only:vec", "output": "vec: where can someone find emotional support during difficult times or mental health challenges\nvec: what resources are available for people seeking emotional support and counseling"}
{"input": "soccer training drills /only:vec", "output": "vec: what are effective soccer training drills for improving skills and fitness\nvec: which soccer drills help players improve dribbling, passing, and shooting"}
{"input": "how to participate in a protest /only:hyde", "output": "hyde: Know your rights: the First Amendment protects peaceful assembly on public property. Bring water, snacks, a phone charger, and ID. Write an emergency contact number on your arm. Stay with a buddy and agree on a meeting point. Wear comfortable shoes and weather-appropriate clothing. If tear gas is used, move upwind. Document police interactions by filming at a safe distance."}
{"input": "what is the importance of peer review /only:lex", "output": "lex: peer review importance scientific publishing\nlex: peer review process academic research"}
{"input": "idempotency api design /only:hyde", "output": "hyde: Idempotent operations produce the same result regardless of how many times called. GET, PUT, DELETE are naturally idempotent. POST needs idempotency keys: client sends unique key, server stores result, returns cached result on retry. Store keys with TTL (24h). Critical for payment APIs\u2014prevents double charges on network retry."}
{"input": "AWS Lambda functions setup /only:lex", "output": "lex: AWS Lambda functions setup documentation\nlex: AWS Lambda functions setup examples\nlex: AWS Lambda functions setup tutorial"}
{"input": "how to negotiate a business deal /only:hyde", "output": "hyde: Prepare by researching the other party's priorities and constraints. Define your BATNA (best alternative to a negotiated agreement) and walk-away point. Open with an ambitious but defensible anchor. Listen more than you talk. Focus on interests, not positions, to find creative win-win solutions."}
{"input": "how do ethical theories apply to social issues /only:vec", "output": "vec: how are ethical theories like utilitarianism and deontology applied to real-world social issues?\nvec: what ethical frameworks do philosophers use to analyze problems like poverty, inequality, and healthcare?"}
{"input": "S3 bucket policy /only:hyde", "output": "hyde: S3 bucket policies are resource-based JSON policies attached to buckets. Grant public read: {\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":\"*\",\"Action\":\"s3:GetObject\",\"Resource\":\"arn:aws:s3:::bucket/*\"}]}. IAM policies attach to users/roles. Use bucket policies for cross-account access, IAM for user-specific permissions. Block public access settings override policies."}
{"input": "what is virtue signaling? /only:vec", "output": "vec: what does virtue signaling mean and how is the term used in political and social discourse?\nvec: how do people use virtue signaling to publicly express moral values without substantive action?"}
{"input": "what is the significance of the sacred tree in various faiths? /only:hyde", "output": "hyde: Sacred trees appear across religions: the Bodhi tree where Buddha attained enlightenment, the Tree of Life in Genesis, Yggdrasil in Norse mythology connecting the nine worlds, and the banyan in Hinduism symbolizing eternal life. Trees represent growth, connection between earth and heaven, and renewal."}
{"input": "list sort /only:lex", "output": "lex: sort list programming algorithm\nlex: list sort Python Java ascending descending\nlex: array sorting methods comparison"}
{"input": "what makes a good thriller novel? /only:lex", "output": "lex: thriller novel elements writing techniques\nlex: good thriller pacing suspense plot twists"}
{"input": "what is the significance of community in spirituality? /only:hyde", "output": "hyde: Spiritual communities provide shared worship, accountability, and mutual support that deepen individual faith. In Christianity, the church body gathers for fellowship; in Buddhism, the sangha is one of the Three Jewels; in Judaism, a minyan of ten is required for communal prayer. Communal practice reinforces commitment and provides belonging."}
{"input": "latest Python updates /only:vec", "output": "vec: how to latest Python updates\nvec: complete latest Python updates reference"}
{"input": "how to pursue a career in scientific research /only:lex", "output": "lex: scientific research career path academia\nlex: career scientist PhD research position"}
{"input": "where to buy organic seeds? /only:lex", "output": "lex: buy organic seeds online garden\nlex: organic seed suppliers heirloom non-GMO"}
{"input": "industrial revolution inventions /only:vec", "output": "vec: how to industrial revolution inventions\nvec: guide for industrial revolution inventions"}
{"input": "what is an elevator pitch /only:hyde", "output": "hyde: An elevator pitch is a concise, 30-60 second summary of who you are and what you offer. Structure it as: hook (attention-grabbing opening), problem you solve, your solution, and a call to action. Practice until it sounds conversational, not rehearsed."}
{"input": "how to meditate for beginners /only:hyde", "output": "hyde: Sit comfortably with your back straight. Close your eyes and focus on your breath\u2014notice each inhale and exhale. When thoughts arise, gently return attention to your breathing without judgment. Start with 5 minutes daily and gradually increase. Consistency matters more than duration."}
{"input": "where to find datasets for scientific research /only:hyde", "output": "hyde: Public research datasets are available from repositories such as Kaggle, the UCI Machine Learning Repository, NASA's Open Data Portal, NOAA Climate Data, and institutional data archives like Harvard Dataverse and Zenodo."}

File diff suppressed because it is too large Load Diff

View File

@ -1,8 +1,8 @@
{
"dataset_name": "qmd-query-expansion",
"train_samples": 5562,
"val_samples": 618,
"short_query_pct": 26.5,
"train_samples": 1891,
"val_samples": 211,
"short_query_pct": 33.6,
"columns": [
"prompt",
"completion",

View File

@ -13,40 +13,471 @@ except ImportError:
print("Install anthropic: pip install anthropic")
exit(1)
# Sample query templates for diverse training data
# Sample query templates for diverse training data - organized by category
QUERY_TEMPLATES = [
# Technical documentation
# === Technical documentation (35% of queries) ===
"how to {action} {technology}",
"{technology} {concept} example",
"configure {technology} for {use_case}",
"{error_type} error in {technology}",
"best practices for {concept}",
# Personal notes / journals
"{technology} vs {technology2}",
"{action} {technology} {use_case}",
"setup {technology} {use_case}",
"{technology} tutorial for beginners",
"{technology} documentation",
"{technology} {error_type} troubleshooting",
"{concept} in {technology}",
"migrate from {technology} to {technology2}",
"{action} {concept} {technology}",
# === Personal notes / journals (15% of queries) ===
"meeting notes {topic}",
"ideas for {project}",
"{date} journal entry",
"thoughts on {topic}",
# Research / learning
"{project} {topic} notes",
"{topic} meeting {date}",
"reflect on {topic}",
"brainstorm {project}",
# === Research / learning (20% of queries) ===
"what is {concept}",
"difference between {thing1} and {thing2}",
"{topic} tutorial",
"learn {skill}",
# Short queries
"understand {concept}",
"explain {concept}",
"{topic} fundamentals",
"intro to {skill}",
"{thing1} or {thing2}",
"when to use {concept}",
# === Short / keyword queries (15% of queries) ===
"{keyword}",
"{keyword} {modifier}",
"{keyword} {action}",
"{keyword} {use_case}",
"{technology} {keyword}",
"{concept} {keyword}",
# === Temporal / recency queries (10% of queries) ===
"latest {topic}",
"recent {concept} changes",
"new {technology} features",
"{topic} update {date}",
"what changed in {technology}",
"{technology} changelog {date}",
"{topic} news {date}",
# === Named entities / specific topics (5% of queries) ===
"{named_entity} {topic}",
"{person} {concept}",
"{organization} {use_case}",
"{product} {action}",
]
ACTIONS = ["install", "configure", "setup", "debug", "deploy", "test", "optimize", "migrate"]
TECHNOLOGIES = ["python", "typescript", "react", "docker", "kubernetes", "postgres", "redis", "nginx", "git", "linux"]
CONCEPTS = ["authentication", "caching", "logging", "testing", "deployment", "API", "database", "security"]
USE_CASES = ["production", "development", "CI/CD", "local", "cloud"]
ERROR_TYPES = ["connection", "timeout", "permission", "memory", "syntax"]
TOPICS = ["productivity", "workflow", "architecture", "design", "performance"]
KEYWORDS = ["auth", "config", "setup", "api", "data", "cache", "log", "test"]
MODIFIERS = ["best", "fast", "simple", "advanced", "secure"]
# Category weights for balanced sampling
TEMPLATE_CATEGORIES = {
"technical": list(range(0, 14)), # 0-13
"personal": list(range(14, 22)), # 14-21
"research": list(range(22, 31)), # 22-30
"short": list(range(31, 36)), # 31-35
"temporal": list(range(36, 42)), # 36-41
"entities": list(range(42, 46)), # 42-45
}
ACTIONS = [
"install",
"configure",
"setup",
"debug",
"deploy",
"test",
"optimize",
"migrate",
"build",
"run",
"lint",
"format",
"backup",
"restore",
"update",
"rollback",
"monitor",
"scale",
"secure",
"integrate",
"automate",
"refactor",
"initialize",
]
TECHNOLOGIES = [
# Languages
"python",
"typescript",
"javascript",
"rust",
"golang",
"java",
"kotlin",
"swift",
"ruby",
"php",
"cpp",
"c",
"elixir",
"scala",
"clojure",
"dart",
# Frameworks/Frontend
"react",
"vue",
"angular",
"svelte",
"solid",
"htmx",
"alpine",
"nextjs",
"nuxt",
# Backend
"django",
"flask",
"fastapi",
"express",
"rails",
"spring",
"laravel",
# Infrastructure
"docker",
"kubernetes",
"terraform",
"ansible",
"jenkins",
"github-actions",
# Databases
"postgres",
"mysql",
"mongodb",
"redis",
"elasticsearch",
"sqlite",
"dynamodb",
"cassandra",
"cockroachdb",
"supabase",
"firebase",
# Tools
"git",
"nginx",
"apache",
"linux",
"aws",
"gcp",
"azure",
"vercel",
"netlify",
# Data/ML
"pandas",
"numpy",
"tensorflow",
"pytorch",
"scikit-learn",
"jupyter",
"spark",
"kafka",
"airflow",
"dbt",
]
TECHNOLOGIES_2 = [
"docker",
"kubernetes",
"postgres",
"mysql",
"redis",
"mongodb",
"aws",
"gcp",
"react",
"vue",
"angular",
"python",
"javascript",
"typescript",
"github-actions",
"gitlab-ci",
"jenkins",
"terraform",
"ansible",
]
CONCEPTS = [
"authentication",
"caching",
"logging",
"testing",
"deployment",
"API",
"database",
"security",
"monitoring",
"performance",
"scalability",
"reliability",
"observability",
"microservices",
"serverless",
"virtualization",
"containerization",
"orchestration",
"CI/CD",
"version control",
"dependency injection",
"event sourcing",
"CQRS",
"load balancing",
"rate limiting",
"circuit breaker",
"retry logic",
"idempotency",
]
USE_CASES = [
"production",
"development",
"CI/CD",
"local",
"cloud",
"staging",
"testing",
"microservices",
"serverless",
"hybrid",
"multi-tenant",
"high-availability",
"real-time",
"batch processing",
"stream processing",
"data pipeline",
]
ERROR_TYPES = [
"connection",
"timeout",
"permission",
"memory",
"syntax",
"runtime",
"configuration",
"dependency",
"network",
"authentication",
"authorization",
"validation",
"concurrency",
"deadlock",
"resource",
"quota",
]
TOPICS = [
"productivity",
"workflow",
"architecture",
"design",
"performance",
"security",
"scalability",
"reliability",
"observability",
"maintainability",
"testing",
"documentation",
"refactoring",
"debugging",
"optimization",
"best practices",
"patterns",
"anti-patterns",
"trade-offs",
"decision making",
]
KEYWORDS = [
"auth",
"config",
"setup",
"api",
"cache",
"log",
"test",
"debug",
"env",
"vars",
"secrets",
"tokens",
"headers",
"params",
"query",
"body",
"route",
"middleware",
"handler",
"controller",
"model",
"view",
"template",
"migration",
"seed",
"fixture",
"mock",
"stub",
"spy",
"fake",
"build",
"bundle",
"compile",
"transpile",
"minify",
"optimize",
"deploy",
"release",
"rollback",
"promote",
"freeze",
"thaw",
"pull",
"push",
"commit",
"merge",
"rebase",
"cherry-pick",
"stash",
"up",
"down",
"scale",
"restart",
"reload",
"refresh",
"flush",
"cron",
"queue",
"job",
"worker",
"scheduler",
"trigger",
"webhook",
"alert",
"metric",
"trace",
"span",
"event",
"incident",
"oncall",
]
MODIFIERS = [
"best",
"fast",
"simple",
"advanced",
"secure",
"quick",
"easy",
"proper",
"correct",
"safe",
"efficient",
"reliable",
"robust",
"latest",
"recent",
"new",
"old",
"legacy",
"modern",
"local",
"remote",
"global",
"shared",
"private",
"public",
]
NAMED_ENTITIES = [
"React",
"Vue",
"Angular",
"Docker",
"Kubernetes",
"AWS",
"GCP",
"GitHub",
"GitLab",
"Vercel",
"Netlify",
"Supabase",
"Firebase",
"Stripe",
"Twilio",
"SendGrid",
"Datadog",
"PagerDuty",
"Sentry",
"Terraform",
"Ansible",
"Jenkins",
"CircleCI",
"TravisCI",
]
PERSONS = [
"Kent Beck",
"Martin Fowler",
"Robert Martin",
"Dave Thomas",
"Guido van Rossum",
"Brendan Eich",
"Ryan Dahl",
"Anders Hejlsberg",
"Linus Torvalds",
"DHH",
"Yukihiro Matsumoto",
"Rich Hickey",
]
ORGANIZATIONS = [
"Google",
"Microsoft",
"Amazon",
"Meta",
"Apple",
"Netflix",
"Spotify",
"Stripe",
"Shopify",
"Airbnb",
"Uber",
"Lyft",
"Slack",
"Discord",
]
PRODUCTS = [
"VS Code",
"IntelliJ",
"PyCharm",
"WebStorm",
"DataGrip",
"Postman",
"Insomnia",
"TablePlus",
"Docker Desktop",
"Lens",
"Figma",
"Sketch",
"Notion",
"Linear",
"Jira",
"Trello",
]
SYSTEM_PROMPT = """You are a search query optimization expert for a markdown document search system called QMD.
@ -90,24 +521,72 @@ Query: {query}
Respond with ONLY the lex/vec/hyde lines, nothing else."""
def generate_random_query() -> str:
"""Generate a random query from templates."""
template = random.choice(QUERY_TEMPLATES)
# Category weights - BALANCED approach
# Tech at 15% (reasonable for QMD's technical document use case)
CATEGORY_WEIGHTS = {
"technical": 0.15, # 15% - Technical documentation
"personal": 0.10, # 10% - Personal notes, journals
"research": 0.10, # 10% - Research and learning
"short": 0.15, # 15% - Short keyword queries
"temporal": 0.10, # 10% - Temporal/recency queries (2025/2026)
"entities": 0.05, # 5% - Named entity queries
"health": 0.10, # 10% - Health & wellness
"finance": 0.10, # 10% - Finance & business
"lifestyle": 0.10, # 10% - Home, food, hobbies, travel
"education": 0.05, # 5% - Education & arts
}
def generate_random_query() -> str:
"""Generate a random query from templates with category-weighted sampling."""
# Select category based on weights
categories = list(CATEGORY_WEIGHTS.keys())
weights = list(CATEGORY_WEIGHTS.values())
selected_category = random.choices(categories, weights=weights, k=1)[0]
# Select template from that category
template_idx = random.choice(TEMPLATE_CATEGORIES[selected_category])
template = QUERY_TEMPLATES[template_idx]
# Build replacements based on template type
replacements = {
"{action}": random.choice(ACTIONS),
"{technology}": random.choice(TECHNOLOGIES),
"{technology2}": random.choice(TECHNOLOGIES_2),
"{concept}": random.choice(CONCEPTS),
"{use_case}": random.choice(USE_CASES),
"{error_type}": random.choice(ERROR_TYPES),
"{topic}": random.choice(TOPICS),
"{project}": random.choice(["website", "app", "CLI tool", "API", "library"]),
"{date}": random.choice(["2024-01", "2024-06", "yesterday", "today"]),
"{thing1}": random.choice(CONCEPTS[:4]),
"{thing2}": random.choice(CONCEPTS[4:]),
"{project}": random.choice(
["website", "app", "CLI tool", "API", "library", "service", "platform"]
),
"{date}": random.choice(
# Emphasize 2025/2026 for recency queries (current era)
[
"2026",
"2026",
"2025",
"2025",
"January 2026",
"February 2026",
"March 2026",
"last month",
"this week",
"yesterday",
"today",
"recently",
"latest",
]
),
"{thing1}": random.choice(CONCEPTS[:10]),
"{thing2}": random.choice(CONCEPTS[10:] if len(CONCEPTS) > 10 else CONCEPTS),
"{skill}": random.choice(TECHNOLOGIES),
"{keyword}": random.choice(KEYWORDS),
"{modifier}": random.choice(MODIFIERS),
"{named_entity}": random.choice(NAMED_ENTITIES),
"{person}": random.choice(PERSONS),
"{organization}": random.choice(ORGANIZATIONS),
"{product}": random.choice(PRODUCTS),
}
query = template
@ -126,7 +605,7 @@ def generate_expansion(client: anthropic.Anthropic, query: str) -> str | None:
system=SYSTEM_PROMPT,
messages=[
{"role": "user", "content": USER_PROMPT_TEMPLATE.format(query=query)}
]
],
)
return response.content[0].text.strip()
except Exception as e:
@ -160,10 +639,21 @@ def validate_output(output: str) -> bool:
def main():
parser = argparse.ArgumentParser(description="Generate QMD query expansion training data")
parser.add_argument("--count", type=int, default=100, help="Number of examples to generate")
parser.add_argument("--output", type=str, default="data/qmd_expansion.jsonl", help="Output file path")
parser.add_argument("--queries", type=str, help="Optional file with custom queries (one per line)")
parser = argparse.ArgumentParser(
description="Generate QMD query expansion training data"
)
parser.add_argument(
"--count", type=int, default=100, help="Number of examples to generate"
)
parser.add_argument(
"--output",
type=str,
default="data/qmd_expansion.jsonl",
help="Output file path",
)
parser.add_argument(
"--queries", type=str, help="Optional file with custom queries (one per line)"
)
args = parser.parse_args()
api_key = os.environ.get("ANTHROPIC_API_KEY")

View File

@ -74,7 +74,9 @@ def load_model(model_path: str, base_model: str = None, sft_model: str = None):
print(f"Loading base model {base_model}...", file=sys.stderr)
model = AutoModelForCausalLM.from_pretrained(
base_model, torch_dtype=torch.bfloat16, device_map="auto",
base_model,
torch_dtype=torch.float32,
device_map="auto",
)
if sft_model:
@ -94,16 +96,21 @@ def generate_expansion(model, tokenizer, query: str, max_new_tokens: int = 200)
"""Generate a query expansion using Qwen3 chat template with /no_think."""
import torch
messages = [{"role": "user", "content": f"/no_think Expand this search query: {query}"}]
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
messages = [
{"role": "user", "content": f"/no_think Expand this search query: {query}"}
]
prompt = tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
with torch.no_grad():
outputs = model.generate(
**inputs,
max_new_tokens=max_new_tokens,
temperature=0.7,
temperature=0.1,
do_sample=True,
top_p=0.9,
pad_token_id=tokenizer.pad_token_id,
eos_token_id=tokenizer.eos_token_id,
)
@ -116,27 +123,32 @@ def generate_expansion(model, tokenizer, query: str, max_new_tokens: int = 200)
elif "assistant\n" in full_output:
expansion = full_output.split("assistant\n")[-1].strip()
else:
expansion = full_output[len(prompt):].strip()
expansion = full_output[len(prompt) :].strip()
# Strip leftover <think> blocks
import re
if "<think>" in expansion:
expansion = re.sub(r'<think>.*?</think>', '', expansion, flags=re.DOTALL).strip()
expansion = re.sub(
r"<think>.*?</think>", "", expansion, flags=re.DOTALL
).strip()
return expansion
def print_result(query: str, expansion: str, scores: dict, verbose: bool = False):
"""Print a single scored result."""
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Query: {query}")
print(f"{'~'*60}")
print(f"{'~' * 60}")
print(expansion)
print(f"{'~'*60}")
print(f"{'~' * 60}")
print(f"Score: {scores['percentage']:.0f}% ({scores['rating']})")
print(f" Format: {scores['format']}/30 Diversity: {scores['diversity']}/30 "
f"Hyde: {scores['hyde']}/20 Quality: {scores['quality']}/20 "
f"Entity: {scores['entity']}/20 Think: {scores['think_bonus']}/20")
print(
f" Format: {scores['format']}/30 Diversity: {scores['diversity']}/30 "
f"Hyde: {scores['hyde']}/20 Quality: {scores['quality']}/20 "
f"Entity: {scores['entity']}/20 Think: {scores['think_bonus']}/20"
)
if verbose and scores["deductions"]:
print(f" Issues: {', '.join(scores['deductions'][:5])}")
if verbose and scores["entities_detected"]:
@ -145,11 +157,13 @@ def print_result(query: str, expansion: str, scores: dict, verbose: bool = False
def print_summary(scored_results: list):
"""Print aggregate summary."""
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print("SUMMARY")
print(f"{'='*60}")
print(f"{'=' * 60}")
avg_score = sum(r["scores"]["percentage"] for r in scored_results) / len(scored_results)
avg_score = sum(r["scores"]["percentage"] for r in scored_results) / len(
scored_results
)
ratings = Counter(r["scores"]["rating"] for r in scored_results)
print(f" Total queries: {len(scored_results)}")
@ -176,13 +190,19 @@ def cmd_generate_and_score(args):
if not args.summary_only:
print_result(query, expansion, scores, args.verbose)
scored_results.append({
"query": query,
"expansion": expansion,
"scores": {k: v for k, v in scores.items() if k not in ("parsed", "deductions", "entities_detected")},
"deductions": scores["deductions"],
"entities_detected": scores["entities_detected"],
})
scored_results.append(
{
"query": query,
"expansion": expansion,
"scores": {
k: v
for k, v in scores.items()
if k not in ("parsed", "deductions", "entities_detected")
},
"deductions": scores["deductions"],
"entities_detected": scores["entities_detected"],
}
)
print_summary(scored_results)
@ -191,7 +211,11 @@ def cmd_generate_and_score(args):
"metadata": {"model": args.model, "timestamp": datetime.now().isoformat()},
"summary": {
"total": len(scored_results),
"average_score": round(sum(r["scores"]["percentage"] for r in scored_results) / len(scored_results), 1),
"average_score": round(
sum(r["scores"]["percentage"] for r in scored_results)
/ len(scored_results),
1,
),
},
"results": scored_results,
}
@ -218,13 +242,19 @@ def cmd_score_only(args):
if not args.summary_only:
print_result(query, expansion, scores, args.verbose)
scored_results.append({
"query": query,
"expansion": expansion,
"scores": {k: v for k, v in scores.items() if k not in ("parsed", "deductions", "entities_detected")},
"deductions": scores["deductions"],
"entities_detected": scores["entities_detected"],
})
scored_results.append(
{
"query": query,
"expansion": expansion,
"scores": {
k: v
for k, v in scores.items()
if k not in ("parsed", "deductions", "entities_detected")
},
"deductions": scores["deductions"],
"entities_detected": scores["entities_detected"],
}
)
print_summary(scored_results)
@ -244,13 +274,25 @@ Examples:
# Model evaluation mode
parser.add_argument("--model", help="Model path (HF Hub or local)")
parser.add_argument("--base-model", default=None, help="Base model for tokenizer (default: Qwen/Qwen3-1.7B)")
parser.add_argument("--sft-model", default=None, help="SFT adapter to merge first (for GRPO models)")
parser.add_argument(
"--base-model",
default=None,
help="Base model for tokenizer (default: Qwen/Qwen3-1.7B)",
)
parser.add_argument(
"--sft-model", default=None, help="SFT adapter to merge first (for GRPO models)"
)
parser.add_argument("--queries", default="evals/queries.txt", help="Queries file")
parser.add_argument("--max-tokens", type=int, default=200, help="Max tokens per generation")
parser.add_argument(
"--max-tokens", type=int, default=200, help="Max tokens per generation"
)
# Score-only mode
parser.add_argument("--score-only", metavar="JSONL", help="Score existing JSONL file instead of generating")
parser.add_argument(
"--score-only",
metavar="JSONL",
help="Score existing JSONL file instead of generating",
)
# Output options
parser.add_argument("--output", "-o", help="Save detailed scores to JSON file")

View File

@ -0,0 +1,214 @@
# /// script
# requires-python = ">=3.10"
# dependencies = []
# ///
"""
Generate 'only:' variant training data from high-quality expansions.
Takes existing training data and creates derivative examples where the query
ends with 'only: lex', 'only: hyde', or 'only: vec', and the output contains
ONLY that component type.
Usage:
uv run generate_only_variants.py data/qmd_expansion_handcrafted.jsonl
uv run generate_only_variants.py data/qmd_expansion_handcrafted.jsonl -o data/qmd_only_variants.jsonl
uv run generate_only_variants.py data/*.jsonl --combine # combine all inputs
"""
import argparse
import json
import sys
from pathlib import Path
def parse_expansion(text: str) -> dict:
"""Parse a multi-line expansion into {lex, vec, hyde} lists."""
result = {"lex": [], "vec": [], "hyde": []}
for line in text.strip().split("\n"):
line = line.strip()
if not line:
continue
if line.startswith("lex:"):
result["lex"].append(line[4:].strip())
elif line.startswith("vec:"):
result["vec"].append(line[4:].strip())
elif line.startswith("hyde:"):
result["hyde"].append(line[5:].strip())
return result
# Templated patterns to filter out from hyde (low quality)
TEMPLATED_PATTERNS = [
"This comprehensive guide covers",
"This comprehensive guide to",
"requires practice and patience",
"This resource provides",
"Follow the steps carefully",
"covers all the essential information",
"includes practical examples, best practices",
]
def is_templated_hyde(hyde_text: str) -> bool:
"""Check if a hyde output is a low-quality templated response."""
return any(pattern in hyde_text for pattern in TEMPLATED_PATTERNS)
def format_output(parsed: dict, only_type: str) -> str | None:
"""Format output for a single type. Returns None if type is empty or low quality."""
items = parsed.get(only_type, [])
if not items:
return None
# Filter out templated hyde outputs
if only_type == "hyde":
filtered = [item for item in items if not is_templated_hyde(item)]
if not filtered:
return None
items = filtered
lines = []
for item in items:
lines.append(f"{only_type}: {item}")
return "\n".join(lines)
def generate_only_variants(input_query: str, output: str) -> list[dict]:
"""Generate all valid 'only:' variants from a single example."""
variants = []
parsed = parse_expansion(output)
for only_type in ["lex", "vec", "hyde"]:
formatted = format_output(parsed, only_type)
if formatted:
# Add the '/only:' suffix to the query (slash prefix)
new_query = f"{input_query} /only:{only_type}"
variants.append({
"input": new_query,
"output": formatted,
"_source_type": only_type,
"_source_query": input_query,
})
return variants
def process_file(input_path: Path) -> list[dict]:
"""Process a single JSONL file and return all 'only:' variants."""
variants = []
seen_queries = set()
with open(input_path) as f:
for line_num, line in enumerate(f, 1):
line = line.strip()
if not line:
continue
try:
data = json.loads(line)
except json.JSONDecodeError as e:
print(f" Warning: Skipping line {line_num} (invalid JSON): {e}", file=sys.stderr)
continue
# Skip metadata lines
if data.get("_meta"):
continue
input_query = data.get("input", "")
output = data.get("output", "")
if not input_query or not output:
continue
# Skip if query already has '/only:' suffix
if " /only:" in input_query.lower():
continue
# Skip duplicates
if input_query in seen_queries:
continue
seen_queries.add(input_query)
# Generate variants
for variant in generate_only_variants(input_query, output):
variants.append(variant)
return variants
def main():
parser = argparse.ArgumentParser(
description="Generate 'only:' variant training data from high-quality expansions",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"input_files",
nargs="+",
help="Input JSONL files with training data",
)
parser.add_argument(
"-o", "--output",
default="data/qmd_only_variants.jsonl",
help="Output JSONL file (default: data/qmd_only_variants.jsonl)",
)
parser.add_argument(
"--combine",
action="store_true",
help="Combine all input files into one output",
)
parser.add_argument(
"--stats",
action="store_true",
help="Print statistics about generated variants",
)
args = parser.parse_args()
all_variants = []
stats = {"lex": 0, "vec": 0, "hyde": 0}
for input_file in args.input_files:
input_path = Path(input_file)
if not input_path.exists():
print(f"Warning: {input_file} not found, skipping", file=sys.stderr)
continue
print(f"Processing {input_path.name}...", file=sys.stderr)
variants = process_file(input_path)
for v in variants:
stats[v["_source_type"]] += 1
if args.combine:
all_variants.extend(variants)
else:
# Write to separate output files per input
output_path = input_path.parent / f"{input_path.stem}_only.jsonl"
with open(output_path, "w") as f:
for variant in variants:
# Remove internal fields before writing
clean = {"input": variant["input"], "output": variant["output"]}
f.write(json.dumps(clean) + "\n")
print(f" -> {len(variants)} variants written to {output_path}", file=sys.stderr)
if args.combine and all_variants:
output_path = Path(args.output)
output_path.parent.mkdir(parents=True, exist_ok=True)
with open(output_path, "w") as f:
for variant in all_variants:
clean = {"input": variant["input"], "output": variant["output"]}
f.write(json.dumps(clean) + "\n")
print(f"\nTotal: {len(all_variants)} variants written to {output_path}", file=sys.stderr)
if args.stats or args.combine:
print(f"\nStats:", file=sys.stderr)
print(f" lex: {stats['lex']}", file=sys.stderr)
print(f" vec: {stats['vec']}", file=sys.stderr)
print(f" hyde: {stats['hyde']}", file=sys.stderr)
print(f" total: {sum(stats.values())}", file=sys.stderr)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,107 @@
# /// script
# requires-python = ">=3.10"
# dependencies = []
# ///
"""Prepare v4 dataset: high-quality expansions + /only: variants."""
import json
import random
from pathlib import Path
def to_chat_format(query: str, output: str) -> dict:
"""Convert input/output to chat format with /no_think."""
# For /only: queries, keep the suffix in the prompt
prompt = f"/no_think Expand this search query: {query}"
text = f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n{output}<|im_end|>\n"
messages = [
{"role": "user", "content": prompt},
{"role": "assistant", "content": output}
]
return {"text": text, "messages": messages}
def load_jsonl(path: Path) -> list[dict]:
"""Load JSONL file."""
data = []
with open(path) as f:
for line in f:
line = line.strip()
if line:
data.append(json.loads(line))
return data
def main():
data_dir = Path("data")
# High-quality sources
sources = [
("qmd_expansion_v2.jsonl", "v2"),
("qmd_expansion_handcrafted.jsonl", "handcrafted"),
("qmd_only_variants.jsonl", "only"),
]
all_examples = []
stats = {}
for filename, label in sources:
path = data_dir / filename
if not path.exists():
print(f" Skipping {filename} (not found)")
continue
raw = load_jsonl(path)
converted = []
for item in raw:
query = item.get("input", "")
output = item.get("output", "")
if query and output:
converted.append(to_chat_format(query, output))
all_examples.extend(converted)
stats[label] = len(converted)
print(f" {label}: {len(converted)} examples")
# Shuffle
random.seed(42)
random.shuffle(all_examples)
# Split 90/10
split_idx = int(len(all_examples) * 0.9)
train = all_examples[:split_idx]
val = all_examples[split_idx:]
# Write output
out_dir = data_dir / "train_v4"
out_dir.mkdir(exist_ok=True)
with open(out_dir / "train.jsonl", "w") as f:
for ex in train:
f.write(json.dumps(ex) + "\n")
with open(out_dir / "val.jsonl", "w") as f:
for ex in val:
f.write(json.dumps(ex) + "\n")
# Dataset info
info = {
"dataset_name": "qmd-query-expansion-v4",
"train_samples": len(train),
"val_samples": len(val),
"sources": stats,
}
with open(out_dir / "dataset_info.json", "w") as f:
json.dump(info, f, indent=2)
print(f"\n✓ Dataset prepared in {out_dir}/")
print(f" Train: {len(train)}")
print(f" Val: {len(val)}")
print(f" Total: {len(all_examples)}")
if __name__ == "__main__":
main()

View File

@ -26,6 +26,10 @@ from collections import Counter
# Constants
# =============================================================================
# "only:" mode patterns - when query ends with these, expect only that type
# Format: "query /only:lex" (slash prefix, no space after colon)
ONLY_MODE_PATTERN = re.compile(r'\s+/only:(lex|vec|hyde)\s*$', re.IGNORECASE)
STOPWORDS = frozenset({
'the', 'a', 'an', 'is', 'are', 'to', 'for', 'of', 'in',
'and', 'or', 'it', 'this', 'that', 'be', 'with', 'as', 'on', 'by',
@ -72,6 +76,19 @@ def parse_expansion(text: str) -> dict:
return result
def detect_only_mode(query: str) -> tuple[str | None, str]:
"""Detect if query ends with 'only: lex/vec/hyde'.
Returns (only_type, base_query) where only_type is None for normal queries.
"""
match = ONLY_MODE_PATTERN.search(query)
if match:
only_type = match.group(1).lower()
base_query = query[:match.start()].strip()
return only_type, base_query
return None, query
def clean_model_output(text: str) -> tuple[str, bool]:
"""Strip chat template artifacts from model output.
@ -195,11 +212,149 @@ def word_repetition_penalty(text: str) -> int:
# Scoring
# =============================================================================
def _score_only_mode(query: str, base_query: str, text: str, used_thinking: bool, only_type: str) -> dict:
"""Score an 'only:' mode expansion. Expects ONLY the requested type."""
parsed = parse_expansion(text)
deductions = []
# Expected type must be present
expected_items = parsed.get(only_type, [])
if not expected_items:
return {
"format": 0, "diversity": 0, "hyde": 0, "quality": 0, "entity": 0,
"think_bonus": 0, "total": 0, "max_possible": 100,
"percentage": 0.0, "rating": "Failed",
"deductions": [f"missing expected {only_type}: output"],
"parsed": parsed,
"entities_detected": [],
"only_mode": only_type,
}
# Penalize presence of OTHER types
other_types = {"lex", "vec", "hyde"} - {only_type}
unwanted_count = sum(len(parsed.get(t, [])) for t in other_types)
if unwanted_count > 0:
deductions.append(f"contains unwanted types (expected only {only_type})")
# --- Format (0-30) ---
format_score = 30 if unwanted_count == 0 else max(0, 30 - unwanted_count * 10)
# --- Diversity (0-30) ---
diversity_score = 0
if len(expected_items) >= 2:
diversity_score += 15
# Check for diversity among items
div_score = 15
for i, a in enumerate(expected_items):
for b in expected_items[i+1:]:
if not is_diverse(a, b, 2):
div_score -= 5
deductions.append(f"{only_type} duplicate: {a[:20]}...")
diversity_score += max(0, div_score)
elif len(expected_items) == 1:
diversity_score = 15 # One item is fine for single-type output
# Check for echoes
for exp in expected_items:
if echoes_query(exp, base_query):
diversity_score -= 5
deductions.append(f"echoes query: {exp[:20]}...")
diversity_score = max(0, diversity_score)
# --- Type-specific quality (0-20) ---
quality_score = 10 # base
entities = extract_named_entities(base_query)
if only_type == "lex":
# Lex should be short keyword phrases with key terms
with_terms = sum(1 for l in expected_items if lex_preserves_key_terms(l, base_query))
if with_terms == len(expected_items):
quality_score += 5
# Check for generic phrases
generic = sum(1 for l in expected_items if lex_is_generic(l))
if generic == 0:
quality_score += 5
else:
deductions.append(f"{generic} generic lex phrases")
elif only_type == "vec":
# Vec should be natural language sentences
natural = sum(1 for v in expected_items if " " in v and len(v) > 15)
if natural == len(expected_items):
quality_score += 10
else:
quality_score += 5
deductions.append("vec not all natural language")
elif only_type == "hyde":
# Hyde should be a document snippet (50-200 chars)
hyde_text = expected_items[0]
hyde_len = len(hyde_text)
if 50 <= hyde_len <= 200:
quality_score += 10
elif 30 <= hyde_len <= 300:
quality_score += 5
deductions.append(f"hyde length {hyde_len} (ideal: 50-200)")
else:
deductions.append(f"hyde length {hyde_len} out of range")
# --- Entity preservation (0-20) ---
entity_score = 10 # base
if entities:
with_entities = sum(1 for item in expected_items if lex_preserves_entities(item, entities))
if with_entities == len(expected_items):
entity_score += 10
elif with_entities > 0:
entity_score += 5
else:
entity_score = 0
deductions.append(f"missing entities: {entities}")
# --- Think bonus (0-20) ---
think_bonus = 0 if used_thinking else 20
# --- Total ---
total = format_score + diversity_score + quality_score + entity_score + think_bonus
max_possible = 120
percentage = max(0.0, min(100.0, total / max_possible * 100))
if percentage >= 80:
rating = "Excellent"
elif percentage >= 60:
rating = "Good"
elif percentage >= 40:
rating = "Acceptable"
elif percentage >= 20:
rating = "Poor"
else:
rating = "Failed"
return {
"format": format_score,
"diversity": diversity_score,
"hyde": 0, # not used in only mode (quality covers it)
"quality": quality_score,
"entity": entity_score,
"think_bonus": think_bonus,
"total": total,
"max_possible": max_possible,
"percentage": round(percentage, 1),
"rating": rating,
"deductions": deductions,
"parsed": parsed,
"entities_detected": list(entities) if entities else [],
"only_mode": only_type,
}
def score_expansion_detailed(query: str, expansion: str) -> dict:
"""Score an expansion with full breakdown. Returns dict with all dimensions."""
text, used_thinking = clean_model_output(expansion.strip())
deductions = []
# Detect "only:" mode
only_type, base_query = detect_only_mode(query)
def _fail(reason):
return {
"format": 0, "diversity": 0, "hyde": 0, "quality": 0, "entity": 0,
@ -208,6 +363,7 @@ def score_expansion_detailed(query: str, expansion: str) -> dict:
"deductions": [reason],
"parsed": parse_expansion(expansion),
"entities_detected": [],
"only_mode": only_type,
}
# Hard fail: remaining chat template tokens
@ -220,6 +376,10 @@ def score_expansion_detailed(query: str, expansion: str) -> dict:
if line and not line.startswith(("lex:", "vec:", "hyde:")):
return _fail(f"INVALID LINE: {line[:50]}")
# --- Handle "only:" mode separately ---
if only_type:
return _score_only_mode(query, base_query, text, used_thinking, only_type)
parsed = parse_expansion(text)
# --- Format (0-30) ---
@ -364,6 +524,7 @@ def score_expansion_detailed(query: str, expansion: str) -> dict:
"deductions": deductions,
"parsed": parsed,
"entities_detected": list(entities) if entities else [],
"only_mode": None,
}
@ -417,12 +578,19 @@ if __name__ == "__main__":
("how to use React hooks", "lex: React hooks tutorial\nlex: useEffect useState\nvec: how to use React hooks in functional components"),
("auth", "<think>Let me think...</think>\nlex: auth"),
("auth", "lex: auth\nThis is some explanation\nvec: more"),
# "/only:" mode tests (slash prefix)
("auth /only:lex", "lex: auth setup\nlex: authentication config\nlex: login credentials"),
("auth /only:lex", "lex: auth setup\nvec: how to configure authentication"), # should fail - has vec
("React hooks /only:vec", "vec: how to use React hooks in functional components\nvec: useState and useEffect patterns in React"),
("PostgreSQL indexing /only:hyde", "hyde: PostgreSQL uses B-tree indexes by default. Create indexes with CREATE INDEX idx_name ON table(column). EXPLAIN ANALYZE shows whether queries use indexes efficiently."),
]
for query, expansion in tests:
score = score_expansion(query, expansion)
detail = score_expansion_detailed(query, expansion)
print(f"\n Query: '{query}'")
only_mode = detail.get("only_mode")
mode_str = f" [only:{only_mode}]" if only_mode else ""
print(f"\n Query: '{query}'{mode_str}")
print(f" Score: {score:.2f} ({detail['rating']})")
if detail["deductions"]:
print(f" Issues: {', '.join(detail['deductions'][:3])}")

View File

@ -1,6 +1,7 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "torch",
# "trl>=0.12.0",
# "peft>=0.7.0",
# "transformers>=4.45.0",
@ -34,7 +35,6 @@ import yaml
def cmd_sft(args):
"""Run supervised fine-tuning."""
import trackio
from datasets import load_dataset
from peft import LoraConfig
from trl import SFTTrainer, SFTConfig
@ -47,8 +47,20 @@ def cmd_sft(args):
print(yaml.dump(cfg, default_flow_style=False))
return
print(f"Loading dataset: {cfg['dataset']['name']}...")
dataset = load_dataset(cfg["dataset"]["name"], split=cfg["dataset"]["split"])
dataset_name = cfg["dataset"]["name"]
print(f"Loading dataset: {dataset_name}...")
# Support local JSONL files
if dataset_name.startswith("data/") or dataset_name.endswith(".jsonl"):
from pathlib import Path
data_path = Path(dataset_name)
if data_path.is_dir():
train_file = data_path / "train.jsonl"
dataset = load_dataset("json", data_files=str(train_file), split="train")
else:
dataset = load_dataset("json", data_files=dataset_name, split="train")
else:
dataset = load_dataset(dataset_name, split=cfg["dataset"]["split"])
print(f"Dataset loaded: {len(dataset)} examples")
split = dataset.train_test_split(test_size=cfg["dataset"]["eval_split"], seed=42)
@ -56,11 +68,15 @@ def cmd_sft(args):
eval_dataset = split["test"]
print(f" Train: {len(train_dataset)}, Eval: {len(eval_dataset)}")
# Check if output looks like a HF Hub path (contains /)
output_name = cfg["model"]["output"]
push_to_hub = "/" in output_name
config = SFTConfig(
output_dir=cfg["model"]["output"].split("/")[-1],
push_to_hub=True,
hub_model_id=cfg["model"]["output"],
hub_strategy="every_save",
output_dir=output_name.split("/")[-1] if push_to_hub else output_name,
push_to_hub=push_to_hub,
hub_model_id=output_name if push_to_hub else None,
hub_strategy="every_save" if push_to_hub else "end",
num_train_epochs=cfg["training"]["epochs"],
per_device_train_batch_size=cfg["training"]["batch_size"],
@ -78,9 +94,7 @@ def cmd_sft(args):
warmup_ratio=cfg["training"]["warmup_ratio"],
lr_scheduler_type=cfg["training"]["lr_scheduler"],
report_to="trackio",
project=cfg["tracking"]["project"],
run_name=cfg["tracking"]["run_name"],
report_to="none", # Disable tracking for local training
)
peft_config = LoraConfig(
@ -104,10 +118,13 @@ def cmd_sft(args):
print("Starting SFT training...")
trainer.train()
print("Pushing to Hub...")
trainer.push_to_hub()
trackio.finish()
print(f"Done! Model: https://huggingface.co/{cfg['model']['output']}")
if push_to_hub:
print("Pushing to Hub...")
trainer.push_to_hub()
print(f"Done! Model: https://huggingface.co/{output_name}")
else:
trainer.save_model()
print(f"Done! Model saved to: {output_name}")
def cmd_grpo(args):

61
flake.lock generated Normal file
View File

@ -0,0 +1,61 @@
{
"nodes": {
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1769188852,
"narHash": "sha256-aBAGyMum27K7cP5OR7BMioJOF3icquJMZDDgk6ZEg1A=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "a1bab9e494f5f4939442a57a58d0449a109593fe",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

View File

@ -11,6 +11,13 @@
let
pkgs = nixpkgs.legacyPackages.${system};
# SQLite with loadable extension support for sqlite-vec
sqliteWithExtensions = pkgs.sqlite.overrideAttrs (old: {
configureFlags = (old.configureFlags or []) ++ [
"--enable-load-extension"
];
});
qmd = pkgs.stdenv.mkDerivation {
pname = "qmd";
version = "1.0.0";
@ -62,10 +69,11 @@
devShells.default = pkgs.mkShell {
buildInputs = [
pkgs.bun
pkgs.sqlite
sqliteWithExtensions
];
shellHook = ''
export BREW_PREFIX="''${BREW_PREFIX:-${sqliteWithExtensions.out}}"
echo "QMD development shell"
echo "Run: bun src/qmd.ts <command>"
'';

160
skills/qmd/SKILL.md Normal file
View File

@ -0,0 +1,160 @@
---
name: qmd
description: Search personal markdown knowledge bases, notes, meeting transcripts, and documentation using QMD - a local hybrid search engine. Combines BM25 keyword search, vector semantic search, and LLM re-ranking. Use when users ask to search notes, find documents, look up information in their knowledge base, retrieve meeting notes, or search documentation. Triggers on "search my notes", "find in docs", "look up", "what did I write about", "meeting notes about".
license: MIT
compatibility: Requires qmd installed via `bun install -g https://github.com/tobi/qmd`. Works with Claude Code CLI and MCP-compatible agents.
metadata:
author: tobi
version: "1.0"
allowed-tools: Bash(qmd:*)
---
# QMD - Quick Markdown Search
QMD is a local, on-device search engine for markdown content. It indexes your notes, meeting transcripts, documentation, and knowledge bases for fast retrieval.
## When to Use This Skill
- User asks to search their notes, documents, or knowledge base
- User needs to find information in their markdown files
- User wants to retrieve specific documents or search across collections
- User asks "what did I write about X" or "find my notes on Y"
- User needs semantic search (conceptual similarity) not just keyword matching
- User mentions meeting notes, transcripts, or documentation lookup
## Search Commands
Choose the right search mode for the task:
| Command | Use When | Speed |
|---------|----------|-------|
| `qmd search` | Exact keyword matches needed | Fast |
| `qmd vsearch` | Keywords aren't working, need conceptual matches | Medium |
| `qmd query` | Best results needed, speed not critical | Slower |
```sh
# Fast keyword search (BM25)
qmd search "your query"
# Semantic vector search (finds conceptually similar content)
qmd vsearch "your query"
# Hybrid search with re-ranking (best quality)
qmd query "your query"
```
## Common Options
```sh
-n <num> # Number of results (default: 5)
-c, --collection <name> # Restrict to specific collection
--all # Return all matches
--min-score <num> # Minimum score threshold (0.0-1.0)
--full # Show full document content
--json # JSON output for processing
--files # List files with scores
--line-numbers # Add line numbers to output
```
## Document Retrieval
```sh
# Get document by path
qmd get "collection/path/to/doc.md"
# Get document by docid (shown in search results as #abc123)
qmd get "#abc123"
# Get with line numbers for code review
qmd get "docs/api.md" --line-numbers
# Get multiple documents by glob pattern
qmd multi-get "docs/*.md"
# Get multiple documents by list
qmd multi-get "doc1.md, doc2.md, #abc123"
```
## Index Management
```sh
# Check index status and available collections
qmd status
# List all collections
qmd collection list
# List files in a collection
qmd ls <collection-name>
# Update index (re-scan files for changes)
qmd update
```
## Score Interpretation
| Score | Meaning | Action |
|-------|---------|--------|
| 0.8 - 1.0 | Highly relevant | Show to user |
| 0.5 - 0.8 | Moderately relevant | Include if few results |
| 0.2 - 0.5 | Somewhat relevant | Only if user wants more |
| 0.0 - 0.2 | Low relevance | Usually skip |
## Recommended Workflow
1. **Check what's available**: `qmd status`
2. **Start with keyword search**: `qmd search "topic" -n 10`
3. **Try semantic if needed**: `qmd vsearch "describe the concept"`
4. **Use hybrid for best results**: `qmd query "question" --min-score 0.4`
5. **Retrieve full documents**: `qmd get "#docid" --full`
## Example: Finding Meeting Notes
```sh
# Search for meetings about a topic
qmd search "quarterly review" -c meetings -n 5
# Get semantic matches
qmd vsearch "performance discussion" -c meetings
# Retrieve the full meeting notes
qmd get "#abc123" --full
```
## Example: Research Across All Notes
```sh
# Hybrid search for best results
qmd query "authentication implementation" --min-score 0.3 --json
# Get all relevant files for deeper analysis
qmd query "auth flow" --all --files --min-score 0.4
```
## MCP Server Integration
QMD also works as an MCP server, providing these tools directly:
| MCP Tool | Equivalent CLI | Purpose |
|----------|---------------|---------|
| `qmd_search` | `qmd search` | Fast BM25 keyword search |
| `qmd_vsearch` | `qmd vsearch` | Semantic vector search |
| `qmd_query` | `qmd query` | Hybrid search with reranking |
| `qmd_get` | `qmd get` | Retrieve document by path or docid |
| `qmd_multi_get` | `qmd multi-get` | Retrieve multiple documents |
| `qmd_status` | `qmd status` | Index health and collection info |
To enable MCP tools, add to `~/.claude/settings.json`:
```json
{
"mcpServers": {
"qmd": {
"command": "qmd",
"args": ["mcp"]
}
}
}
```
When MCP is configured, prefer using the `qmd_*` tools directly instead of Bash commands for better integration.

View File

@ -0,0 +1,128 @@
# QMD MCP Server Setup
Detailed instructions for configuring QMD as an MCP (Model Context Protocol) server.
## Prerequisites
1. Install qmd globally:
```sh
bun install -g https://github.com/tobi/qmd
```
2. Verify installation:
```sh
qmd --help
```
3. Set up at least one collection:
```sh
qmd collection add ~/Documents/notes --name notes
qmd embed # Generate vector embeddings
```
## Claude Code Configuration
Add to `~/.claude/settings.json`:
```json
{
"mcpServers": {
"qmd": {
"command": "qmd",
"args": ["mcp"]
}
}
}
```
## Claude Desktop Configuration
Add to `~/Library/Application Support/Claude/claude_desktop_config.json`:
```json
{
"mcpServers": {
"qmd": {
"command": "qmd",
"args": ["mcp"]
}
}
}
```
## Available MCP Tools
Once configured, these tools become available:
### qmd_search
Fast BM25 keyword search.
**Parameters:**
- `query` (required): Search query string
- `collection` (optional): Restrict to specific collection
- `limit` (optional): Number of results (default: 5)
- `minScore` (optional): Minimum relevance score
### qmd_vsearch
Semantic vector search for conceptual similarity.
**Parameters:**
- `query` (required): Search query string
- `collection` (optional): Restrict to specific collection
- `limit` (optional): Number of results (default: 5)
- `minScore` (optional): Minimum relevance score
### qmd_query
Hybrid search combining BM25, vector search, and LLM re-ranking.
**Parameters:**
- `query` (required): Search query string
- `collection` (optional): Restrict to specific collection
- `limit` (optional): Number of results (default: 5)
- `minScore` (optional): Minimum relevance score
### qmd_get
Retrieve a document by path or docid.
**Parameters:**
- `path` (required): Document path or docid (e.g., `#abc123`)
- `full` (optional): Return full content (default: true)
- `lineNumbers` (optional): Include line numbers
### qmd_multi_get
Retrieve multiple documents.
**Parameters:**
- `pattern` (required): Glob pattern or comma-separated list
- `maxBytes` (optional): Skip files larger than this (default: 10KB)
### qmd_status
Get index health and collection information.
**Parameters:** None
## Troubleshooting
### MCP server not starting
- Ensure qmd is in your PATH: `which qmd`
- Try running `qmd mcp` manually to see errors
- Check that Bun is installed: `bun --version`
### No results returned
- Verify collections exist: `qmd collection list`
- Check index status: `qmd status`
- Ensure embeddings are generated: `qmd embed`
### Slow searches
- For faster results, use `qmd_search` instead of `qmd_query`
- The first search may be slow while models load (~3GB)
- Subsequent searches are much faster
## Choosing Between CLI and MCP
| Scenario | Recommendation |
|----------|---------------|
| MCP configured | Use `qmd_*` tools directly |
| No MCP | Use Bash with `qmd` commands |
| Complex pipelines | Bash may be more flexible |
| Simple lookups | MCP tools are cleaner |

View File

@ -12,7 +12,11 @@ import {
LlamaCpp,
getDefaultLlamaCpp,
disposeDefaultLlamaCpp,
withLLMSession,
canUnloadLLM,
SessionReleasedError,
type RerankDocument,
type ILLMSession,
} from "./llm.js";
// =============================================================================
@ -167,6 +171,63 @@ describe("LlamaCpp Integration", () => {
// Performance is machine/load dependent. We only assert batch isn't drastically worse.
expect(batchTime).toBeLessThanOrEqual(seqTime * 3);
});
test("handles concurrent embedBatch calls on fresh instance without race condition", async () => {
// This test verifies the fix for a race condition where concurrent calls to
// ensureEmbedContext() could create multiple contexts. Without the promise guard,
// each concurrent embedBatch call sees embedContext === null and creates its own
// context, causing resource leaks and potential "Context is disposed" errors.
//
// See: https://github.com/tobi/qmd/pull/54
//
// The fix uses a promise guard to ensure only one context creation runs at a time.
// We verify this by instrumenting createEmbeddingContext to count invocations.
const freshLlm = new LlamaCpp({});
let contextCreateCount = 0;
// Instrument the model's createEmbeddingContext to count calls
const originalEnsureEmbedModel = (freshLlm as any).ensureEmbedModel.bind(freshLlm);
let modelInstrumented = false;
(freshLlm as any).ensureEmbedModel = async function() {
const model = await originalEnsureEmbedModel();
if (!modelInstrumented) {
modelInstrumented = true;
const originalCreate = model.createEmbeddingContext.bind(model);
model.createEmbeddingContext = async function(...args: any[]) {
contextCreateCount++;
return originalCreate(...args);
};
}
return model;
};
const texts = Array(10).fill(null).map((_, i) => `Document ${i}`);
// Call embedBatch 5 TIMES in parallel on fresh instance.
// Without the promise guard fix, this would create 5 contexts (one per call).
// With the fix, only 1 context should be created.
const batches = await Promise.all([
freshLlm.embedBatch(texts.slice(0, 2)),
freshLlm.embedBatch(texts.slice(2, 4)),
freshLlm.embedBatch(texts.slice(4, 6)),
freshLlm.embedBatch(texts.slice(6, 8)),
freshLlm.embedBatch(texts.slice(8, 10)),
]);
const allResults = batches.flat();
expect(allResults).toHaveLength(10);
const successCount = allResults.filter(r => r !== null).length;
expect(successCount).toBe(10);
// THE KEY ASSERTION: Only 1 context should be created, not 5
// Without the fix, contextCreateCount would be 5 (one per concurrent embedBatch call)
console.log(`Context creation count: ${contextCreateCount} (expected: 1)`);
expect(contextCreateCount).toBe(1);
await freshLlm.dispose();
}, 60000);
});
describe("rerank", () => {
@ -325,3 +386,174 @@ describe("LlamaCpp Integration", () => {
});
});
// =============================================================================
// Session Management Tests
// =============================================================================
describe("LLM Session Management", () => {
describe("withLLMSession", () => {
test("session provides access to LLM operations", async () => {
const result = await withLLMSession(async (session) => {
expect(session.isValid).toBe(true);
const embedding = await session.embed("test text");
expect(embedding).not.toBeNull();
expect(embedding!.embedding.length).toBe(768);
return "success";
});
expect(result).toBe("success");
});
test("session is invalid after release", async () => {
let capturedSession: ILLMSession | null = null;
await withLLMSession(async (session) => {
capturedSession = session;
expect(session.isValid).toBe(true);
});
// Session should be invalid after withLLMSession returns
expect(capturedSession).not.toBeNull();
expect(capturedSession!.isValid).toBe(false);
});
test("session prevents idle unload during operations", async () => {
await withLLMSession(async (session) => {
// While inside a session, canUnloadLLM should return false
expect(canUnloadLLM()).toBe(false);
// Perform an operation
await session.embed("test");
// Still should not be able to unload
expect(canUnloadLLM()).toBe(false);
});
// After session ends, should be able to unload
expect(canUnloadLLM()).toBe(true);
});
test("nested sessions increment ref count", async () => {
await withLLMSession(async (outerSession) => {
expect(canUnloadLLM()).toBe(false);
await withLLMSession(async (innerSession) => {
expect(canUnloadLLM()).toBe(false);
expect(innerSession.isValid).toBe(true);
expect(outerSession.isValid).toBe(true);
});
// Inner session released, but outer still active
expect(canUnloadLLM()).toBe(false);
expect(outerSession.isValid).toBe(true);
});
// All sessions released
expect(canUnloadLLM()).toBe(true);
});
test("session embedBatch works correctly", async () => {
await withLLMSession(async (session) => {
const texts = ["Hello world", "Test text", "Another document"];
const results = await session.embedBatch(texts);
expect(results).toHaveLength(3);
for (const result of results) {
expect(result).not.toBeNull();
expect(result!.embedding.length).toBe(768);
}
});
});
test("session rerank works correctly", async () => {
await withLLMSession(async (session) => {
const documents: RerankDocument[] = [
{ file: "a.txt", text: "The capital of France is Paris." },
{ file: "b.txt", text: "Dogs are great pets." },
];
const result = await session.rerank("What is the capital of France?", documents);
expect(result.results).toHaveLength(2);
expect(result.results[0]!.file).toBe("a.txt");
expect(result.results[0]!.score).toBeGreaterThan(result.results[1]!.score);
});
});
test("max duration aborts session after timeout", async () => {
let aborted = false;
try {
await withLLMSession(async (session) => {
// Wait longer than max duration
await new Promise(resolve => setTimeout(resolve, 150));
// This operation should throw because session was aborted
await session.embed("test");
}, { maxDuration: 50 }); // 50ms max
} catch (err) {
if (err instanceof SessionReleasedError) {
aborted = true;
} else {
throw err;
}
}
expect(aborted).toBe(true);
}, 5000);
test("external abort signal propagates to session", async () => {
const abortController = new AbortController();
let sessionAborted = false;
const promise = withLLMSession(async (session) => {
// Wait a bit then check if aborted
await new Promise(resolve => setTimeout(resolve, 100));
if (!session.isValid) {
sessionAborted = true;
throw new SessionReleasedError("Session aborted");
}
return "should not reach";
}, { signal: abortController.signal });
// Abort after 20ms
setTimeout(() => abortController.abort(), 20);
try {
await promise;
} catch (err) {
// Expected
}
expect(sessionAborted).toBe(true);
}, 5000);
test("session provides abort signal for monitoring", async () => {
await withLLMSession(async (session) => {
expect(session.signal).toBeInstanceOf(AbortSignal);
expect(session.signal.aborted).toBe(false);
});
});
test("returns value from callback", async () => {
const result = await withLLMSession(async (session) => {
await session.embed("test");
return { status: "complete", count: 42 };
});
expect(result).toEqual({ status: "complete", count: 42 });
});
test("propagates errors from callback", async () => {
const customError = new Error("Custom test error");
await expect(
withLLMSession(async () => {
throw customError;
})
).rejects.toThrow("Custom test error");
});
});
});

View File

@ -119,6 +119,32 @@ export type RerankOptions = {
model?: string;
};
/**
* Options for LLM sessions
*/
export type LLMSessionOptions = {
/** Max session duration in ms (default: 10 minutes) */
maxDuration?: number;
/** External abort signal */
signal?: AbortSignal;
/** Debug name for logging */
name?: string;
};
/**
* Session interface for scoped LLM access with lifecycle guarantees
*/
export interface ILLMSession {
embed(text: string, options?: EmbedOptions): Promise<EmbeddingResult | null>;
embedBatch(texts: string[]): Promise<(EmbeddingResult | null)[]>;
expandQuery(query: string, options?: { context?: string; includeLexical?: boolean }): Promise<Queryable[]>;
rerank(query: string, documents: RerankDocument[], options?: RerankOptions): Promise<RerankResult>;
/** Whether this session is still valid (not released or aborted) */
readonly isValid: boolean;
/** Abort signal for this session (aborts on release or maxDuration) */
readonly signal: AbortSignal;
}
/**
* Supported query types for different search backends
*/
@ -225,8 +251,8 @@ export type LlamaCppConfig = {
/**
* LLM implementation using node-llama-cpp
*/
// Default inactivity timeout: 2 minutes
const DEFAULT_INACTIVITY_TIMEOUT_MS = 2 * 60 * 1000;
// Default inactivity timeout: 5 minutes (keep models warm during typical search sessions)
const DEFAULT_INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000;
export class LlamaCpp implements LLM {
private llama: Llama | null = null;
@ -241,8 +267,9 @@ export class LlamaCpp implements LLM {
private rerankModelUri: string;
private modelCacheDir: string;
// Ensure we don't load the same model concurrently (which can allocate duplicate VRAM).
// Ensure we don't load the same model/context concurrently (which can allocate duplicate VRAM).
private embedModelLoadPromise: Promise<LlamaModel> | null = null;
private embedContextCreatePromise: Promise<LlamaEmbeddingContext> | null = null;
private generateModelLoadPromise: Promise<LlamaModel> | null = null;
private rerankModelLoadPromise: Promise<LlamaModel> | null = null;
@ -266,7 +293,7 @@ export class LlamaCpp implements LLM {
/**
* Reset the inactivity timer. Called after each model operation.
* When timer fires, models are unloaded to free memory.
* When timer fires, models are unloaded to free memory (if no active sessions).
*/
private touchActivity(): void {
// Clear existing timer
@ -278,6 +305,14 @@ export class LlamaCpp implements LLM {
// Only set timer if we have disposable contexts and timeout is enabled
if (this.inactivityTimeoutMs > 0 && this.hasLoadedContexts()) {
this.inactivityTimer = setTimeout(() => {
// Check if session manager allows unloading
// canUnloadLLM is defined later in this file - it checks the session manager
// We use dynamic import pattern to avoid circular dependency issues
if (typeof canUnloadLLM === 'function' && !canUnloadLLM()) {
// Active sessions/operations - reschedule timer
this.touchActivity();
return;
}
this.unloadIdleResources().catch(err => {
console.error("Error unloading idle resources:", err);
});
@ -389,6 +424,8 @@ export class LlamaCpp implements LLM {
const modelPath = await this.resolveModel(this.embedModelUri);
const model = await llama.loadModel({ modelPath });
this.embedModel = model;
// Model loading counts as activity - ping to keep alive
this.touchActivity();
return model;
})();
@ -402,11 +439,30 @@ export class LlamaCpp implements LLM {
/**
* Load embedding context (lazy). Context can be disposed and recreated without reloading the model.
* Uses promise guard to prevent concurrent context creation race condition.
*/
private async ensureEmbedContext(): Promise<LlamaEmbeddingContext> {
if (!this.embedContext) {
const model = await this.ensureEmbedModel();
this.embedContext = await model.createEmbeddingContext();
// If context creation is already in progress, wait for it
if (this.embedContextCreatePromise) {
return await this.embedContextCreatePromise;
}
// Start context creation and store promise so concurrent calls wait
this.embedContextCreatePromise = (async () => {
const model = await this.ensureEmbedModel();
const context = await model.createEmbeddingContext();
this.embedContext = context;
return context;
})();
try {
const context = await this.embedContextCreatePromise;
this.touchActivity();
return context;
} finally {
this.embedContextCreatePromise = null;
}
}
this.touchActivity();
return this.embedContext;
@ -458,6 +514,8 @@ export class LlamaCpp implements LLM {
const modelPath = await this.resolveModel(this.rerankModelUri);
const model = await llama.loadModel({ modelPath });
this.rerankModel = model;
// Model loading counts as activity - ping to keep alive
this.touchActivity();
return model;
})();
@ -520,6 +578,9 @@ export class LlamaCpp implements LLM {
// ==========================================================================
async embed(text: string, options: EmbedOptions = {}): Promise<EmbeddingResult | null> {
// Ping activity at start to keep models alive during this operation
this.touchActivity();
try {
const context = await this.ensureEmbedContext();
const embedding = await context.getEmbeddingFor(text);
@ -539,6 +600,9 @@ export class LlamaCpp implements LLM {
* Uses Promise.all for parallel embedding - node-llama-cpp handles batching internally
*/
async embedBatch(texts: string[]): Promise<(EmbeddingResult | null)[]> {
// Ping activity at start to keep models alive during this operation
this.touchActivity();
if (texts.length === 0) return [];
try {
@ -549,6 +613,7 @@ export class LlamaCpp implements LLM {
texts.map(async (text) => {
try {
const embedding = await context.getEmbeddingFor(text);
this.touchActivity(); // Keep-alive during slow batches
return {
embedding: Array.from(embedding.vector),
model: this.embedModelUri,
@ -568,6 +633,9 @@ export class LlamaCpp implements LLM {
}
async generate(prompt: string, options: GenerateOptions = {}): Promise<GenerateResult | null> {
// Ping activity at start to keep models alive during this operation
this.touchActivity();
// Ensure model is loaded
await this.ensureGenerateModel();
@ -620,6 +688,9 @@ export class LlamaCpp implements LLM {
// ==========================================================================
async expandQuery(query: string, options: { context?: string, includeLexical?: boolean } = {}): Promise<Queryable[]> {
// Ping activity at start to keep models alive during this operation
this.touchActivity();
const llama = await this.ensureLlama();
await this.ensureGenerateModel();
@ -721,6 +792,9 @@ Final Output:`;
documents: RerankDocument[],
options: RerankOptions = {}
): Promise<RerankResult> {
// Ping activity at start to keep models alive during this operation
this.touchActivity();
const context = await this.ensureRerankContext();
// Build a map from document text to original indices (for lookup after sorting)
@ -781,13 +855,240 @@ Final Output:`;
this.rerankModel = null;
this.llama = null;
// Clear any in-flight load promises
// Clear any in-flight load/create promises
this.embedModelLoadPromise = null;
this.embedContextCreatePromise = null;
this.generateModelLoadPromise = null;
this.rerankModelLoadPromise = null;
}
}
// =============================================================================
// Session Management Layer
// =============================================================================
/**
* Manages LLM session lifecycle with reference counting.
* Coordinates with LlamaCpp idle timeout to prevent disposal during active sessions.
*/
class LLMSessionManager {
private llm: LlamaCpp;
private _activeSessionCount = 0;
private _inFlightOperations = 0;
constructor(llm: LlamaCpp) {
this.llm = llm;
}
get activeSessionCount(): number {
return this._activeSessionCount;
}
get inFlightOperations(): number {
return this._inFlightOperations;
}
/**
* Returns true only when both session count and in-flight operations are 0.
* Used by LlamaCpp to determine if idle unload is safe.
*/
canUnload(): boolean {
return this._activeSessionCount === 0 && this._inFlightOperations === 0;
}
acquire(): void {
this._activeSessionCount++;
}
release(): void {
this._activeSessionCount = Math.max(0, this._activeSessionCount - 1);
}
operationStart(): void {
this._inFlightOperations++;
}
operationEnd(): void {
this._inFlightOperations = Math.max(0, this._inFlightOperations - 1);
}
getLlamaCpp(): LlamaCpp {
return this.llm;
}
}
/**
* Error thrown when an operation is attempted on a released or aborted session.
*/
export class SessionReleasedError extends Error {
constructor(message = "LLM session has been released or aborted") {
super(message);
this.name = "SessionReleasedError";
}
}
/**
* Scoped LLM session with automatic lifecycle management.
* Wraps LlamaCpp methods with operation tracking and abort handling.
*/
class LLMSession implements ILLMSession {
private manager: LLMSessionManager;
private released = false;
private abortController: AbortController;
private maxDurationTimer: ReturnType<typeof setTimeout> | null = null;
private name: string;
constructor(manager: LLMSessionManager, options: LLMSessionOptions = {}) {
this.manager = manager;
this.name = options.name || "unnamed";
this.abortController = new AbortController();
// Link external abort signal if provided
if (options.signal) {
if (options.signal.aborted) {
this.abortController.abort(options.signal.reason);
} else {
options.signal.addEventListener("abort", () => {
this.abortController.abort(options.signal!.reason);
}, { once: true });
}
}
// Set up max duration timer
const maxDuration = options.maxDuration ?? 10 * 60 * 1000; // Default 10 minutes
if (maxDuration > 0) {
this.maxDurationTimer = setTimeout(() => {
this.abortController.abort(new Error(`Session "${this.name}" exceeded max duration of ${maxDuration}ms`));
}, maxDuration);
this.maxDurationTimer.unref(); // Don't keep process alive
}
// Acquire session lease
this.manager.acquire();
}
get isValid(): boolean {
return !this.released && !this.abortController.signal.aborted;
}
get signal(): AbortSignal {
return this.abortController.signal;
}
/**
* Release the session and decrement ref count.
* Called automatically by withLLMSession when the callback completes.
*/
release(): void {
if (this.released) return;
this.released = true;
if (this.maxDurationTimer) {
clearTimeout(this.maxDurationTimer);
this.maxDurationTimer = null;
}
this.abortController.abort(new Error("Session released"));
this.manager.release();
}
/**
* Wrap an operation with tracking and abort checking.
*/
private async withOperation<T>(fn: () => Promise<T>): Promise<T> {
if (!this.isValid) {
throw new SessionReleasedError();
}
this.manager.operationStart();
try {
// Check abort before starting
if (this.abortController.signal.aborted) {
throw new SessionReleasedError(
this.abortController.signal.reason?.message || "Session aborted"
);
}
return await fn();
} finally {
this.manager.operationEnd();
}
}
async embed(text: string, options?: EmbedOptions): Promise<EmbeddingResult | null> {
return this.withOperation(() => this.manager.getLlamaCpp().embed(text, options));
}
async embedBatch(texts: string[]): Promise<(EmbeddingResult | null)[]> {
return this.withOperation(() => this.manager.getLlamaCpp().embedBatch(texts));
}
async expandQuery(
query: string,
options?: { context?: string; includeLexical?: boolean }
): Promise<Queryable[]> {
return this.withOperation(() => this.manager.getLlamaCpp().expandQuery(query, options));
}
async rerank(
query: string,
documents: RerankDocument[],
options?: RerankOptions
): Promise<RerankResult> {
return this.withOperation(() => this.manager.getLlamaCpp().rerank(query, documents, options));
}
}
// Session manager for the default LlamaCpp instance
let defaultSessionManager: LLMSessionManager | null = null;
/**
* Get the session manager for the default LlamaCpp instance.
*/
function getSessionManager(): LLMSessionManager {
const llm = getDefaultLlamaCpp();
if (!defaultSessionManager || defaultSessionManager.getLlamaCpp() !== llm) {
defaultSessionManager = new LLMSessionManager(llm);
}
return defaultSessionManager;
}
/**
* Execute a function with a scoped LLM session.
* The session provides lifecycle guarantees - resources won't be disposed mid-operation.
*
* @example
* ```typescript
* await withLLMSession(async (session) => {
* const expanded = await session.expandQuery(query);
* const embeddings = await session.embedBatch(texts);
* const reranked = await session.rerank(query, docs);
* return reranked;
* }, { maxDuration: 10 * 60 * 1000, name: 'querySearch' });
* ```
*/
export async function withLLMSession<T>(
fn: (session: ILLMSession) => Promise<T>,
options?: LLMSessionOptions
): Promise<T> {
const manager = getSessionManager();
const session = new LLMSession(manager, options);
try {
return await fn(session);
} finally {
session.release();
}
}
/**
* Check if idle unload is safe (no active sessions or operations).
* Used internally by LlamaCpp idle timer.
*/
export function canUnloadLLM(): boolean {
if (!defaultSessionManager) return true;
return defaultSessionManager.canUnload();
}
// =============================================================================
// Singleton for default LlamaCpp instance
// =============================================================================

View File

@ -65,7 +65,7 @@ import {
createStore,
getDefaultDbPath,
} from "./store.js";
import { getDefaultLlamaCpp, disposeDefaultLlamaCpp, type RerankDocument, type Queryable, type QueryType } from "./llm.js";
import { getDefaultLlamaCpp, disposeDefaultLlamaCpp, withLLMSession, type ILLMSession, type RerankDocument, type Queryable, type QueryType } from "./llm.js";
import type { SearchResult, RankedResult } from "./store.js";
import {
formatSearchResults,
@ -231,20 +231,21 @@ function computeDisplayPath(
}
// Rerank documents using node-llama-cpp cross-encoder model
async function rerank(query: string, documents: { file: string; text: string }[], _model: string = DEFAULT_RERANK_MODEL, _db?: Database): Promise<{ file: string; score: number }[]> {
async function rerank(query: string, documents: { file: string; text: string }[], _model: string = DEFAULT_RERANK_MODEL, _db?: Database, session?: ILLMSession): Promise<{ file: string; score: number }[]> {
if (documents.length === 0) return [];
const total = documents.length;
process.stderr.write(`Reranking ${total} documents...\n`);
progress.indeterminate();
const llm = getDefaultLlamaCpp();
const rerankDocs: RerankDocument[] = documents.map((doc) => ({
file: doc.file,
text: doc.text.slice(0, 4000), // Truncate to context limit
}));
const result = await llm.rerank(query, rerankDocs);
const result = session
? await session.rerank(query, rerankDocs)
: await getDefaultLlamaCpp().rerank(query, rerankDocs);
progress.clear();
process.stderr.write("\n");
@ -1543,7 +1544,7 @@ async function vectorIndex(model: string = DEFAULT_EMBED_MODEL, force: boolean =
return;
}
const totalBytes = allChunks.reduce((sum, c) => sum + c.bytes, 0);
const totalBytes = allChunks.reduce((sum, chk) => sum + chk.bytes, 0);
const totalChunks = allChunks.length;
const totalDocs = hashesToEmbed.length;
@ -1556,99 +1557,103 @@ async function vectorIndex(model: string = DEFAULT_EMBED_MODEL, force: boolean =
// Hide cursor during embedding
cursor.hide();
// Get embedding dimensions from first chunk
progress.indeterminate();
const llm = getDefaultLlamaCpp();
const firstChunk = allChunks[0];
if (!firstChunk) {
throw new Error("No chunks available to embed");
}
const firstText = formatDocForEmbedding(firstChunk.text, firstChunk.title);
const firstResult = await llm.embed(firstText);
if (!firstResult) {
throw new Error("Failed to get embedding dimensions from first chunk");
}
ensureVecTable(db, firstResult.embedding.length);
// Wrap all LLM embedding operations in a session for lifecycle management
// Use 30 minute timeout for large collections
await withLLMSession(async (session) => {
// Get embedding dimensions from first chunk
progress.indeterminate();
const firstChunk = allChunks[0];
if (!firstChunk) {
throw new Error("No chunks available to embed");
}
const firstText = formatDocForEmbedding(firstChunk.text, firstChunk.title);
const firstResult = await session.embed(firstText);
if (!firstResult) {
throw new Error("Failed to get embedding dimensions from first chunk");
}
ensureVecTable(db, firstResult.embedding.length);
let chunksEmbedded = 0, errors = 0, bytesProcessed = 0;
const startTime = Date.now();
let chunksEmbedded = 0, errors = 0, bytesProcessed = 0;
const startTime = Date.now();
// Batch embedding for better throughput
// Process in batches of 32 to balance memory usage and efficiency
const BATCH_SIZE = 32;
// Batch embedding for better throughput
// Process in batches of 32 to balance memory usage and efficiency
const BATCH_SIZE = 32;
for (let batchStart = 0; batchStart < allChunks.length; batchStart += BATCH_SIZE) {
const batchEnd = Math.min(batchStart + BATCH_SIZE, allChunks.length);
const batch = allChunks.slice(batchStart, batchEnd);
for (let batchStart = 0; batchStart < allChunks.length; batchStart += BATCH_SIZE) {
const batchEnd = Math.min(batchStart + BATCH_SIZE, allChunks.length);
const batch = allChunks.slice(batchStart, batchEnd);
// Format texts for embedding
const texts = batch.map(chunk => formatDocForEmbedding(chunk.text, chunk.title));
// Format texts for embedding
const texts = batch.map(chunk => formatDocForEmbedding(chunk.text, chunk.title));
try {
// Batch embed all texts at once
const embeddings = await llm.embedBatch(texts);
try {
// Batch embed all texts at once
const embeddings = await session.embedBatch(texts);
// Insert each embedding
for (let i = 0; i < batch.length; i++) {
const chunk = batch[i]!;
const embedding = embeddings[i];
// Insert each embedding
for (let i = 0; i < batch.length; i++) {
const chunk = batch[i]!;
const embedding = embeddings[i];
if (embedding) {
insertEmbedding(db, chunk.hash, chunk.seq, chunk.pos, new Float32Array(embedding.embedding), model, now);
chunksEmbedded++;
} else {
errors++;
console.error(`\n${c.yellow}⚠ Error embedding "${chunk.displayName}" chunk ${chunk.seq}${c.reset}`);
}
bytesProcessed += chunk.bytes;
}
} catch (err) {
// If batch fails, try individual embeddings as fallback
for (const chunk of batch) {
try {
const text = formatDocForEmbedding(chunk.text, chunk.title);
const result = await llm.embed(text);
if (result) {
insertEmbedding(db, chunk.hash, chunk.seq, chunk.pos, new Float32Array(result.embedding), model, now);
if (embedding) {
insertEmbedding(db, chunk.hash, chunk.seq, chunk.pos, new Float32Array(embedding.embedding), model, now);
chunksEmbedded++;
} else {
errors++;
console.error(`\n${c.yellow}⚠ Error embedding "${chunk.displayName}" chunk ${chunk.seq}${c.reset}`);
}
} catch (innerErr) {
errors++;
console.error(`\n${c.yellow}⚠ Error embedding "${chunk.displayName}" chunk ${chunk.seq}: ${innerErr}${c.reset}`);
bytesProcessed += chunk.bytes;
}
} catch (err) {
// If batch fails, try individual embeddings as fallback
for (const chunk of batch) {
try {
const text = formatDocForEmbedding(chunk.text, chunk.title);
const result = await session.embed(text);
if (result) {
insertEmbedding(db, chunk.hash, chunk.seq, chunk.pos, new Float32Array(result.embedding), model, now);
chunksEmbedded++;
} else {
errors++;
}
} catch (innerErr) {
errors++;
console.error(`\n${c.yellow}⚠ Error embedding "${chunk.displayName}" chunk ${chunk.seq}: ${innerErr}${c.reset}`);
}
bytesProcessed += chunk.bytes;
}
bytesProcessed += chunk.bytes;
}
const percent = (bytesProcessed / totalBytes) * 100;
progress.set(percent);
const elapsed = (Date.now() - startTime) / 1000;
const bytesPerSec = bytesProcessed / elapsed;
const remainingBytes = totalBytes - bytesProcessed;
const etaSec = remainingBytes / bytesPerSec;
const bar = renderProgressBar(percent);
const percentStr = percent.toFixed(0).padStart(3);
const throughput = `${formatBytes(bytesPerSec)}/s`;
const eta = elapsed > 2 ? formatETA(etaSec) : "...";
const errStr = errors > 0 ? ` ${c.yellow}${errors} err${c.reset}` : "";
process.stderr.write(`\r${c.cyan}${bar}${c.reset} ${c.bold}${percentStr}%${c.reset} ${c.dim}${chunksEmbedded}/${totalChunks}${c.reset}${errStr} ${c.dim}${throughput} ETA ${eta}${c.reset} `);
}
const percent = (bytesProcessed / totalBytes) * 100;
progress.set(percent);
progress.clear();
cursor.show();
const totalTimeSec = (Date.now() - startTime) / 1000;
const avgThroughput = formatBytes(totalBytes / totalTimeSec);
const elapsed = (Date.now() - startTime) / 1000;
const bytesPerSec = bytesProcessed / elapsed;
const remainingBytes = totalBytes - bytesProcessed;
const etaSec = remainingBytes / bytesPerSec;
console.log(`\r${c.green}${renderProgressBar(100)}${c.reset} ${c.bold}100%${c.reset} `);
console.log(`\n${c.green}✓ Done!${c.reset} Embedded ${c.bold}${chunksEmbedded}${c.reset} chunks from ${c.bold}${totalDocs}${c.reset} documents in ${c.bold}${formatETA(totalTimeSec)}${c.reset} ${c.dim}(${avgThroughput}/s)${c.reset}`);
if (errors > 0) {
console.log(`${c.yellow}${errors} chunks failed${c.reset}`);
}
}, { maxDuration: 30 * 60 * 1000, name: 'embed-command' });
const bar = renderProgressBar(percent);
const percentStr = percent.toFixed(0).padStart(3);
const throughput = `${formatBytes(bytesPerSec)}/s`;
const eta = elapsed > 2 ? formatETA(etaSec) : "...";
const errStr = errors > 0 ? ` ${c.yellow}${errors} err${c.reset}` : "";
process.stderr.write(`\r${c.cyan}${bar}${c.reset} ${c.bold}${percentStr}%${c.reset} ${c.dim}${chunksEmbedded}/${totalChunks}${c.reset}${errStr} ${c.dim}${throughput} ETA ${eta}${c.reset} `);
}
progress.clear();
cursor.show();
const totalTimeSec = (Date.now() - startTime) / 1000;
const avgThroughput = formatBytes(totalBytes / totalTimeSec);
console.log(`\r${c.green}${renderProgressBar(100)}${c.reset} ${c.bold}100%${c.reset} `);
console.log(`\n${c.green}✓ Done!${c.reset} Embedded ${c.bold}${chunksEmbedded}${c.reset} chunks from ${c.bold}${totalDocs}${c.reset} documents in ${c.bold}${formatETA(totalTimeSec)}${c.reset} ${c.dim}(${avgThroughput}/s)${c.reset}`);
if (errors > 0) {
console.log(`${c.yellow}${errors} chunks failed${c.reset}`);
}
closeDb();
}
@ -1975,60 +1980,64 @@ async function vectorSearch(query: string, opts: OutputOptions, model: string =
// Check index health and warn about issues
checkIndexHealth(db);
// Expand query using structured output (no lexical for vector-only search)
const queryables = await expandQueryStructured(query, false, opts.context);
// Wrap LLM operations in a session for lifecycle management
await withLLMSession(async (session) => {
// Expand query using structured output (no lexical for vector-only search)
const queryables = await expandQueryStructured(query, false, opts.context, session);
// Build list of queries for vector search: original, vec, and hyde
const vectorQueries: string[] = [query];
for (const q of queryables) {
if (q.type === 'vec' || q.type === 'hyde') {
if (q.text && q.text !== query) {
vectorQueries.push(q.text);
// Build list of queries for vector search: original, vec, and hyde
const vectorQueries: string[] = [query];
for (const q of queryables) {
if (q.type === 'vec' || q.type === 'hyde') {
if (q.text && q.text !== query) {
vectorQueries.push(q.text);
}
}
}
}
process.stderr.write(`${c.dim}Searching ${vectorQueries.length} vector queries...${c.reset}\n`);
process.stderr.write(`${c.dim}Searching ${vectorQueries.length} vector queries...${c.reset}\n`);
// Collect results from all query variations
const perQueryLimit = opts.all ? 500 : 20;
const allResults = new Map<string, { file: string; displayPath: string; title: string; body: string; score: number; hash: string }>();
// Collect results from all query variations
const perQueryLimit = opts.all ? 500 : 20;
const allResults = new Map<string, { file: string; displayPath: string; title: string; body: string; score: number; hash: string }>();
// IMPORTANT: Run vector searches sequentially, not with Promise.all.
// node-llama-cpp's embedding context hangs when multiple concurrent embed() calls
// are made. This is a known limitation of the LlamaEmbeddingContext.
// See: https://github.com/tobi/qmd/pull/23
for (const q of vectorQueries) {
const vecResults = await searchVec(db, q, model, perQueryLimit, collectionName as any);
for (const r of vecResults) {
const existing = allResults.get(r.filepath);
if (!existing || r.score > existing.score) {
allResults.set(r.filepath, { file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score, hash: r.hash });
// IMPORTANT: Run vector searches sequentially, not with Promise.all.
// node-llama-cpp's embedding context hangs when multiple concurrent embed() calls
// are made. This is a known limitation of the LlamaEmbeddingContext.
// See: https://github.com/tobi/qmd/pull/23
for (const q of vectorQueries) {
const vecResults = await searchVec(db, q, model, perQueryLimit, collectionName as any, session);
for (const r of vecResults) {
const existing = allResults.get(r.filepath);
if (!existing || r.score > existing.score) {
allResults.set(r.filepath, { file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score, hash: r.hash });
}
}
}
}
// Sort by max score and limit to requested count
const results = Array.from(allResults.values())
.sort((a, b) => b.score - a.score)
.slice(0, opts.limit)
.map(r => ({ ...r, context: getContextForFile(db, r.file) }));
// Sort by max score and limit to requested count
const results = Array.from(allResults.values())
.sort((a, b) => b.score - a.score)
.slice(0, opts.limit)
.map(r => ({ ...r, context: getContextForFile(db, r.file) }));
closeDb();
closeDb();
if (results.length === 0) {
console.log("No results found.");
return;
}
outputResults(results, query, { ...opts, limit: results.length }); // Already limited
if (results.length === 0) {
console.log("No results found.");
return;
}
outputResults(results, query, { ...opts, limit: results.length }); // Already limited
}, { maxDuration: 10 * 60 * 1000, name: 'vectorSearch' });
}
// Expand query using structured output with GBNF grammar
async function expandQueryStructured(query: string, includeLexical: boolean = true, context?: string): Promise<Queryable[]> {
async function expandQueryStructured(query: string, includeLexical: boolean = true, context?: string, session?: ILLMSession): Promise<Queryable[]> {
process.stderr.write(`${c.dim}Expanding query...${c.reset}\n`);
const llm = getDefaultLlamaCpp();
const queryables = await llm.expandQuery(query, { includeLexical, context });
const queryables = session
? await session.expandQuery(query, { includeLexical, context })
: await getDefaultLlamaCpp().expandQuery(query, { includeLexical, context });
// Log the expansion as a tree
const lines: string[] = [];
@ -2060,8 +2069,8 @@ async function expandQueryStructured(query: string, includeLexical: boolean = tr
return queryables;
}
async function expandQuery(query: string, _model: string = DEFAULT_QUERY_MODEL, _db?: Database): Promise<string[]> {
const queryables = await expandQueryStructured(query, true);
async function expandQuery(query: string, _model: string = DEFAULT_QUERY_MODEL, _db?: Database, session?: ILLMSession): Promise<string[]> {
const queryables = await expandQueryStructured(query, true, undefined, session);
const queries = new Set<string>([query]);
for (const q of queryables) {
queries.add(q.text);
@ -2098,178 +2107,182 @@ async function querySearch(query: string, opts: OutputOptions, embedModel: strin
const secondScore = initialFts[1]?.score ?? 0;
const hasStrongSignal = initialFts.length > 0 && topScore >= 0.85 && (topScore - secondScore) >= 0.15;
let ftsQueries: string[] = [query];
let vectorQueries: string[] = [query];
// Wrap LLM operations in a session for lifecycle management
await withLLMSession(async (session) => {
let ftsQueries: string[] = [query];
let vectorQueries: string[] = [query];
if (hasStrongSignal) {
// Strong BM25 signal - skip expensive LLM expansion
process.stderr.write(`${c.dim}Strong BM25 signal (${topScore.toFixed(2)}) - skipping expansion${c.reset}\n`);
// Still log the "expansion tree" in the same style as vsearch for consistency.
{
const lines: string[] = [];
lines.push(`${c.dim}├─ ${query} · (lexical+vector)${c.reset}`);
lines[lines.length - 1] = lines[lines.length - 1]!.replace('├─', '└─');
for (const line of lines) process.stderr.write(line + '\n');
}
} else {
// Weak signal - expand query for better recall
const queryables = await expandQueryStructured(query, true, opts.context);
for (const q of queryables) {
if (q.type === 'lex') {
if (q.text && q.text !== query) ftsQueries.push(q.text);
} else if (q.type === 'vec' || q.type === 'hyde') {
if (q.text && q.text !== query) vectorQueries.push(q.text);
if (hasStrongSignal) {
// Strong BM25 signal - skip expensive LLM expansion
process.stderr.write(`${c.dim}Strong BM25 signal (${topScore.toFixed(2)}) - skipping expansion${c.reset}\n`);
// Still log the "expansion tree" in the same style as vsearch for consistency.
{
const lines: string[] = [];
lines.push(`${c.dim}├─ ${query} · (lexical+vector)${c.reset}`);
lines[lines.length - 1] = lines[lines.length - 1]!.replace('├─', '└─');
for (const line of lines) process.stderr.write(line + '\n');
}
}
}
} else {
// Weak signal - expand query for better recall
const queryables = await expandQueryStructured(query, true, opts.context, session);
process.stderr.write(`${c.dim}Searching ${ftsQueries.length} lexical + ${vectorQueries.length} vector queries...${c.reset}\n`);
// Collect ranked result lists for RRF fusion
const rankedLists: RankedResult[][] = [];
// Map to store hash by filepath for final results
const hashMap = new Map<string, string>();
// Run all searches concurrently (FTS + Vector)
const searchPromises: Promise<void>[] = [];
// FTS searches
for (const q of ftsQueries) {
if (!q) continue;
searchPromises.push((async () => {
const ftsResults = searchFTS(db, q, 20, (collectionName || "") as any);
if (ftsResults.length > 0) {
for (const r of ftsResults) {
// Mutex for hashMap is not strictly needed as it's just adding values
hashMap.set(r.filepath, r.hash);
for (const q of queryables) {
if (q.type === 'lex') {
if (q.text && q.text !== query) ftsQueries.push(q.text);
} else if (q.type === 'vec' || q.type === 'hyde') {
if (q.text && q.text !== query) vectorQueries.push(q.text);
}
rankedLists.push(ftsResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
}
})());
}
}
// Vector searches
if (hasVectors) {
for (const q of vectorQueries) {
process.stderr.write(`${c.dim}Searching ${ftsQueries.length} lexical + ${vectorQueries.length} vector queries...${c.reset}\n`);
// Collect ranked result lists for RRF fusion
const rankedLists: RankedResult[][] = [];
// Map to store hash by filepath for final results
const hashMap = new Map<string, string>();
// Run all searches concurrently (FTS + Vector)
const searchPromises: Promise<void>[] = [];
// FTS searches
for (const q of ftsQueries) {
if (!q) continue;
searchPromises.push((async () => {
const vecResults = await searchVec(db, q, embedModel, 20, (collectionName || "") as any);
if (vecResults.length > 0) {
for (const r of vecResults) hashMap.set(r.filepath, r.hash);
rankedLists.push(vecResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
const ftsResults = searchFTS(db, q, 20, (collectionName || "") as any);
if (ftsResults.length > 0) {
for (const r of ftsResults) {
// Mutex for hashMap is not strictly needed as it's just adding values
hashMap.set(r.filepath, r.hash);
}
rankedLists.push(ftsResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
}
})());
}
}
await Promise.all(searchPromises);
// Apply Reciprocal Rank Fusion to combine all ranked lists
// Give 2x weight to original query results (first 2 lists: FTS + vector)
const weights = rankedLists.map((_, i) => i < 2 ? 2.0 : 1.0);
const fused = reciprocalRankFusion(rankedLists, weights);
// Hard cap reranking for latency/cost. We rerank per-document (best chunk only).
const RERANK_DOC_LIMIT = 40;
const candidates = fused.slice(0, RERANK_DOC_LIMIT);
if (candidates.length === 0) {
console.log("No results found.");
closeDb();
return;
}
// Rerank multiple chunks per document, then aggregate scores
// This improves ranking for long documents where keyword-matched chunk isn't always best
// We only rerank ONE chunk per document (best chunk by a simple keyword heuristic),
// so we never rerank more than RERANK_DOC_LIMIT items.
const chunksToRerank: { file: string; text: string; chunkIdx: number }[] = [];
const docChunkMap = new Map<string, { chunks: { text: string; pos: number }[]; bestIdx: number }>();
const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length > 2);
for (const c of candidates) {
const chunks = chunkDocument(c.body);
if (chunks.length === 0) continue;
// Choose best chunk by keyword matches; fall back to first chunk.
let bestIdx = 0;
let bestScore = -1;
for (let i = 0; i < chunks.length; i++) {
const chunkLower = chunks[i]!.text.toLowerCase();
const score = queryTerms.reduce((acc, term) => acc + (chunkLower.includes(term) ? 1 : 0), 0);
if (score > bestScore) {
bestScore = score;
bestIdx = i;
// Vector searches (session ensures contexts stay alive)
if (hasVectors) {
for (const q of vectorQueries) {
if (!q) continue;
searchPromises.push((async () => {
const vecResults = await searchVec(db, q, embedModel, 20, (collectionName || "") as any, session);
if (vecResults.length > 0) {
for (const r of vecResults) hashMap.set(r.filepath, r.hash);
rankedLists.push(vecResults.map(r => ({ file: r.filepath, displayPath: r.displayPath, title: r.title, body: r.body || "", score: r.score })));
}
})());
}
}
chunksToRerank.push({ file: c.file, text: chunks[bestIdx]!.text, chunkIdx: bestIdx });
docChunkMap.set(c.file, { chunks, bestIdx });
}
await Promise.all(searchPromises);
// Rerank selected chunks (with caching). One chunk per doc -> one rerank item per doc.
const reranked = await rerank(
query,
chunksToRerank.map(c => ({ file: c.file, text: c.text })),
rerankModel,
db
);
// Apply Reciprocal Rank Fusion to combine all ranked lists
// Give 2x weight to original query results (first 2 lists: FTS + vector)
const weights = rankedLists.map((_, i) => i < 2 ? 2.0 : 1.0);
const fused = reciprocalRankFusion(rankedLists, weights);
// Hard cap reranking for latency/cost. We rerank per-document (best chunk only).
const RERANK_DOC_LIMIT = 40;
const candidates = fused.slice(0, RERANK_DOC_LIMIT);
const aggregatedScores = new Map<string, { score: number; bestChunkIdx: number }>();
for (const r of reranked) {
const chunkInfo = docChunkMap.get(r.file);
aggregatedScores.set(r.file, { score: r.score, bestChunkIdx: chunkInfo?.bestIdx ?? 0 });
}
// Blend RRF position score with aggregated reranker score using position-aware weights
// Top retrieval results get more protection from reranker disagreement
const candidateMap = new Map(candidates.map(c => [c.file, { displayPath: c.displayPath, title: c.title, body: c.body }]));
const rrfRankMap = new Map(candidates.map((c, i) => [c.file, i + 1])); // 1-indexed rank
const finalResults = Array.from(aggregatedScores.entries()).map(([file, { score: rerankScore, bestChunkIdx }]) => {
const rrfRank = rrfRankMap.get(file) || 30;
// Position-aware blending: top retrieval results preserved more
// Rank 1-3: 75% RRF, 25% reranker (trust retrieval for exact matches)
// Rank 4-10: 60% RRF, 40% reranker
// Rank 11+: 40% RRF, 60% reranker (trust reranker for lower-ranked)
let rrfWeight: number;
if (rrfRank <= 3) {
rrfWeight = 0.75;
} else if (rrfRank <= 10) {
rrfWeight = 0.60;
} else {
rrfWeight = 0.40;
if (candidates.length === 0) {
console.log("No results found.");
closeDb();
return;
}
const rrfScore = 1 / rrfRank; // Position-based: 1, 0.5, 0.33...
const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * rerankScore;
const candidate = candidateMap.get(file);
// Use the best-scoring chunk's text for the body (better for snippets)
const chunkInfo = docChunkMap.get(file);
const chunkBody = chunkInfo ? (chunkInfo.chunks[bestChunkIdx]?.text || chunkInfo.chunks[0]!.text) : candidate?.body || "";
const chunkPos = chunkInfo ? (chunkInfo.chunks[bestChunkIdx]?.pos || 0) : 0;
return {
file,
displayPath: candidate?.displayPath || "",
title: candidate?.title || "",
body: chunkBody,
chunkPos,
score: blendedScore,
context: getContextForFile(db, file),
hash: hashMap.get(file) || "",
};
}).sort((a, b) => b.score - a.score);
// Deduplicate by file (safety net - shouldn't happen but prevents duplicate output)
const seenFiles = new Set<string>();
const dedupedResults = finalResults.filter(r => {
if (seenFiles.has(r.file)) return false;
seenFiles.add(r.file);
return true;
});
// Rerank multiple chunks per document, then aggregate scores
// This improves ranking for long documents where keyword-matched chunk isn't always best
// We only rerank ONE chunk per document (best chunk by a simple keyword heuristic),
// so we never rerank more than RERANK_DOC_LIMIT items.
const chunksToRerank: { file: string; text: string; chunkIdx: number }[] = [];
const docChunkMap = new Map<string, { chunks: { text: string; pos: number }[]; bestIdx: number }>();
closeDb();
outputResults(dedupedResults, query, opts);
const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length > 2);
for (const cand of candidates) {
const chunks = chunkDocument(cand.body);
if (chunks.length === 0) continue;
// Choose best chunk by keyword matches; fall back to first chunk.
let bestIdx = 0;
let bestScore = -1;
for (let i = 0; i < chunks.length; i++) {
const chunkLower = chunks[i]!.text.toLowerCase();
const score = queryTerms.reduce((acc, term) => acc + (chunkLower.includes(term) ? 1 : 0), 0);
if (score > bestScore) {
bestScore = score;
bestIdx = i;
}
}
chunksToRerank.push({ file: cand.file, text: chunks[bestIdx]!.text, chunkIdx: bestIdx });
docChunkMap.set(cand.file, { chunks, bestIdx });
}
// Rerank selected chunks (with caching). One chunk per doc -> one rerank item per doc.
const reranked = await rerank(
query,
chunksToRerank.map(ch => ({ file: ch.file, text: ch.text })),
rerankModel,
db,
session
);
const aggregatedScores = new Map<string, { score: number; bestChunkIdx: number }>();
for (const r of reranked) {
const chunkInfo = docChunkMap.get(r.file);
aggregatedScores.set(r.file, { score: r.score, bestChunkIdx: chunkInfo?.bestIdx ?? 0 });
}
// Blend RRF position score with aggregated reranker score using position-aware weights
// Top retrieval results get more protection from reranker disagreement
const candidateMap = new Map(candidates.map(cand => [cand.file, { displayPath: cand.displayPath, title: cand.title, body: cand.body }]));
const rrfRankMap = new Map(candidates.map((cand, i) => [cand.file, i + 1])); // 1-indexed rank
const finalResults = Array.from(aggregatedScores.entries()).map(([file, { score: rerankScore, bestChunkIdx }]) => {
const rrfRank = rrfRankMap.get(file) || 30;
// Position-aware blending: top retrieval results preserved more
// Rank 1-3: 75% RRF, 25% reranker (trust retrieval for exact matches)
// Rank 4-10: 60% RRF, 40% reranker
// Rank 11+: 40% RRF, 60% reranker (trust reranker for lower-ranked)
let rrfWeight: number;
if (rrfRank <= 3) {
rrfWeight = 0.75;
} else if (rrfRank <= 10) {
rrfWeight = 0.60;
} else {
rrfWeight = 0.40;
}
const rrfScore = 1 / rrfRank; // Position-based: 1, 0.5, 0.33...
const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * rerankScore;
const candidate = candidateMap.get(file);
// Use the best-scoring chunk's text for the body (better for snippets)
const chunkInfo = docChunkMap.get(file);
const chunkBody = chunkInfo ? (chunkInfo.chunks[bestChunkIdx]?.text || chunkInfo.chunks[0]!.text) : candidate?.body || "";
const chunkPos = chunkInfo ? (chunkInfo.chunks[bestChunkIdx]?.pos || 0) : 0;
return {
file,
displayPath: candidate?.displayPath || "",
title: candidate?.title || "",
body: chunkBody,
chunkPos,
score: blendedScore,
context: getContextForFile(db, file),
hash: hashMap.get(file) || "",
};
}).sort((a, b) => b.score - a.score);
// Deduplicate by file (safety net - shouldn't happen but prevents duplicate output)
const seenFiles = new Set<string>();
const dedupedResults = finalResults.filter(r => {
if (seenFiles.has(r.file)) return false;
seenFiles.add(r.file);
return true;
});
closeDb();
outputResults(dedupedResults, query, opts);
}, { maxDuration: 10 * 60 * 1000, name: 'querySearch' });
}
// Parse CLI arguments using util.parseArgs

395
src/store-paths.test.ts Normal file
View File

@ -0,0 +1,395 @@
/**
* store-paths.test.ts - Comprehensive unit tests for Windows path support
*
* Tests all path-related utility functions for cross-platform compatibility:
* - isAbsolutePath() - Unix, Windows (C:\, C:/), and Git Bash (/c/) paths
* - normalizePathSeparators() - backslash to forward slash conversion
* - getRelativePathFromPrefix() - relative path extraction
* - resolve() - path resolution with Unix and Windows paths
*
* Run with: bun test store-paths.test.ts
*/
import { describe, test, expect, beforeEach, afterEach } from "bun:test";
import {
isAbsolutePath,
normalizePathSeparators,
getRelativePathFromPrefix,
resolve,
} from "./store.js";
// =============================================================================
// Test Utilities
// =============================================================================
let originalPWD: string | undefined;
let originalProcessCwd: () => string;
beforeEach(() => {
// Save original environment
originalPWD = Bun.env.PWD;
originalProcessCwd = process.cwd;
});
afterEach(() => {
// Restore original environment
if (originalPWD !== undefined) {
Bun.env.PWD = originalPWD;
} else {
delete Bun.env.PWD;
}
process.cwd = originalProcessCwd;
});
/**
* Mock the current working directory for testing.
* Sets both Bun.env.PWD and process.cwd() to simulate different environments.
*/
function mockPWD(path: string): void {
Bun.env.PWD = path;
process.cwd = () => path;
}
// =============================================================================
// Path Utilities - Cross-platform Support
// =============================================================================
describe("Path utilities - Cross-platform support", () => {
// ===========================================================================
// isAbsolutePath
// ===========================================================================
describe("isAbsolutePath", () => {
test("Unix absolute paths", () => {
expect(isAbsolutePath("/path/to/file")).toBe(true);
expect(isAbsolutePath("/")).toBe(true);
expect(isAbsolutePath("/home/user/documents")).toBe(true);
expect(isAbsolutePath("/usr/local/bin")).toBe(true);
});
test("Unix relative paths", () => {
expect(isAbsolutePath("path/to/file")).toBe(false);
expect(isAbsolutePath("./path/to/file")).toBe(false);
expect(isAbsolutePath("../path/to/file")).toBe(false);
expect(isAbsolutePath("./file")).toBe(false);
expect(isAbsolutePath("../file")).toBe(false);
expect(isAbsolutePath("file.txt")).toBe(false);
});
test("Windows absolute paths (native) - forward slash", () => {
expect(isAbsolutePath("C:/path/to/file")).toBe(true);
expect(isAbsolutePath("C:/")).toBe(true);
expect(isAbsolutePath("D:/Users/Documents")).toBe(true);
expect(isAbsolutePath("Z:/")).toBe(true);
expect(isAbsolutePath("c:/lowercase")).toBe(true);
});
test("Windows absolute paths (native) - backslash", () => {
expect(isAbsolutePath("C:\\path\\to\\file")).toBe(true);
expect(isAbsolutePath("C:\\")).toBe(true);
expect(isAbsolutePath("D:\\Users\\Documents")).toBe(true);
expect(isAbsolutePath("Z:\\")).toBe(true);
expect(isAbsolutePath("c:\\lowercase")).toBe(true);
});
test("Windows relative paths", () => {
expect(isAbsolutePath("path\\to\\file")).toBe(false);
expect(isAbsolutePath(".\\path\\to\\file")).toBe(false);
expect(isAbsolutePath("..\\path\\to\\file")).toBe(false);
expect(isAbsolutePath(".\\file")).toBe(false);
expect(isAbsolutePath("..\\file")).toBe(false);
expect(isAbsolutePath("file.txt")).toBe(false);
});
test("Git Bash style paths", () => {
expect(isAbsolutePath("/c/Users/name/file")).toBe(true);
expect(isAbsolutePath("/C/Users/name/file")).toBe(true);
expect(isAbsolutePath("/d/Projects")).toBe(true);
expect(isAbsolutePath("/D/Projects")).toBe(true);
expect(isAbsolutePath("/z/")).toBe(true);
});
test("Edge cases", () => {
expect(isAbsolutePath("")).toBe(false);
expect(isAbsolutePath("C:")).toBe(true); // Drive letter only
expect(isAbsolutePath("C")).toBe(false); // Just a letter
expect(isAbsolutePath(":")).toBe(false);
expect(isAbsolutePath("/a")).toBe(true); // Short Unix path
expect(isAbsolutePath("/1/")).toBe(true); // Number after slash (not Git Bash)
});
});
// ===========================================================================
// normalizePathSeparators
// ===========================================================================
describe("normalizePathSeparators", () => {
test("Windows paths with backslashes", () => {
expect(normalizePathSeparators("C:\\Users\\name\\file.txt"))
.toBe("C:/Users/name/file.txt");
expect(normalizePathSeparators("D:\\Projects\\qmd\\src"))
.toBe("D:/Projects/qmd/src");
expect(normalizePathSeparators("\\path\\to\\file"))
.toBe("/path/to/file");
});
test("Mixed separators", () => {
expect(normalizePathSeparators("C:\\Users/name\\file.txt"))
.toBe("C:/Users/name/file.txt");
expect(normalizePathSeparators("path\\to/file/here"))
.toBe("path/to/file/here");
});
test("Unix paths (should remain unchanged)", () => {
expect(normalizePathSeparators("/path/to/file"))
.toBe("/path/to/file");
expect(normalizePathSeparators("/usr/local/bin"))
.toBe("/usr/local/bin");
expect(normalizePathSeparators("relative/path"))
.toBe("relative/path");
});
test("Multiple consecutive backslashes", () => {
expect(normalizePathSeparators("path\\\\to\\\\file"))
.toBe("path//to//file");
expect(normalizePathSeparators("C:\\\\Users\\\\name"))
.toBe("C://Users//name");
});
test("Edge cases", () => {
expect(normalizePathSeparators("")).toBe("");
expect(normalizePathSeparators("\\")).toBe("/");
expect(normalizePathSeparators("\\\\")).toBe("//");
expect(normalizePathSeparators("file.txt")).toBe("file.txt");
});
});
// ===========================================================================
// getRelativePathFromPrefix
// ===========================================================================
describe("getRelativePathFromPrefix", () => {
test("Exact match (path equals prefix)", () => {
expect(getRelativePathFromPrefix("/home/user", "/home/user")).toBe("");
expect(getRelativePathFromPrefix("C:/Users/name", "C:/Users/name")).toBe("");
expect(getRelativePathFromPrefix("/path", "/path")).toBe("");
});
test("Path under prefix", () => {
expect(getRelativePathFromPrefix("/home/user/documents", "/home/user"))
.toBe("documents");
expect(getRelativePathFromPrefix("/home/user/documents/file.txt", "/home/user"))
.toBe("documents/file.txt");
expect(getRelativePathFromPrefix("C:/Users/name/Documents/file.txt", "C:/Users/name"))
.toBe("Documents/file.txt");
});
test("Path not under prefix", () => {
expect(getRelativePathFromPrefix("/home/other", "/home/user")).toBeNull();
expect(getRelativePathFromPrefix("/usr/local", "/home/user")).toBeNull();
expect(getRelativePathFromPrefix("C:/Users/other", "D:/Users")).toBeNull();
});
test("Windows paths with normalized separators", () => {
// Backslashes should be normalized
expect(getRelativePathFromPrefix("C:\\Users\\name\\Documents", "C:\\Users\\name"))
.toBe("Documents");
expect(getRelativePathFromPrefix("C:\\Users\\name\\Documents\\file.txt", "C:/Users/name"))
.toBe("Documents/file.txt");
});
test("Prefix with trailing slash", () => {
expect(getRelativePathFromPrefix("/home/user/documents", "/home/user/"))
.toBe("documents");
expect(getRelativePathFromPrefix("C:/Users/name/Documents", "C:/Users/name/"))
.toBe("Documents");
});
test("Prefix without trailing slash", () => {
expect(getRelativePathFromPrefix("/home/user/documents", "/home/user"))
.toBe("documents");
expect(getRelativePathFromPrefix("C:/Users/name/Documents", "C:/Users/name"))
.toBe("Documents");
});
test("Edge cases", () => {
// Empty prefix
expect(getRelativePathFromPrefix("/path/to/file", "")).toBeNull();
// Path is prefix substring but not in hierarchy
expect(getRelativePathFromPrefix("/home/username", "/home/user")).toBeNull();
// Root prefix
expect(getRelativePathFromPrefix("/home/user", "/")).toBe("home/user");
});
});
// ===========================================================================
// resolve - Unix environment
// ===========================================================================
describe("resolve - Unix environment", () => {
beforeEach(() => {
mockPWD("/home/user");
});
test("Unix relative paths", () => {
expect(resolve("/base", "relative")).toBe("/base/relative");
expect(resolve("/base", "a/b/c")).toBe("/base/a/b/c");
expect(resolve("/home", "user/documents")).toBe("/home/user/documents");
});
test("Unix absolute paths", () => {
expect(resolve("/base", "/absolute")).toBe("/absolute");
expect(resolve("/home/user", "/usr/local")).toBe("/usr/local");
expect(resolve("/any", "/")).toBe("/");
});
test("Path with .. and .", () => {
expect(resolve("/base", "../other")).toBe("/other");
expect(resolve("/base/sub", "..")).toBe("/base");
expect(resolve("/base", "./file")).toBe("/base/file");
expect(resolve("/base/a/b", "../../c")).toBe("/base/c");
});
test("Multiple path segments", () => {
expect(resolve("/a", "b", "c")).toBe("/a/b/c");
expect(resolve("/a", "b", "../c")).toBe("/a/c");
expect(resolve("/a", "b", "/c")).toBe("/c");
});
test("Relative path without base (uses PWD)", () => {
expect(resolve("relative")).toBe("/home/user/relative");
expect(resolve("a/b/c")).toBe("/home/user/a/b/c");
expect(resolve("./file")).toBe("/home/user/file");
});
test("Absolute path alone", () => {
expect(resolve("/absolute/path")).toBe("/absolute/path");
expect(resolve("/")).toBe("/");
});
});
// ===========================================================================
// resolve - Windows environment
// ===========================================================================
describe("resolve - Windows environment", () => {
beforeEach(() => {
mockPWD("C:/Users/name");
});
test("Windows relative paths", () => {
expect(resolve("C:/base", "relative")).toBe("C:/base/relative");
expect(resolve("C:/base", "a/b/c")).toBe("C:/base/a/b/c");
expect(resolve("D:/Projects", "qmd/src")).toBe("D:/Projects/qmd/src");
});
test("Windows absolute paths", () => {
expect(resolve("C:/base", "D:/other")).toBe("D:/other");
expect(resolve("C:/Users", "C:/Program Files")).toBe("C:/Program Files");
expect(resolve("D:/any", "E:/other")).toBe("E:/other");
});
test("Windows with backslashes", () => {
expect(resolve("C:\\base", "relative")).toBe("C:/base/relative");
expect(resolve("C:\\Users\\name", "Documents")).toBe("C:/Users/name/Documents");
expect(resolve("C:\\base", "a\\b\\c")).toBe("C:/base/a/b/c");
});
test("Path with .. and .", () => {
expect(resolve("C:/base", "../other")).toBe("C:/other");
expect(resolve("C:/base/sub", "..")).toBe("C:/base");
expect(resolve("C:/base", "./file")).toBe("C:/base/file");
expect(resolve("C:/base/a/b", "../../c")).toBe("C:/base/c");
});
test("Multiple path segments", () => {
expect(resolve("C:/a", "b", "c")).toBe("C:/a/b/c");
expect(resolve("C:/a", "b", "../c")).toBe("C:/a/c");
expect(resolve("C:/a", "b", "D:/c")).toBe("D:/c");
});
test("Relative path without base (uses PWD)", () => {
expect(resolve("relative")).toBe("C:/Users/name/relative");
expect(resolve("a/b/c")).toBe("C:/Users/name/a/b/c");
expect(resolve(".\\file")).toBe("C:/Users/name/file");
});
test("Drive letter only", () => {
expect(resolve("C:")).toBe("C:/");
expect(resolve("D:")).toBe("D:/");
});
});
// ===========================================================================
// resolve - Git Bash style paths
// ===========================================================================
describe("resolve - Git Bash style paths", () => {
test("Git Bash to Windows conversion", () => {
expect(resolve("/c/Users/name")).toBe("C:/Users/name");
expect(resolve("/C/Users/name")).toBe("C:/Users/name");
expect(resolve("/d/Projects")).toBe("D:/Projects");
expect(resolve("/D/Projects")).toBe("D:/Projects");
});
test("Git Bash with relative paths", () => {
expect(resolve("/c/base", "relative")).toBe("C:/base/relative");
expect(resolve("/d/Projects", "qmd/src")).toBe("D:/Projects/qmd/src");
});
test("Git Bash with .. and .", () => {
expect(resolve("/c/base", "../other")).toBe("C:/other");
expect(resolve("/c/base/sub", "..")).toBe("C:/base");
expect(resolve("/c/base", "./file")).toBe("C:/base/file");
});
test("Multiple Git Bash segments", () => {
expect(resolve("/c/a", "b", "c")).toBe("C:/a/b/c");
expect(resolve("/c/a", "b", "/d/c")).toBe("D:/c");
});
});
// ===========================================================================
// resolve - Edge cases and mixed scenarios
// ===========================================================================
describe("resolve - Edge cases", () => {
test("Empty path segments are filtered", () => {
expect(resolve("/base", "", "file")).toBe("/base/file");
expect(resolve("C:/base", "", "file")).toBe("C:/base/file");
});
test("Multiple consecutive slashes", () => {
expect(resolve("/base//path///file")).toBe("/base/path/file");
expect(resolve("C:/base//path///file")).toBe("C:/base/path/file");
});
test("Trailing slashes", () => {
expect(resolve("/base/", "file")).toBe("/base/file");
expect(resolve("C:/base/", "file")).toBe("C:/base/file");
});
test("Complex .. navigation", () => {
expect(resolve("/a/b/c/d", "../../../e")).toBe("/a/e");
expect(resolve("C:/a/b/c/d", "../../../e")).toBe("C:/a/e");
});
test("Too many .. (should not go above root)", () => {
expect(resolve("/base", "../../../../other")).toBe("/other");
expect(resolve("C:/base", "../../../../other")).toBe("C:/other");
});
test("Mixed Unix and Windows (normalized)", () => {
mockPWD("C:/Users/name");
expect(resolve("/unix/path")).toBe("/unix/path");
expect(resolve("relative")).toBe("C:/Users/name/relative");
});
test("Error on no arguments", () => {
expect(() => resolve()).toThrow("resolve: at least one path segment is required");
});
});
});

View File

@ -1850,7 +1850,7 @@ describe("LlamaCpp Integration", () => {
expect(allResults).toHaveLength(2);
// Search with collection filter - should return only from collection1
const filtered = await store.searchVec("content", "embeddinggemma", 10, collection1 as unknown as number);
const filtered = await store.searchVec("content", "embeddinggemma", 10, collection1);
expect(filtered).toHaveLength(1);
expect(filtered[0]!.collectionName).toBe(collection1);

View File

@ -21,6 +21,7 @@ import {
formatQueryForEmbedding,
formatDocForEmbedding,
type RerankDocument,
type ILLMSession,
} from "./llm";
import {
findContextForPath as collectionsFindContextForPath,
@ -63,25 +64,171 @@ export function homedir(): string {
return HOME;
}
/**
* Check if a path is absolute.
* Supports:
* - Unix paths: /path/to/file
* - Windows native: C:\path or C:/path
* - Git Bash: /c/path or /C/path (C-Z drives, excluding A/B floppy drives)
*
* Note: /c without trailing slash is treated as Unix path (directory named "c"),
* while /c/ or /c/path are treated as Git Bash paths (C: drive).
*/
export function isAbsolutePath(path: string): boolean {
if (!path) return false;
// Unix absolute path
if (path.startsWith('/')) {
// Check if it's a Git Bash style path like /c/ or /c/Users (C-Z only, not A or B)
// Requires path[2] === '/' to distinguish from Unix paths like /c or /cache
if (path.length >= 3 && path[2] === '/') {
const driveLetter = path[1];
if (driveLetter && /[c-zC-Z]/.test(driveLetter)) {
return true;
}
}
// Any other path starting with / is Unix absolute
return true;
}
// Windows native path: C:\ or C:/ (any letter A-Z)
if (path.length >= 2 && /[a-zA-Z]/.test(path[0]!) && path[1] === ':') {
return true;
}
return false;
}
/**
* Normalize path separators to forward slashes.
* Converts Windows backslashes to forward slashes.
*/
export function normalizePathSeparators(path: string): string {
return path.replace(/\\/g, '/');
}
/**
* Get the relative path from a prefix.
* Returns null if path is not under prefix.
* Returns empty string if path equals prefix.
*/
export function getRelativePathFromPrefix(path: string, prefix: string): string | null {
// Empty prefix is invalid
if (!prefix) {
return null;
}
const normalizedPath = normalizePathSeparators(path);
const normalizedPrefix = normalizePathSeparators(prefix);
// Ensure prefix ends with / for proper matching
const prefixWithSlash = !normalizedPrefix.endsWith('/')
? normalizedPrefix + '/'
: normalizedPrefix;
// Exact match
if (normalizedPath === normalizedPrefix) {
return '';
}
// Check if path starts with prefix
if (normalizedPath.startsWith(prefixWithSlash)) {
return normalizedPath.slice(prefixWithSlash.length);
}
return null;
}
export function resolve(...paths: string[]): string {
if (paths.length === 0) {
throw new Error("resolve: at least one path segment is required");
}
let result = paths[0]!.startsWith('/') ? '' : Bun.env.PWD || process.cwd();
for (const p of paths) {
if (p.startsWith('/')) {
result = p;
// Normalize all paths to use forward slashes
const normalizedPaths = paths.map(normalizePathSeparators);
let result = '';
let windowsDrive = '';
// Check if first path is absolute
const firstPath = normalizedPaths[0]!;
if (isAbsolutePath(firstPath)) {
result = firstPath;
// Extract Windows drive letter if present
if (firstPath.length >= 2 && /[a-zA-Z]/.test(firstPath[0]!) && firstPath[1] === ':') {
windowsDrive = firstPath.slice(0, 2);
result = firstPath.slice(2);
} else if (firstPath.startsWith('/') && firstPath.length >= 3 && firstPath[2] === '/') {
// Git Bash style: /c/ -> C: (C-Z drives only, not A or B)
const driveLetter = firstPath[1];
if (driveLetter && /[c-zC-Z]/.test(driveLetter)) {
windowsDrive = driveLetter.toUpperCase() + ':';
result = firstPath.slice(2);
}
}
} else {
// Start with PWD or cwd, then append the first relative path
const pwd = normalizePathSeparators(Bun.env.PWD || process.cwd());
// Extract Windows drive from PWD if present
if (pwd.length >= 2 && /[a-zA-Z]/.test(pwd[0]!) && pwd[1] === ':') {
windowsDrive = pwd.slice(0, 2);
result = pwd.slice(2) + '/' + firstPath;
} else {
result = pwd + '/' + firstPath;
}
}
// Process remaining paths
for (let i = 1; i < normalizedPaths.length; i++) {
const p = normalizedPaths[i]!;
if (isAbsolutePath(p)) {
// Absolute path replaces everything
result = p;
// Update Windows drive if present
if (p.length >= 2 && /[a-zA-Z]/.test(p[0]!) && p[1] === ':') {
windowsDrive = p.slice(0, 2);
result = p.slice(2);
} else if (p.startsWith('/') && p.length >= 3 && p[2] === '/') {
// Git Bash style (C-Z drives only, not A or B)
const driveLetter = p[1];
if (driveLetter && /[c-zC-Z]/.test(driveLetter)) {
windowsDrive = driveLetter.toUpperCase() + ':';
result = p.slice(2);
} else {
windowsDrive = '';
}
} else {
windowsDrive = '';
}
} else {
// Relative path - append
result = result + '/' + p;
}
}
// Normalize . and .. components
const parts = result.split('/').filter(Boolean);
const normalized: string[] = [];
for (const part of parts) {
if (part === '..') normalized.pop();
else if (part !== '.') normalized.push(part);
if (part === '..') {
normalized.pop();
} else if (part !== '.') {
normalized.push(part);
}
}
return '/' + normalized.join('/');
// Build final path
const finalPath = '/' + normalized.join('/');
// Prepend Windows drive if present
if (windowsDrive) {
return windowsDrive + finalPath;
}
return finalPath;
}
// Flag to indicate production mode (set by qmd.ts at startup)
@ -473,7 +620,7 @@ export type Store = {
// Search
searchFTS: (query: string, limit?: number, collectionId?: number) => SearchResult[];
searchVec: (query: string, model: string, limit?: number, collectionId?: number) => Promise<SearchResult[]>;
searchVec: (query: string, model: string, limit?: number, collectionName?: string) => Promise<SearchResult[]>;
// Query expansion & reranking
expandQuery: (query: string, model?: string) => Promise<string[]>;
@ -556,7 +703,7 @@ export function createStore(dbPath?: string): Store {
// Search
searchFTS: (query: string, limit?: number, collectionId?: number) => searchFTS(db, query, limit, collectionId),
searchVec: (query: string, model: string, limit?: number, collectionId?: number) => searchVec(db, query, model, limit, collectionId),
searchVec: (query: string, model: string, limit?: number, collectionName?: string) => searchVec(db, query, model, limit, collectionName),
// Query expansion & reranking
expandQuery: (query: string, model?: string) => expandQuery(query, model, db),
@ -891,17 +1038,36 @@ export async function hashContent(content: string): Promise<string> {
return hash.digest("hex");
}
export function extractTitle(content: string, filename: string): string {
const match = content.match(/^##?\s+(.+)$/m);
if (match) {
const title = (match[1] ?? "").trim();
if (title === "📝 Notes" || title === "Notes") {
const nextMatch = content.match(/^##\s+(.+)$/m);
if (nextMatch?.[1]) return nextMatch[1].trim();
const titleExtractors: Record<string, (content: string) => string | null> = {
'.md': (content) => {
const match = content.match(/^##?\s+(.+)$/m);
if (match) {
const title = (match[1] ?? "").trim();
if (title === "📝 Notes" || title === "Notes") {
const nextMatch = content.match(/^##\s+(.+)$/m);
if (nextMatch?.[1]) return nextMatch[1].trim();
}
return title;
}
return title;
return null;
},
'.org': (content) => {
const titleProp = content.match(/^#\+TITLE:\s*(.+)$/im);
if (titleProp?.[1]) return titleProp[1].trim();
const heading = content.match(/^\*+\s+(.+)$/m);
if (heading?.[1]) return heading[1].trim();
return null;
},
};
export function extractTitle(content: string, filename: string): string {
const ext = filename.slice(filename.lastIndexOf('.')).toLowerCase();
const extractor = titleExtractors[ext];
if (extractor) {
const title = extractor(content);
if (title) return title;
}
return filename.replace(/\.md$/, "").split("/").pop() || filename;
return filename.replace(/\.[^.]+$/, "").split("/").pop() || filename;
}
// =============================================================================
@ -1735,11 +1901,11 @@ export function searchFTS(db: Database, query: string, limit: number = 20, colle
// Vector Search
// =============================================================================
export async function searchVec(db: Database, query: string, model: string, limit: number = 20, collectionId?: number): Promise<SearchResult[]> {
export async function searchVec(db: Database, query: string, model: string, limit: number = 20, collectionName?: string, session?: ILLMSession): Promise<SearchResult[]> {
const tableExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
if (!tableExists) return [];
const embedding = await getEmbedding(query, model, true);
const embedding = await getEmbedding(query, model, true, session);
if (!embedding) return [];
// IMPORTANT: We use a two-step query approach here because sqlite-vec virtual tables
@ -1778,9 +1944,9 @@ export async function searchVec(db: Database, query: string, model: string, limi
`;
const params: string[] = [...hashSeqs];
if (collectionId) {
if (collectionName) {
docSql += ` AND d.collection = ?`;
params.push(String(collectionId));
params.push(collectionName);
}
const docRows = db.prepare(docSql).all(...params) as {
@ -1825,11 +1991,12 @@ export async function searchVec(db: Database, query: string, model: string, limi
// Embeddings
// =============================================================================
async function getEmbedding(text: string, model: string, isQuery: boolean): Promise<number[] | null> {
const llm = getDefaultLlamaCpp();
async function getEmbedding(text: string, model: string, isQuery: boolean, session?: ILLMSession): Promise<number[] | null> {
// Format text using the appropriate prompt template
const formattedText = isQuery ? formatQueryForEmbedding(text) : formatDocForEmbedding(text);
const result = await llm.embed(formattedText, { model, isQuery });
const result = session
? await session.embed(formattedText, { model, isQuery })
: await getDefaultLlamaCpp().embed(formattedText, { model, isQuery });
return result?.embedding || null;
}