diff --git a/CHANGELOG.md b/CHANGELOG.md index 700a09b..6870d4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Doctor: explicitly warn when `content_vectors` contains multiple non-empty embedding fingerprint names, with the per-fingerprint document/chunk breakdown. - Embed: make the TTY progress line label byte-based input progress explicitly, show embedded chunks as a count, and shorten the displayed model name. - Embed: retain per-chunk failure details, retry failed chunks after later successful embeds and again when no other chunks remain, clear recovered errors, and cap retries to avoid endless loops. +- Tests: expand the container smoke harness to cover npm-global, npx-style, and Bun-global install scenarios, always checking auto and `QMD_FORCE_CPU=1` doctor modes, with opt-in tiny `qmd embed` and GPU probe runs for supported container runtimes. - Embedding: fingerprint vector metadata using the active embedding model and formatting/chunking parameters so stale vectors are treated as pending after search semantics change. Legacy `content_vectors` columns are migrated lazily on first vector-health/write use to preserve fast QMD startup. - Skill: expand the packaged QMD skill with retrieval-first workflows, structured query examples, wiki/source collection guidance, and safe fallbacks when model-backed search is unavailable. diff --git a/test/Containerfile b/test/Containerfile index 28f32d0..a413bfa 100644 --- a/test/Containerfile +++ b/test/Containerfile @@ -13,10 +13,12 @@ ENV PATH="/root/.local/bin:$PATH" # Pre-install node and bun RUN mise use -g node@latest bun@latest -# Copy the packed tarball and install via both package managers -COPY tobilu-qmd-*.tgz /tmp/ -RUN mise exec node@latest -- npm install -g /tmp/tobilu-qmd-*.tgz -RUN mise exec bun@latest -- bun install -g /tmp/tobilu-qmd-*.tgz +# Copy the packed tarball and install via both package managers. Keep a stable +# tarball path for npm-exec/npx-style smoke scenarios. +COPY tobilu-qmd-*.tgz /tmp/qmd-package.tgz +RUN cp /tmp/qmd-package.tgz /tmp/tobilu-qmd.tgz +RUN mise exec node@latest -- npm install -g /tmp/qmd-package.tgz +RUN mise exec bun@latest -- bun install -g /tmp/qmd-package.tgz # Copy test project (src + test + configs) and install deps COPY test-src/ /opt/qmd/ diff --git a/test/smoke-install.sh b/test/smoke-install.sh index 9acf2b7..1674dfb 100755 --- a/test/smoke-install.sh +++ b/test/smoke-install.sh @@ -1,17 +1,28 @@ #!/usr/bin/env bash -# Build a container image with qmd installed via npm and bun, then run smoke tests. -# Works with docker or podman (whichever is available). +# Build a clean container image from the current checkout package and exercise +# install/runtime scenarios under npm, npx, and Bun. Supports optional qmd embed +# and GPU probes, but keeps those expensive/device-specific checks opt-in. # # Usage: -# test/smoke-install.sh # build + run all smoke tests -# test/smoke-install.sh --build # build image only -# test/smoke-install.sh --shell # drop into container shell -# test/smoke-install.sh -- CMD... # run arbitrary command in container +# test/smoke-install.sh # build + run default smoke scenarios +# test/smoke-install.sh --build # build image only +# test/smoke-install.sh --shell # drop into container shell +# test/smoke-install.sh --scenario node # run one scenario (node|npx|bun|all) +# test/smoke-install.sh --with-embed # also run tiny qmd embed smoke tests +# test/smoke-install.sh --with-gpu # also probe GPU in doctor/embed scenarios +# QMD_SMOKE_GPU_BACKEND=cuda|vulkan|auto # backend for --with-gpu (default: auto) +# test/smoke-install.sh --no-build # reuse existing image +# test/smoke-install.sh -- CMD... # run arbitrary command in container +# +# GPU notes: +# Docker uses: --gpus all +# Podman uses: --device nvidia.com/gpu=all +# If your podman setup uses a different CDI device name, override with: +# QMD_SMOKE_GPU_ARGS='--device nvidia.com/gpu=all' test/smoke-install.sh --with-gpu set -euo pipefail cd "$(dirname "$0")/.." -# Pick container runtime if command -v podman &>/dev/null; then CTR=podman elif command -v docker &>/dev/null; then @@ -21,10 +32,50 @@ else exit 1 fi -IMAGE=qmd-smoke +IMAGE=${QMD_SMOKE_IMAGE:-qmd-smoke} +SCENARIO=all +DO_BUILD=1 +WITH_EMBED=0 +WITH_GPU=0 +GPU_BACKEND=${QMD_SMOKE_GPU_BACKEND:-auto} +declare -a ARBITRARY_CMD=() + +usage() { + sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//' +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --build) DO_BUILD=1; BUILD_ONLY=1; shift ;; + --no-build) DO_BUILD=0; shift ;; + --shell) SHELL_ONLY=1; shift ;; + --scenario) SCENARIO="${2:-}"; shift 2 ;; + --with-embed) WITH_EMBED=1; shift ;; + --with-gpu) WITH_GPU=1; shift ;; + --help|-h) usage; exit 0 ;; + --) shift; ARBITRARY_CMD=("$@"); break ;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 1 ;; + esac +done + +BUILD_ONLY=${BUILD_ONLY:-0} +SHELL_ONLY=${SHELL_ONLY:-0} + +gpu_args() { + if [[ $WITH_GPU -ne 1 ]]; then return 0; fi + if [[ -n "${QMD_SMOKE_GPU_ARGS:-}" ]]; then + # shellcheck disable=SC2206 + echo ${QMD_SMOKE_GPU_ARGS} + return 0 + fi + case "$CTR" in + docker) echo "--gpus all" ;; + podman) echo "--device nvidia.com/gpu=all" ;; + esac +} build_image() { - echo "==> Building TypeScript..." + echo "==> Building TypeScript package..." npm run build --silent echo "==> Packing tarball..." @@ -32,32 +83,35 @@ build_image() { TARBALL=$(npm pack --pack-destination test/ 2>/dev/null | tail -1) echo " $TARBALL" - # Copy project files into build context so vitest/bun tests can run inside + echo "==> Preparing container test project..." rm -rf test/test-src - mkdir -p test/test-src/src test/test-src/test - cp src/*.ts test/test-src/src/ + mkdir -p test/test-src/test + cp -r src test/test-src/ cp -r dist test/test-src/ - cp test/*.test.ts test/test-src/test/ + cp -r test/*.test.ts test/test-src/test/ cp package.json tsconfig.json tsconfig.build.json test/test-src/ - echo "==> Building container image ($CTR)..." + echo "==> Building container image ($CTR): $IMAGE" $CTR build -f test/Containerfile -t "$IMAGE" test/ - # Clean up rm -f test/tobilu-qmd-*.tgz rm -rf test/test-src echo "==> Image ready: $IMAGE" } run() { - $CTR run --rm "$IMAGE" bash -c "$*" + local args=() + # Intentionally word-split GPU args: container CLIs expect separate flags. + # shellcheck disable=SC2206 + args=( $(gpu_args) ) + $CTR run --rm "${args[@]}" "$IMAGE" bash -lc "$*" } PASS=0 FAIL=0 -ok() { printf " %-50s OK\n" "$1"; PASS=$((PASS + 1)); } -fail() { printf " %-50s FAIL\n" "$1"; FAIL=$((FAIL + 1)); echo "$2" | sed 's/^/ /'; } +ok() { printf " %-58s OK\n" "$1"; PASS=$((PASS + 1)); } +fail() { printf " %-58s FAIL\n" "$1"; FAIL=$((FAIL + 1)); echo "$2" | sed 's/^/ /'; } smoke_test() { local label="$1"; shift @@ -73,97 +127,136 @@ smoke_test_output() { local label="$1"; local expect="$2"; shift 2 local out out=$(run "$@" 2>&1) || true - if echo "$out" | grep -q "$expect"; then + if grep -q "$expect" <<<"$out"; then ok "$label" else fail "$label" "$out" fi } -run_smoke_tests() { - # ------------------------------------------------------------------ - # Node (npm-installed qmd) - # ------------------------------------------------------------------ +fixture_setup='rm -rf /tmp/qmd-fixture /tmp/qmd-cache /tmp/qmd-config /tmp/qmd-models; mkdir -p /tmp/qmd-fixture; printf "# Smoke Doc\n\nGPU and CPU embedding smoke test.\n" > /tmp/qmd-fixture/doc.md; export XDG_CACHE_HOME=/tmp/qmd-cache QMD_CONFIG_DIR=/tmp/qmd-config' + +gpu_env() { + case "$GPU_BACKEND" in + auto|"") echo "" ;; + cuda|vulkan|metal) echo "QMD_LLAMA_GPU=$GPU_BACKEND" ;; + *) echo "Unsupported QMD_SMOKE_GPU_BACKEND=$GPU_BACKEND" >&2; exit 1 ;; + esac +} + +run_doctor_smoke() { + local label="$1" bin="$2" extra_env="${3:-}" + smoke_test_output "$label doctor" "QMD Doctor" \ + "$fixture_setup; $extra_env $bin doctor" +} + +run_collection_smoke() { + local label="$1" bin="$2" extra_env="${3:-}" + smoke_test "$label collection add/list/status" \ + "$fixture_setup; cd /tmp/qmd-fixture; $extra_env $bin collection add . --name smoke; $extra_env $bin collection list; $extra_env $bin status" +} + +run_embed_smoke() { + local label="$1" bin="$2" extra_env="${3:-}" + [[ $WITH_EMBED -eq 1 ]] || return 0 + smoke_test "$label qmd embed tiny fixture" \ + "$fixture_setup; cd /tmp/qmd-fixture; $extra_env $bin collection add . --name smoke; $extra_env $bin embed --max-docs-per-batch 1 --max-batch-mb 1; $extra_env $bin doctor" +} + +run_runtime_matrix() { + local label="$1" bin="$2" path_env="$3" + smoke_test_output "$label qmd help" "Usage:" "$path_env; $bin" + run_doctor_smoke "$label auto" "$path_env; $bin" + run_doctor_smoke "$label force-cpu" "$path_env; $bin" "QMD_FORCE_CPU=1" + run_collection_smoke "$label" "$path_env; $bin" "QMD_FORCE_CPU=1" + run_embed_smoke "$label force-cpu" "$path_env; $bin" "QMD_FORCE_CPU=1" + run_embed_smoke "$label auto" "$path_env; $bin" + if [[ $WITH_GPU -eq 1 ]]; then + local ge + ge=$(gpu_env) + run_doctor_smoke "$label gpu-$GPU_BACKEND" "$path_env; $bin" "$ge" + run_embed_smoke "$label gpu-$GPU_BACKEND" "$path_env; $bin" "$ge" + fi +} + +run_node_scenario() { local NODE_BIN='$(mise where node@latest)/bin' - echo "=== Node (npm install) ===" - - smoke_test_output "qmd shows help" "Usage:" \ - "export PATH=$NODE_BIN:\$PATH; qmd" - - smoke_test "qmd collection list" \ - "export PATH=$NODE_BIN:\$PATH; qmd collection list" - - smoke_test "qmd status" \ - "export PATH=$NODE_BIN:\$PATH; qmd status" - - smoke_test "sqlite-vec loads" \ - "export PATH=$NODE_BIN:\$PATH; - NPM_GLOBAL=\$(npm root -g); - node -e \" - const {openDatabase, loadSqliteVec} = await import('\$NPM_GLOBAL/@tobilu/qmd/dist/db.js'); + local bin='qmd' + echo "=== Node: npm install -g packed tarball ===" + run_runtime_matrix "node" "$bin" "export PATH=$NODE_BIN:\$PATH" + smoke_test "node sqlite-vec loads" \ + "export PATH=$NODE_BIN:\$PATH; NPM_GLOBAL=\$(npm root -g); node -e \" + const {openDatabase, loadSqliteVec} = await import('\\$NPM_GLOBAL/@tobilu/qmd/dist/db.js'); const db = openDatabase(':memory:'); loadSqliteVec(db); const r = db.prepare('SELECT vec_version() as v').get(); console.log('sqlite-vec', r.v); if (!r.v) process.exit(1); \"" - - smoke_test "vitest (node)" \ + smoke_test "node vitest store subset" \ "export PATH=$NODE_BIN:\$PATH; cd /opt/qmd && npx vitest run --reporter=verbose test/store.test.ts 2>&1 | tail -5" +} - # ------------------------------------------------------------------ - # Bun (bun-installed qmd) - # ------------------------------------------------------------------ +run_npx_scenario() { + local NODE_BIN='$(mise where node@latest)/bin' + local bin='npm exec --yes --package /tmp/tobilu-qmd.tgz -- qmd' + echo "=== Node: npm exec/npx-style packed tarball ===" + run_runtime_matrix "npx-style" "$bin" "export PATH=$NODE_BIN:\$PATH" +} + +run_bun_scenario() { + local NODE_BIN='$(mise where node@latest)/bin' local BUN_BIN='$(mise where bun@latest)/bin' - echo "" - echo "=== Bun (bun install) ===" - - smoke_test_output "qmd shows help" "Usage:" \ - "export PATH=$BUN_BIN:$NODE_BIN:\$PATH; \$HOME/.bun/bin/qmd" - - smoke_test "qmd collection list" \ - "export PATH=$BUN_BIN:$NODE_BIN:\$PATH; \$HOME/.bun/bin/qmd collection list" - - smoke_test "qmd status" \ - "export PATH=$BUN_BIN:$NODE_BIN:\$PATH; \$HOME/.bun/bin/qmd status" - - smoke_test "sqlite-vec loads (bun)" \ + local bin='$HOME/.bun/bin/qmd' + echo "=== Bun: bun install -g packed tarball ===" + run_runtime_matrix "bun" "$bin" "export PATH=$BUN_BIN:$NODE_BIN:\$PATH" + smoke_test "bun sqlite-vec loads" \ "export PATH=$BUN_BIN:\$PATH; bun -e \" - const {openDatabase, loadSqliteVec} = await import('\$HOME/.bun/install/global/node_modules/@tobilu/qmd/dist/db.js'); + const {openDatabase, loadSqliteVec} = await import('\\$HOME/.bun/install/global/node_modules/@tobilu/qmd/dist/db.js'); const db = openDatabase(':memory:'); loadSqliteVec(db); const r = db.prepare('SELECT vec_version() as v').get(); console.log('sqlite-vec', r.v); if (!r.v) process.exit(1); \"" - - smoke_test "bun test store" \ + smoke_test "bun test store subset" \ "export PATH=$BUN_BIN:\$PATH; cd /opt/qmd && bun test --preload ./src/test-preload.ts --timeout 30000 test/store.test.ts 2>&1 | tail -10" +} - # ------------------------------------------------------------------ +run_smoke_tests() { + case "$SCENARIO" in + node) run_node_scenario ;; + npx) run_npx_scenario ;; + bun) run_bun_scenario ;; + all) run_node_scenario; echo; run_npx_scenario; echo; run_bun_scenario ;; + *) echo "Unknown scenario: $SCENARIO" >&2; exit 1 ;; + esac echo "" echo "=== Results: $PASS passed, $FAIL failed ===" [[ $FAIL -eq 0 ]] } -# Parse arguments -case "${1:-}" in - --build) - build_image - ;; - --shell) - build_image - echo "==> Dropping into container shell..." - $CTR run --rm -it "$IMAGE" bash - ;; - --) - shift - run "$@" - ;; - *) - build_image - echo "" - echo "==> Running smoke tests..." - run_smoke_tests - ;; -esac +if [[ $DO_BUILD -eq 1 ]]; then + build_image +fi + +if [[ ${#ARBITRARY_CMD[@]} -gt 0 ]]; then + run "${ARBITRARY_CMD[*]}" + exit $? +fi + +if [[ $BUILD_ONLY -eq 1 ]]; then + exit 0 +fi + +if [[ $SHELL_ONLY -eq 1 ]]; then + echo "==> Dropping into container shell..." + # shellcheck disable=SC2206 + gpu=( $(gpu_args) ) + $CTR run --rm -it "${gpu[@]}" "$IMAGE" bash + exit $? +fi + +echo "" +echo "==> Running smoke tests..." +run_smoke_tests