From 5fd3078e3ccdf9cc8104d2f527d276239707ec5d Mon Sep 17 00:00:00 2001 From: Haitao Pan Date: Sun, 10 Aug 2025 16:22:53 +0800 Subject: [PATCH] add offline embedding server & model downloader --- .gitignore | 2 + docs/models_downloading.py | 154 +++++++++++++++++++++++++++++++++++ docs/offline_embed_server.py | 125 ++++++++++++++++++++++++++++ docs/overview.md | 33 ++++++++ docs/setup_macos_m4.sh | 53 ++++++++++++ server/config/server.yaml | 35 +++++--- ui/dist/index.html | 2 +- 7 files changed, 391 insertions(+), 13 deletions(-) create mode 100644 docs/models_downloading.py create mode 100644 docs/offline_embed_server.py create mode 100644 docs/setup_macos_m4.sh diff --git a/.gitignore b/.gitignore index a4cf52e..1b07b36 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ +models/ +hf_cache/ server/server/ diff --git a/docs/models_downloading.py b/docs/models_downloading.py new file mode 100644 index 0000000..bf2b837 --- /dev/null +++ b/docs/models_downloading.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +""" +models_downloading.py +- 优先级：CN 镜像 (hf-mirror.com) -> 代理(PROXY) -> 官方直连 +- 统一缓存：HF_HOME=./hf_cache（可被环境变量覆盖） +- 进度可见：启用 huggingface_hub 的 tqdm 进度 +- 幂等安全：本地目录已有关键文件则跳过下载 + +可选环境变量： +- MODEL_ID 默认 "BAAI/bge-m3" +- MODEL_DIR 默认 "models/bge-m3" +- HF_HOME 默认 "./hf_cache" +- PROXY 默认 "socks5h://127.0.0.1:1081"（留空表示不走代理） +- HF_ENDPOINT 手动指定镜像时可设置（脚本也会自动探测 cn mirror） +""" + +import os +import sys +from pathlib import Path + +# ---------- 配置 ---------- +MODEL_ID = os.getenv("MODEL_ID", "BAAI/bge-m3") +MODEL_DIR = Path(os.getenv("MODEL_DIR", "models/bge-m3")) +HF_HOME = Path(os.getenv("HF_HOME", Path.cwd() / "hf_cache")) + +CN_MIRROR = "https://hf-mirror.com" +PROXY = os.getenv("PROXY", "socks5h://127.0.0.1:1081") + +# ---------- 提前设置缓存目录（在 import 前） ---------- +os.environ["HF_HOME"] = str(HF_HOME) + +# ---------- 依赖安装 ---------- +def _install(pkgs: str): + os.system(f"{sys.executable} -m pip install -U {pkgs}") + +try: + import requests +except ImportError: + _install("requests") + import requests + +# 若走 socks 代理需要 PySocks +if PROXY and "socks" in PROXY: + try: + import socks # noqa: F401 + except ImportError: + _install("'requests[socks]'") + +# ---------- 选择网络模式（镜像 → 代理 → 官方） ---------- +def set_network_mode(): + # 若外部已设置 HF_ENDPOINT，尊重外部配置 + if os.getenv("HF_ENDPOINT"): + print(f"🌏 Using custom HF endpoint: {os.getenv('HF_ENDPOINT')}") + return + + # 1) 尝试 CN 镜像 + try: + r = requests.get(CN_MIRROR, timeout=2) + if r.status_code == 200: + os.environ["HF_ENDPOINT"] = CN_MIRROR + print(f"🌏 Using Hugging Face CN mirror: {CN_MIRROR}") + return + except Exception: + pass + + # 2) 走代理 + if PROXY: + os.environ["HTTP_PROXY"] = PROXY + os.environ["HTTPS_PROXY"] = PROXY + print(f"🌐 Using proxy: {PROXY}") + return + + # 3) 官方直连 + print("⚠️ No mirror or proxy, using official huggingface.co") + +set_network_mode() + +# 现在再导入 huggingface_hub，确保拿到正确的 endpoint/proxy 设置 +try: + from huggingface_hub import snapshot_download +except ImportError: + _install("'huggingface_hub[tqdm]'") + from huggingface_hub import snapshot_download + +# ---------- 工具函数 ---------- +KEY_FILES = ( + "tokenizer.json", + "config.json", + "sentencepiece.bpe.model", + "onnx/model.onnx", + "pytorch_model.bin", + "model.safetensors", +) + +def has_local_model(root: Path) -> bool: + if not root.exists(): + return False + for k in KEY_FILES: + if any(root.rglob(k)): + return True + # 兜底：只要非空也算有内容（对应部分仓库布局） + return any(root.iterdir()) + +# ---------- 主流程 ---------- +def main(): + print("⬇️ Downloading model from Hugging Face…") + print(f" Model ID : {MODEL_ID}") + print(f" Save dir : {MODEL_DIR}") + print(f" HF_HOME : {HF_HOME}") + if os.getenv("HF_ENDPOINT"): + print(f" Endpoint : {os.getenv('HF_ENDPOINT')}") + elif os.getenv("HTTP_PROXY"): + print(f" Proxy : {os.getenv('HTTP_PROXY')}") + else: + print(" Endpoint : official (huggingface.co)") + + MODEL_DIR.parent.mkdir(parents=True, exist_ok=True) + HF_HOME.mkdir(parents=True, exist_ok=True) + + # 已有可用文件 → 跳过下载 + if has_local_model(MODEL_DIR): + print(f"📂 Local model exists, skip download: {MODEL_DIR}") + print("💡 To force re-download, remove the folder and rerun.") + return + + # 下载（显示进度） + try: + snapshot_download( + repo_id=MODEL_ID, + local_dir=str(MODEL_DIR), + local_dir_use_symlinks=False, + tqdm_class=None, # 使用默认 tqdm 进度条 + ) + except Exception as e: + # 失败时检查是否已经有部分或全部文件 + if has_local_model(MODEL_DIR): + print(f"⚠️ Online fetch failed but local files exist: {MODEL_DIR}") + print(f" Error: {e}") + else: + print("❌ Download failed and no local files found.") + print(f" Error: {e}") + print("🔁 Try: 1) 切换镜像/代理 2) 检查网络 3) 稍后重试") + sys.exit(1) + + # 最终确认 + if has_local_model(MODEL_DIR): + print(f"✅ Model cached to {MODEL_DIR}") + print("💡 To run offline later, set: export HF_HUB_OFFLINE=1") + else: + print("❌ No model files found after download attempt.") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/docs/offline_embed_server.py b/docs/offline_embed_server.py new file mode 100644 index 0000000..de36d9b --- /dev/null +++ b/docs/offline_embed_server.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +""" +Offline Embedding Server (Flask + fastembed) + +职责：仅加载本地模型目录并提供 /v1/embeddings 服务。 +下载请使用 docs/models_downloading.py（已含镜像/代理逻辑）。 + +环境变量（可选）： +- MODEL_ID 默认 "BAAI/bge-m3"（仅用于返回值展示） +- BGE_M3_DIR 默认 "models/bge-m3"（本地已下载的模型目录） +- HF_HOME 默认 "./hf_cache"（本地缓存；离线可用） +- EMBED_HOST 默认 "0.0.0.0" +- EMBED_PORT 默认 "9000" +""" + +import os +import sys +import numpy as np +from pathlib import Path + +# ---------------- 配置 ---------------- +MODEL_ID = os.getenv("MODEL_ID", "BAAI/bge-m3") +MODEL_DIR = Path(os.getenv("BGE_M3_DIR", "models/bge-m3")) +HF_HOME = Path(os.getenv("HF_HOME", Path.cwd() / "hf_cache")) +HOST = os.getenv("EMBED_HOST", "0.0.0.0") +PORT = int(os.getenv("EMBED_PORT", 9000)) + +# 关键文件（用于判定目录是否可用） +KEY_FILES = ("tokenizer.json", "config.json", "sentencepiece.bpe.model") + +# -------------- 依赖处理 -------------- +def _pip_install(pkgs: str): + os.system(f"{sys.executable} -m pip install -U {pkgs}") + +try: + from flask import Flask, request, jsonify +except ImportError: + _pip_install("flask") + from flask import Flask, request, jsonify + +try: + from fastembed import TextEmbedding +except ImportError: + _pip_install("fastembed") + from fastembed import TextEmbedding + +try: + import numpy as _np # 仅用于确认已装 +except ImportError: + _pip_install("numpy") + import numpy as _np # noqa + +# -------------- 校验本地模型 -------------- +def has_local_model(p: Path) -> bool: + if not p.exists(): + return False + # 任一关键文件存在即认为可用；或目录非空兜底 + for k in KEY_FILES: + if any(p.rglob(k)): + return True + return any(p.iterdir()) + +if not has_local_model(MODEL_DIR): + print(f"❌ Model not found or incomplete in: {MODEL_DIR}") + print(" 请先执行下载：python docs/models_downloading.py") + sys.exit(1) + +# -------------- 设置离线运行 -------------- +os.environ["HF_HOME"] = str(HF_HOME) # 统一缓存目录（Mac/Linux 一致） +os.environ["HF_HUB_OFFLINE"] = "1" # 强制完全离线 +HF_HOME.mkdir(parents=True, exist_ok=True) + +# -------------- 启动服务 -------------- +app = Flask(__name__) +model = TextEmbedding(str(MODEL_DIR)) # fastembed 直接从本地目录加载 + +@app.post("/v1/embeddings") +def embeddings(): + data = request.get_json(force=True) or {} + inp = data.get("input", []) + if isinstance(inp, str): + texts = [inp] + elif isinstance(inp, list): + # 过滤保证都是字符串 + texts = [str(x) for x in inp] + else: + return jsonify({"error": "invalid input type"}), 400 + + # fastembed 默认产出已归一向量；这里再 L2 保底 + vecs = [] + for v in model.embed(texts): + v = np.asarray(v, dtype=np.float32) + v = v / (np.linalg.norm(v) + 1e-12) + vecs.append(v.tolist()) + + return jsonify({ + "object": "list", + "data": [ + {"object": "embedding", "index": i, "embedding": e} + for i, e in enumerate(vecs) + ], + "model": data.get("model", MODEL_ID), + }) + +@app.get("/healthz") +def healthz(): + # 存在即健康 + return "ok", 200 + +@app.get("/readyz") +def readyz(): + # 模型已加载即就绪 + try: + _ = model # 触发引用 + return "ready", 200 + except Exception as e: + return f"not ready: {e}", 503 + +if __name__ == "__main__": + print(f"🚀 Embedding server") + print(f" Model Dir : {MODEL_DIR}") + print(f" HF_HOME : {HF_HOME}") + print(f" Offline : {os.environ.get('HF_HUB_OFFLINE')}") + print(f" Listen on : http://{HOST}:{PORT}") + app.run(host=HOST, port=PORT) diff --git a/docs/overview.md b/docs/overview.md index 4917731..8811874 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -136,3 +136,36 @@ make init-db 使用 Markdown 编写（支持标题、列表、代码块等）。 可使用 plantuml 或 mermaid 绘制架构图并嵌入 Markdown。 + +## DEV + +1. 运行（首次会自动下载模型） +python offline_embed_server.py +2. 测试接口 +编辑 +curl -s http://127.0.0.1:9000/v1/embeddings \ + -H "Content-Type: application/json" \ + -d '{"model":"BAAI/bge-m3","input":["你好","PGVector 怎么建 HNSW？"]}' | jq . +3. 环境变量（可选） +export BGE_M3_DIR="/path/to/bge-m3" +export EMBED_HOST="127.0.0.1" +export EMBED_PORT=9100 +python offline_embed_server.py + +## Ollama API test + +用流式接收（推荐）： + +curl http://127.0.0.1:11434/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-oss:20b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Tell me three tips for optimizing HNSW in PostgreSQL."} + ], + "max_tokens": 512, + "stream": true + }' +这样会实时输出分块数据 + diff --git a/docs/setup_macos_m4.sh b/docs/setup_macos_m4.sh new file mode 100644 index 0000000..0a2cc06 --- /dev/null +++ b/docs/setup_macos_m4.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "==> 1. Xcode Command Line Tools" +xcode-select -p >/dev/null 2>&1 || xcode-select --install || true + +echo "==> 2. Homebrew" +if ! command -v brew >/dev/null 2>&1; then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> ~/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" +fi + +echo "==> 3. 基础工具" +brew update +brew install git gh wget curl jq cmake pkg-config tree htop tmux + +echo "==> 4. Go / Node / Yarn" +brew install go +# Node 推荐用 corepack 管理（pnpm/yarn） +brew install node +corepack enable || true +corepack prepare yarn@stable --activate || true + +echo "==> 5. PostgreSQL + pgvector" +brew install postgresql@16 +brew services start postgresql@16 +# pgvector 扩展（Homebrew 版已包含或单独提供） +brew install pgvector || true + +echo "==> 6. Redis" +brew install redis +brew services start redis + +echo "==> 7. Python 与虚拟环境" +brew install python@3.12 +python3 -m venv ~/.venvs/xcontrol && source ~/.venvs/xcontrol/bin/activate +pip install -U pip wheel + +echo "==> 8. RAG: fastembed + Flask（做本地 /v1/embeddings）" +pip install -U fastembed flask numpy huggingface_hub + +echo "==> 9. （可选）PyTorch + MPS（Apple GPU 加速，用于 Transformers 生成）" +# 官方 pip 已支持 MPS，一般直接安装即可（若失败可按官网指引重装） +pip install -U torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu + +echo "==> 10. （可选）Ollama（本地生成模型）" +if ! command -v ollama >/dev/null 2>&1; then + curl -fsSL https://ollama.com/install.sh | sh +fi + +echo "==> 完成 ✅ 请重新打开终端或执行：" +echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' diff --git a/server/config/server.yaml b/server/config/server.yaml index b0cc851..2f49b0b 100644 --- a/server/config/server.yaml +++ b/server/config/server.yaml @@ -19,22 +19,33 @@ sync: repo: proxy: socks5://127.0.0.1:1080 # 仅在同步仓库时使用代理 -provider: - - name: allama - base_url: http://localhost:11434 - token: "" +# For DEV +models: + embedder: + provider: "huggingface_hub" + models: "bge-m3" + endpoint: "http://127.0.0.1:9000/v1/embeddings" + generator: + provider: "ollama" models: - 'gpt-oss:20b' - - name: chutes - base_url: https://llm.chutes.ai - token: "cpk_xxxxxxxxxxxxxxxxxxxx" - models: - - 'moonshotai/Kimi-K2-Instruct' + endpoint: "http://127.0.0.1:11434/v1/chat/completions" + token: "" +# For PROD +#models: +# embedder: + #provider: "chutes" + #models: "bge-m3" + #endpoint: "https://chutes-baai-bge-m3.chutes.ai/embed/v1/embeddings" + #token: "cpk_xxxxxxxxxxxxxxxxxxxx" +# generator: + #provider: "chutes" + #endpoint: "https://llm.chutes.ai/v1/chat/completions" + #token: "cpk_xxxxxxxxxxxxxxxxxxxx" + #models: + # - 'moonshotai/Kimi-K2-Instruct' embedding: - base_url: http://localhost:11434 - token: "" - models: bge-m3 max_batch: 64 dimension: 1024 #维度 max_chars: 8000 diff --git a/ui/dist/index.html b/ui/dist/index.html index 0fb263c..dc8addf 100644 --- a/ui/dist/index.html +++ b/ui/dist/index.html @@ -1 +1 @@ -
XControl Admin
admin@example.com
A

Dashboard

Welcome to XControl admin panel.

\ No newline at end of file +
XControl Admin
admin@example.com
A

Dashboard

Welcome to XControl admin panel.

\ No newline at end of file