diff --git a/cmd/api/main.go b/cmd/api/main.go index 37f2b7b..9c01db9 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -15,6 +15,7 @@ import ( "xcontrol/server" "xcontrol/server/api" "xcontrol/server/config" + "xcontrol/server/proxy" "xcontrol/ui" ) @@ -24,6 +25,7 @@ func main() { slog.Warn("load config", "err", err) cfg = &config.Config{} } + proxy.Set(cfg.Global.Proxy) level := slog.LevelInfo switch strings.ToLower(cfg.Log.Level) { diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 52819b0..68404b4 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -11,24 +11,51 @@ import ( "path/filepath" "strings" "time" - - rconfig "xcontrol/server/rag/config" "xcontrol/server/rag/embed" "xcontrol/server/rag/ingest" "xcontrol/server/rag/store" rsync "xcontrol/server/rag/sync" + rconfig "xcontrol/server/rag/config" ) -// main performs cloning, parsing and embedding before sending documents to the server. +// main loads server RAG configuration and triggers a manual sync by +// calling the running API server's /api/rag/sync endpoint. When a file +// path is provided via -file, it parses the Markdown locally and emits +// chunk data as newline-delimited JSON. + func main() { - configPath := flag.String("config", "", "Path to RAG configuration file") + configPath := flag.String("config", "", "Path to server RAG configuration file") + filePath := flag.String("file", "", "Markdown file to parse and chunk") flag.Parse() - if *configPath == "" { - log.Fatalf("config path required") + + var cfg *rconfig.Config + var err error + if *configPath != "" { + cfg, err = rconfig.Load(*configPath) + if err != nil { + log.Fatalf("load config: %v", err) + } + } else { + cfg = &rconfig.Config{} } - cfg, err := rconfig.Load(*configPath) - if err != nil { - log.Fatalf("load config: %v", err) + + if *filePath != "" { + chunkCfg := cfg.ResolveChunking() + secs, err := ingest.ParseMarkdown(*filePath) + if err != nil { + log.Fatalf("parse markdown: %v", err) + } + chunks, err := ingest.BuildChunks(secs, chunkCfg) + if err != nil { + log.Fatalf("build chunks: %v", err) + } + enc := json.NewEncoder(os.Stdout) + for _, ch := range chunks { + if err := enc.Encode(ch); err != nil { + log.Fatalf("encode chunk: %v", err) + } + } + return } baseURL := os.Getenv("SERVER_URL") diff --git a/cmd/ingest/main.go b/cmd/ingest/main.go index 0e7b2a2..551df64 100644 --- a/cmd/ingest/main.go +++ b/cmd/ingest/main.go @@ -6,6 +6,7 @@ import ( "log" "runtime" + "xcontrol/server/proxy" cfgpkg "xcontrol/server/rag/config" "xcontrol/server/rag/ingest" ) @@ -23,6 +24,7 @@ func main() { if err != nil { log.Fatalf("load config: %v", err) } + proxy.Set(cfg.Global.Proxy) ctx := context.Background() opt := ingest.Options{MaxFiles: *maxFiles, DryRun: *dryRun, MigrateDim: *migrateDim, Concurrency: *concurrency} diff --git a/example/server/config/server.yaml b/example/server/config/server.yaml index 20f75c8..240273b 100644 --- a/example/server/config/server.yaml +++ b/example/server/config/server.yaml @@ -1,4 +1,5 @@ global: + proxy: socks5://127.0.0.1:1080 # optional redis: addr: "127.0.0.1:6379" password: "" diff --git a/go.mod b/go.mod index b093124..ff8c24e 100644 --- a/go.mod +++ b/go.mod @@ -5,15 +5,15 @@ go 1.23.0 toolchain go1.23.8 require ( - github.com/gin-gonic/gin v1.9.1 - github.com/go-git/go-git/v5 v5.16.2 - github.com/jackc/pgx/v5 v5.7.5 - github.com/redis/go-redis/v9 v9.12.0 - github.com/yuin/goldmark v1.7.13 - github.com/pgvector/pgvector-go v0.3.0 - github.com/pkoukk/tiktoken-go v0.1.7 - gopkg.in/yaml.v3 v3.0.1 - gorm.io/gorm v1.25.5 + github.com/gin-gonic/gin v1.9.1 + github.com/go-git/go-git/v5 v5.16.2 + github.com/jackc/pgx/v5 v5.7.5 + github.com/pgvector/pgvector-go v0.3.0 + github.com/redis/go-redis/v9 v9.12.0 + github.com/yuin/goldmark v1.7.13 + golang.org/x/net v0.39.0 + gopkg.in/yaml.v3 v3.0.1 + gorm.io/gorm v1.25.5 ) require ( @@ -26,7 +26,6 @@ require ( github.com/cloudflare/circl v1.6.1 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/dlclark/regexp2 v1.10.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -37,10 +36,8 @@ require ( github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect @@ -52,9 +49,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/pgvector/pgvector-go v0.3.0 // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect - github.com/pkoukk/tiktoken-go v0.1.7 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect @@ -62,8 +57,6 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.37.0 // indirect - golang.org/x/net v0.39.0 // indirect - golang.org/x/sync v0.13.0 // indirect golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.24.0 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum index 813f97a..242c255 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ= +entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= @@ -30,8 +32,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= -github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -52,6 +52,10 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.16.2 h1:fT6ZIOjE5iEnkzKyxTHK1W4HGAsPhqEqiSAssSO77hM= github.com/go-git/go-git/v5 v5.16.2/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8= +github.com/go-pg/pg/v10 v10.11.0 h1:CMKJqLgTrfpE/aOVeLdybezR2om071Vh38OLZjsyMI0= +github.com/go-pg/pg/v10 v10.11.0/go.mod h1:4BpHRoxE61y4Onpof3x1a2SQvi9c+q1dJnrNdMjsroA= +github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= +github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -83,6 +87,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -99,6 +105,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -116,8 +124,6 @@ github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkoukk/tiktoken-go v0.1.7 h1:qOBHXX4PHtvIvmOtyg1EeKlwFRiMKAcoMp4Q+bLQDmw= -github.com/pkoukk/tiktoken-go v0.1.7/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.12.0 h1:XlVPGlflh4nxfhsNXPA8Qp6EmEfTo0rp8oaBzPipXnU= @@ -143,10 +149,28 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= +github.com/uptrace/bun v1.1.12/go.mod h1:NPG6JGULBeQ9IU6yHp7YGELRa5Agmd7ATZdz4tGZ6z0= +github.com/uptrace/bun/dialect/pgdialect v1.1.12 h1:m/CM1UfOkoBTglGO5CUTKnIKKOApOYxkcP2qn0F9tJk= +github.com/uptrace/bun/dialect/pgdialect v1.1.12/go.mod h1:Ij6WIxQILxLlL2frUBxUBOZJtLElD2QQNDcu/PWDHTc= +github.com/uptrace/bun/driver/pgdriver v1.1.12 h1:3rRWB1GK0psTJrHwxzNfEij2MLibggiLdTqjTtfHc1w= +github.com/uptrace/bun/driver/pgdriver v1.1.12/go.mod h1:ssYUP+qwSEgeDDS1xm2XBip9el1y9Mi5mTAvLoiADLM= +github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= +github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= @@ -194,8 +218,10 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= -gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo= +gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0= gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/server/api/rag.go b/server/api/rag.go index 1e620d0..fea5763 100644 --- a/server/api/rag.go +++ b/server/api/rag.go @@ -5,6 +5,7 @@ import ( "github.com/gin-gonic/gin" + "xcontrol/server/proxy" "xcontrol/server/rag" rconfig "xcontrol/server/rag/config" "xcontrol/server/rag/store" @@ -19,7 +20,11 @@ func initRAG() *rag.Service { if err != nil { return nil } - return rag.New(cfg.ToConfig()) + proxy.Set(cfg.Proxy) + svc := rag.New(cfg.ToConfig()) + go svc.Sync(context.Background()) + go svc.Watch(context.Background()) + return svc } // registerRAGRoutes wires the /api/rag upsert and query endpoints. diff --git a/server/config/config.go b/server/config/config.go index f0a8c00..ed23797 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -55,6 +55,7 @@ type Global struct { Redis Redis `yaml:"redis"` VectorDB VectorDB `yaml:"vectordb"` Datasources []Datasource `yaml:"datasources"` + Proxy string `yaml:"proxy"` } type LLM struct { diff --git a/server/proxy/proxy.go b/server/proxy/proxy.go new file mode 100644 index 0000000..356a6f3 --- /dev/null +++ b/server/proxy/proxy.go @@ -0,0 +1,45 @@ +package proxy + +import ( + "context" + gclient "github.com/go-git/go-git/v5/plumbing/transport/client" + ghttp "github.com/go-git/go-git/v5/plumbing/transport/http" + xproxy "golang.org/x/net/proxy" + "net" + "net/http" + "net/url" +) + +// Set configures global HTTP and go-git clients to route through the given proxy URL. +// The proxyURL may be in formats like "http://host:port" or "socks5://host:port". +func Set(proxyURL string) { + if proxyURL == "" { + return + } + u, err := url.Parse(proxyURL) + if err != nil { + return + } + tr := http.DefaultTransport.(*http.Transport).Clone() + switch u.Scheme { + case "socks5", "socks5h": + dialer, err := xproxy.FromURL(u, xproxy.Direct) + if err != nil { + return + } + tr.Proxy = nil + if d, ok := dialer.(xproxy.ContextDialer); ok { + tr.DialContext = d.DialContext + } else { + tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + return dialer.Dial(network, addr) + } + } + default: + tr.Proxy = http.ProxyURL(u) + } + http.DefaultTransport = tr + c := &http.Client{Transport: tr} + gclient.InstallProtocol("https", ghttp.NewClient(c)) + gclient.InstallProtocol("http", ghttp.NewClient(c)) +} diff --git a/server/rag/config/config.go b/server/rag/config/config.go index a809a44..e570f91 100644 --- a/server/rag/config/config.go +++ b/server/rag/config/config.go @@ -53,6 +53,7 @@ type Global struct { } `yaml:"redis"` VectorDB VectorDB `yaml:"vectordb"` Datasources []DataSource `yaml:"datasources"` + Proxy string `yaml:"proxy"` } // Provider defines an LLM provider which can also serve embeddings. diff --git a/server/rag/config/runtime.go b/server/rag/config/runtime.go index 0059a93..5121f71 100644 --- a/server/rag/config/runtime.go +++ b/server/rag/config/runtime.go @@ -79,6 +79,7 @@ type Runtime struct { } `yaml:"redis"` VectorDB VectorDB `yaml:"vectordb"` Datasources []DataSource `yaml:"datasources"` + Proxy string `yaml:"proxy"` } // LoadServer loads global configuration from server/config/server.yaml. @@ -106,5 +107,6 @@ func (rt *Runtime) ToConfig() *Config { c.Global.Redis = rt.Redis c.Global.VectorDB = rt.VectorDB c.Global.Datasources = rt.Datasources + c.Global.Proxy = rt.Proxy return &c }