ai-workspace-services/accounts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle placeholder Chutes token

Browse Source
This commit is contained in:
Haitao Pan 2025-08-09 20:13:21 +08:00
parent 5902ab8523
commit 69584ad4e3
2 changed files with 52 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
client/main.go
View File

@ -73,11 +73,14 @@ func main() {
embCfg := cfg.ResolveEmbedding()
chunkCfg := cfg.ResolveChunking()
embedder := embed.NewOpenAI(embCfg.BaseURL, embCfg.APIKey, embCfg.Model, embCfg.Dimension)
var syncErrs []string
for _, ds := range cfg.Global.Datasources {
workdir := filepath.Join(os.TempDir(), "xcontrol", ds.Name)
if _, err := rsync.SyncRepo(ctx, ds.Repo, workdir); err != nil {
log.Fatalf("sync repo %s: %v", ds.Name, err)
log.Printf("sync repo %s: %v", ds.Name, err)
syncErrs = append(syncErrs, ds.Name)
continue
}
root := filepath.Join(workdir, ds.Path)
files, err := ingest.ListMarkdown(root, chunkCfg.IncludeExts, chunkCfg.IgnoreDirs, 0)
@ -139,4 +142,7 @@ func main() {
}
log.Printf("ingested %d rows for %s", len(rows), ds.Name)
}
if len(syncErrs) > 0 {
log.Fatalf("failed to sync repositories: %s", strings.Join(syncErrs, ", "))
}
}

82
internal/rag/sync/sync.go
View File

@ -13,54 +13,62 @@ import (
// performs a shallow clone when the directory does not exist, otherwise a
// fetch and reset. The returned string is the current HEAD commit hash.
func SyncRepo(ctx context.Context, url, workdir string) (string, error) {
if _, err := os.Stat(workdir); os.IsNotExist(err) {
// shallow clone
_, err := git.PlainCloneContext(ctx, workdir, false, &git.CloneOptions{
URL: url,
Depth: 1,
})
if err != nil {
return "", err
attempt := func() (string, error) {
if _, err := os.Stat(workdir); os.IsNotExist(err) {
// shallow clone
_, err := git.PlainCloneContext(ctx, workdir, false, &git.CloneOptions{
URL: url,
Depth: 1,
})
if err != nil {
return "", err
}
} else {
r, err := git.PlainOpen(workdir)
if err != nil {
return "", err
}
// fetch
if err := r.FetchContext(ctx, &git.FetchOptions{Depth: 1}); err != nil && err != git.NoErrAlreadyUpToDate {
return "", err
}
// reset to origin/HEAD
head, err := r.ResolveRevision(plumbing.Revision("origin/HEAD"))
if err != nil {
// fallback to master/main
head, err = r.ResolveRevision(plumbing.Revision("origin/main"))
if err != nil {
head, err = r.ResolveRevision(plumbing.Revision("origin/master"))
if err != nil {
return "", err
}
}
}
w, err := r.Worktree()
if err != nil {
return "", err
}
if err := w.Reset(&git.ResetOptions{Mode: git.HardReset, Commit: *head}); err != nil {
return "", err
}
}
} else {
r, err := git.PlainOpen(workdir)
if err != nil {
return "", err
}
// fetch
if err := r.FetchContext(ctx, &git.FetchOptions{Depth: 1}); err != nil && err != git.NoErrAlreadyUpToDate {
return "", err
}
// reset to origin/HEAD
head, err := r.ResolveRevision(plumbing.Revision("origin/HEAD"))
if err != nil {
// fallback to master/main
head, err = r.ResolveRevision(plumbing.Revision("origin/main"))
if err != nil {
head, err = r.ResolveRevision(plumbing.Revision("origin/master"))
if err != nil {
return "", err
}
}
}
w, err := r.Worktree()
ref, err := r.Head()
if err != nil {
return "", err
}
if err := w.Reset(&git.ResetOptions{Mode: git.HardReset, Commit: *head}); err != nil {
return "", err
}
return ref.Hash().String(), nil
}
r, err := git.PlainOpen(workdir)
if err != nil {
return "", err
if hash, err := attempt(); err == nil {
return hash, nil
}
ref, err := r.Head()
if err != nil {
return "", err
}
return ref.Hash().String(), nil
_ = os.RemoveAll(workdir)
return attempt()
}
// WithAuth returns CloneOptions with basic auth if username/token provided in URL.