merged: docs/pglogical.md
This commit is contained in:
commit
2f47c1cfdd
@ -3,7 +3,8 @@ MAIN_FILE := ./cmd/accountsvc/main.go
|
||||
PORT ?= 8080
|
||||
OS := $(shell uname -s)
|
||||
SCHEMA_FILE := ./sql/schema.sql
|
||||
MIGRATION_FILE := ./sql/20251007-migrate_to_rbac.sql
|
||||
MIGRATION_FILES := $(shell ls -1 sql/migrations/*.up.sql 2>/dev/null | sort)
|
||||
|
||||
|
||||
DB_NAME := account
|
||||
DB_USER := shenlan
|
||||
@ -59,18 +60,24 @@ init-db:
|
||||
@echo "使用数据库连接: $(DB_URL)"
|
||||
@psql "$(DB_URL)" -v ON_ERROR_STOP=1 -f $(SCHEMA_FILE)
|
||||
|
||||
# =========================================
|
||||
# migrate-db target
|
||||
# =========================================
|
||||
verify-db:
|
||||
@echo ">>> 执行数据库 schema verify (golang-migrate)"
|
||||
@go run ./cmd/migratectl/main.go verify --schema sql/schema.sql --dsn "$(DB_URL)" --dir account/sql/migrations
|
||||
# =========================================
|
||||
# migrate-db target
|
||||
# =========================================
|
||||
migrate-db:
|
||||
@echo ">>> 执行数据库 schema 迁移 (level/role metadata & MFA/email verification)"
|
||||
@if [ ! -f $(MIGRATION_FILE) ]; then \
|
||||
echo "未找到迁移 SQL 文件: $(MIGRATION_FILE)"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if ! command -v psql >/dev/null 2>&1; then \
|
||||
echo "未检测到 psql,请先安装 PostgreSQL 客户端"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo ">>> 执行数据库 schema 迁移 (golang-migrate)"
|
||||
@echo "--------------------------------------------"
|
||||
@echo "使用数据库连接: $(DB_URL)"
|
||||
@psql "$(DB_URL)" -v ON_ERROR_STOP=1 -f $(MIGRATION_FILE)
|
||||
@if ! command -v go >/dev/null 2>&1; then \
|
||||
echo "未检测到 go,请先安装 Go 环境"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@go run ./cmd/migratectl/main.go migrate --dsn "$(DB_URL)" --dir sql/migrations
|
||||
|
||||
# 删除数据库(安全防呆)
|
||||
drop-db:
|
||||
|
||||
181
account/cmd/migratectl/main.go
Normal file
181
account/cmd/migratectl/main.go
Normal file
@ -0,0 +1,181 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"xcontrol/account/internal/migrate"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultMigrationDir = "account/sql/migrations"
|
||||
defaultSchemaFile = "account/sql/schema.sql"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
rootCmd := newRootCmd()
|
||||
if err := rootCmd.ExecuteContext(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func newRootCmd() *cobra.Command {
|
||||
var migrationDir string
|
||||
cmd := &cobra.Command{
|
||||
Use: "migratectl",
|
||||
Short: "XControl database migration orchestrator",
|
||||
}
|
||||
|
||||
migrationDir = defaultMigrationDir
|
||||
cmd.PersistentFlags().StringVar(&migrationDir, "dir", migrationDir, "directory containing migration files")
|
||||
|
||||
cmd.AddCommand(newMigrateCmd(&migrationDir))
|
||||
cmd.AddCommand(newCleanCmd())
|
||||
cmd.AddCommand(newCheckCmd())
|
||||
cmd.AddCommand(newVerifyCmd())
|
||||
cmd.AddCommand(newResetCmd(&migrationDir))
|
||||
cmd.AddCommand(newVersionCmd(&migrationDir))
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newMigrateCmd(dir *string) *cobra.Command {
|
||||
var dsn string
|
||||
cmd := &cobra.Command{
|
||||
Use: "migrate",
|
||||
Short: "Apply database migrations",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if dsn == "" {
|
||||
return errors.New("--dsn is required")
|
||||
}
|
||||
runner := migrate.NewRunner(*dir)
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
|
||||
defer cancel()
|
||||
return runner.Up(ctx, dsn)
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&dsn, "dsn", "", "PostgreSQL connection string")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newCleanCmd() *cobra.Command {
|
||||
var (
|
||||
dsn string
|
||||
force bool
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "clean",
|
||||
Short: "Clean leftover database structures",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if dsn == "" {
|
||||
return errors.New("--dsn is required")
|
||||
}
|
||||
cleaner := migrate.NewCleaner()
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
|
||||
defer cancel()
|
||||
return cleaner.Clean(ctx, dsn, force)
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&dsn, "dsn", "", "PostgreSQL connection string")
|
||||
cmd.Flags().BoolVar(&force, "force", false, "Confirm clean-up actions")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newCheckCmd() *cobra.Command {
|
||||
var (
|
||||
cnDSN string
|
||||
globalDSN string
|
||||
autoFix bool
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "check",
|
||||
Short: "Compare CN and Global schemas",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
checker := migrate.NewChecker()
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Minute)
|
||||
defer cancel()
|
||||
return checker.Check(ctx, cnDSN, globalDSN, autoFix)
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&cnDSN, "cn", "", "CN region PostgreSQL DSN")
|
||||
cmd.Flags().StringVar(&globalDSN, "global", "", "Global region PostgreSQL DSN")
|
||||
cmd.Flags().BoolVar(&autoFix, "auto-fix", false, "Automatically apply missing statements to CN")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newVerifyCmd() *cobra.Command {
|
||||
var (
|
||||
dsn string
|
||||
schemaPath string
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "verify",
|
||||
Short: "Verify that the database matches schema.sql",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if dsn == "" {
|
||||
return errors.New("--dsn is required")
|
||||
}
|
||||
if schemaPath == "" {
|
||||
schemaPath = defaultSchemaFile
|
||||
}
|
||||
verifier := migrate.NewVerifier()
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
|
||||
defer cancel()
|
||||
return verifier.Verify(ctx, dsn, schemaPath)
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&dsn, "dsn", "", "PostgreSQL connection string")
|
||||
cmd.Flags().StringVar(&schemaPath, "schema", defaultSchemaFile, "Path to schema.sql reference file")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newResetCmd(dir *string) *cobra.Command {
|
||||
var dsn string
|
||||
cmd := &cobra.Command{
|
||||
Use: "reset",
|
||||
Short: "Drop public schema and re-run migrations",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if dsn == "" {
|
||||
return errors.New("--dsn is required")
|
||||
}
|
||||
runner := migrate.NewRunner(*dir)
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Minute)
|
||||
defer cancel()
|
||||
return runner.Reset(ctx, dsn)
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&dsn, "dsn", "", "PostgreSQL connection string")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newVersionCmd(dir *string) *cobra.Command {
|
||||
var dsn string
|
||||
cmd := &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "Show current migration version",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if dsn == "" {
|
||||
return errors.New("--dsn is required")
|
||||
}
|
||||
runner := migrate.NewRunner(*dir)
|
||||
version, dirty, err := runner.Version(dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dirty {
|
||||
fmt.Printf("Current migration version: %d (dirty)\n", version)
|
||||
} else {
|
||||
fmt.Printf("Current migration version: %d\n", version)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&dsn, "dsn", "", "PostgreSQL connection string")
|
||||
return cmd
|
||||
}
|
||||
102
account/internal/migrate/checker.go
Normal file
102
account/internal/migrate/checker.go
Normal file
@ -0,0 +1,102 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"xcontrol/account/internal/utils"
|
||||
)
|
||||
|
||||
// Checker compares two PostgreSQL schemas (CN vs Global) and optionally fixes
|
||||
// missing structures on the CN side.
|
||||
type Checker struct{}
|
||||
|
||||
func NewChecker() *Checker {
|
||||
return &Checker{}
|
||||
}
|
||||
|
||||
func (c *Checker) Check(ctx context.Context, cnDSN, globalDSN string, autoFix bool) error {
|
||||
if cnDSN == "" || globalDSN == "" {
|
||||
return errors.New("both --cn and --global DSNs are required")
|
||||
}
|
||||
|
||||
cnDump, err := utils.RunPgDump(ctx, cnDSN)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dump CN schema: %w", err)
|
||||
}
|
||||
globalDump, err := utils.RunPgDump(ctx, globalDSN)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dump Global schema: %w", err)
|
||||
}
|
||||
|
||||
cnStatements := utils.NormalizeStatements(cnDump)
|
||||
globalStatements := utils.NormalizeStatements(globalDump)
|
||||
|
||||
onlyCN, onlyGlobal := utils.CompareStatements(cnStatements, globalStatements)
|
||||
|
||||
if len(onlyCN) == 0 && len(onlyGlobal) == 0 {
|
||||
fmt.Println("✅ CN and Global schemas are consistent")
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(onlyGlobal) > 0 {
|
||||
fmt.Println("⚠️ Statements missing on CN (present on Global):")
|
||||
for _, stmt := range onlyGlobal {
|
||||
fmt.Printf(" + %s;\n", stmt)
|
||||
}
|
||||
}
|
||||
|
||||
if len(onlyCN) > 0 {
|
||||
fmt.Println("⚠️ Statements only found on CN:")
|
||||
for _, stmt := range onlyCN {
|
||||
fmt.Printf(" - %s;\n", stmt)
|
||||
}
|
||||
}
|
||||
|
||||
if autoFix && len(onlyGlobal) > 0 {
|
||||
if err := applyStatements(ctx, cnDSN, onlyGlobal); err != nil {
|
||||
return fmt.Errorf("apply auto-fix: %w", err)
|
||||
}
|
||||
fmt.Println("✅ Auto-fix applied on CN database — please re-run check to confirm")
|
||||
// After auto-fix we still return an error if CN has extra statements.
|
||||
if len(onlyCN) > 0 {
|
||||
return errors.New("schema differences remain on CN after auto-fix")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return errors.New("schema differences detected")
|
||||
}
|
||||
|
||||
func applyStatements(ctx context.Context, dsn string, statements []string) error {
|
||||
db, err := openDB(ctx, dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
for _, stmt := range statements {
|
||||
if strings.Contains(stmt, "pglogical") {
|
||||
continue
|
||||
}
|
||||
sql := stmt
|
||||
if !strings.HasSuffix(sql, ";") {
|
||||
sql += ";"
|
||||
}
|
||||
fmt.Printf("→ Applying fix: %s\n", sql)
|
||||
if _, err := tx.ExecContext(ctx, sql); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("✅ Applied fix\n")
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
138
account/internal/migrate/cleaner.go
Normal file
138
account/internal/migrate/cleaner.go
Normal file
@ -0,0 +1,138 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Cleaner removes leftover structures such as invalid indexes or disabled
|
||||
// triggers while keeping pglogical untouched.
|
||||
type Cleaner struct{}
|
||||
|
||||
func NewCleaner() *Cleaner {
|
||||
return &Cleaner{}
|
||||
}
|
||||
|
||||
func (c *Cleaner) Clean(ctx context.Context, dsn string, force bool) error {
|
||||
if !force {
|
||||
return errors.New("clean requires --force confirmation")
|
||||
}
|
||||
|
||||
db, err := openDB(ctx, dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
tx, err := db.BeginTx(ctx, &sql.TxOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
if err := dropInvalidIndexes(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := dropDisabledTriggers(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := dropTemporaryTables(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Println("✅ Database clean-up completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
func dropInvalidIndexes(ctx context.Context, tx *sql.Tx) error {
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT quote_ident(n.nspname) || '.' || quote_ident(c.relname)
|
||||
FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname NOT IN ('pglogical', 'pg_catalog', 'information_schema')
|
||||
AND (NOT i.indisvalid OR NOT i.indisready)
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var identifier string
|
||||
if err := rows.Scan(&identifier); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("→ Dropping invalid index %s\n", identifier)
|
||||
if _, err := tx.ExecContext(ctx, fmt.Sprintf("DROP INDEX IF EXISTS %s", identifier)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("✅ Dropped index %s\n", identifier)
|
||||
}
|
||||
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func dropDisabledTriggers(ctx context.Context, tx *sql.Tx) error {
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT quote_ident(n.nspname) || '.' || quote_ident(rel.relname) AS tbl,
|
||||
quote_ident(t.tgname) AS trigger_name
|
||||
FROM pg_trigger t
|
||||
JOIN pg_class rel ON rel.oid = t.tgrelid
|
||||
JOIN pg_namespace n ON n.oid = rel.relnamespace
|
||||
WHERE t.tgenabled = 'D'
|
||||
AND t.tgisinternal = false
|
||||
AND n.nspname NOT IN ('pglogical', 'pg_catalog', 'information_schema')
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var table, trigger string
|
||||
if err := rows.Scan(&table, &trigger); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("→ Dropping disabled trigger %s on %s\n", trigger, table)
|
||||
if _, err := tx.ExecContext(ctx, fmt.Sprintf("DROP TRIGGER IF EXISTS %s ON %s", trigger, table)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("✅ Dropped trigger %s on %s\n", trigger, table)
|
||||
}
|
||||
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func dropTemporaryTables(ctx context.Context, tx *sql.Tx) error {
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT quote_ident(table_schema) || '.' || quote_ident(table_name)
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema NOT IN ('pglogical', 'pg_catalog', 'information_schema')
|
||||
AND (table_name LIKE 'tmp_%' OR table_name LIKE 'temp_%' OR table_name LIKE 'backup_%')
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var table string
|
||||
if err := rows.Scan(&table); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("→ Dropping temporary table %s\n", table)
|
||||
if _, err := tx.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", table)); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("✅ Dropped table %s\n", table)
|
||||
}
|
||||
|
||||
return rows.Err()
|
||||
}
|
||||
28
account/internal/migrate/db.go
Normal file
28
account/internal/migrate/db.go
Normal file
@ -0,0 +1,28 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
|
||||
func openDB(ctx context.Context, dsn string) (*sql.DB, error) {
|
||||
db, err := sql.Open("pgx", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
db.SetConnMaxLifetime(0)
|
||||
db.SetConnMaxIdleTime(5 * time.Minute)
|
||||
db.SetMaxIdleConns(5)
|
||||
db.SetMaxOpenConns(10)
|
||||
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
183
account/internal/migrate/runner.go
Normal file
183
account/internal/migrate/runner.go
Normal file
@ -0,0 +1,183 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
)
|
||||
|
||||
const defaultDir = "account/sql/migrations"
|
||||
|
||||
// Runner coordinates golang-migrate operations.
|
||||
type Runner struct {
|
||||
Dir string
|
||||
}
|
||||
|
||||
// NewRunner creates a runner that reads migration files from dir. When dir is
|
||||
// empty, the default directory under account/sql/migrations is used.
|
||||
func NewRunner(dir string) *Runner {
|
||||
if dir == "" {
|
||||
dir = defaultDir
|
||||
}
|
||||
return &Runner{Dir: dir}
|
||||
}
|
||||
|
||||
// Up executes all migrations that have not been applied yet. Each step logs
|
||||
// its outcome to provide clear visibility.
|
||||
func (r *Runner) Up(ctx context.Context, dsn string) error {
|
||||
absDir, err := filepath.Abs(r.Dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
migrations, err := r.loadMigrations(absDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m, err := migrate.New(fmt.Sprintf("file://%s", absDir), dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeMigrator(m)
|
||||
|
||||
currentVersion, dirty, err := m.Version()
|
||||
if err != nil {
|
||||
if errors.Is(err, migrate.ErrNilVersion) {
|
||||
currentVersion = 0
|
||||
} else {
|
||||
return fmt.Errorf("fetch current version: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if dirty {
|
||||
return fmt.Errorf("database is in a dirty state at version %d; please fix manually", currentVersion)
|
||||
}
|
||||
|
||||
applied := false
|
||||
for _, migration := range migrations {
|
||||
if migration.version <= currentVersion {
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Printf("→ Applying migration %s ...\n", migration.name)
|
||||
if err := m.Migrate(migration.version); err != nil {
|
||||
if errors.Is(err, migrate.ErrNoChange) {
|
||||
fmt.Printf("✅ Migration %s already applied\n", migration.name)
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("apply migration %s: %w", migration.name, err)
|
||||
}
|
||||
applied = true
|
||||
fmt.Printf("✅ Migration %s applied\n", migration.name)
|
||||
}
|
||||
|
||||
if !applied {
|
||||
fmt.Println("✅ Database schema already up-to-date")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Version reports the current schema version tracked by golang-migrate.
|
||||
func (r *Runner) Version(dsn string) (uint, bool, error) {
|
||||
absDir, err := filepath.Abs(r.Dir)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
|
||||
m, err := migrate.New(fmt.Sprintf("file://%s", absDir), dsn)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
defer closeMigrator(m)
|
||||
|
||||
version, dirty, err := m.Version()
|
||||
if err != nil {
|
||||
if errors.Is(err, migrate.ErrNilVersion) {
|
||||
return 0, false, nil
|
||||
}
|
||||
return 0, false, err
|
||||
}
|
||||
|
||||
return version, dirty, nil
|
||||
}
|
||||
|
||||
// Reset drops the public schema before replaying all migrations.
|
||||
func (r *Runner) Reset(ctx context.Context, dsn string) error {
|
||||
db, err := openDB(ctx, dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
fmt.Println("⚠️ Dropping public schema (preserving pglogical)...")
|
||||
if _, err := db.ExecContext(ctx, "DROP SCHEMA IF EXISTS public CASCADE"); err != nil {
|
||||
return fmt.Errorf("drop public schema: %w", err)
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, "CREATE SCHEMA IF NOT EXISTS public"); err != nil {
|
||||
return fmt.Errorf("recreate public schema: %w", err)
|
||||
}
|
||||
|
||||
return r.Up(ctx, dsn)
|
||||
}
|
||||
|
||||
func (r *Runner) loadMigrations(absDir string) ([]*migrationFile, error) {
|
||||
entries, err := os.ReadDir(absDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
migrationMap := make(map[uint]*migrationFile)
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
if !strings.HasSuffix(name, ".up.sql") {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(name, "_", 2)
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
version, err := strconv.ParseUint(parts[0], 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
migrationMap[uint(version)] = &migrationFile{
|
||||
version: uint(version),
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
|
||||
var migrations []*migrationFile
|
||||
for _, m := range migrationMap {
|
||||
migrations = append(migrations, m)
|
||||
}
|
||||
|
||||
sort.Slice(migrations, func(i, j int) bool {
|
||||
return migrations[i].version < migrations[j].version
|
||||
})
|
||||
|
||||
return migrations, nil
|
||||
}
|
||||
|
||||
func closeMigrator(m *migrate.Migrate) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
_, _ = m.Close()
|
||||
}
|
||||
|
||||
type migrationFile struct {
|
||||
version uint
|
||||
name string
|
||||
}
|
||||
64
account/internal/migrate/verifier.go
Normal file
64
account/internal/migrate/verifier.go
Normal file
@ -0,0 +1,64 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"xcontrol/account/internal/utils"
|
||||
)
|
||||
|
||||
const defaultSchemaPath = "account/sql/schema.sql"
|
||||
|
||||
// Verifier validates that the live database matches the canonical schema.sql.
|
||||
type Verifier struct{}
|
||||
|
||||
func NewVerifier() *Verifier {
|
||||
return &Verifier{}
|
||||
}
|
||||
|
||||
func (v *Verifier) Verify(ctx context.Context, dsn, schemaPath string) error {
|
||||
if dsn == "" {
|
||||
return errors.New("--dsn is required")
|
||||
}
|
||||
if schemaPath == "" {
|
||||
schemaPath = defaultSchemaPath
|
||||
}
|
||||
|
||||
dump, err := utils.RunPgDump(ctx, dsn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dump database schema: %w", err)
|
||||
}
|
||||
|
||||
schemaBytes, err := os.ReadFile(schemaPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read schema file: %w", err)
|
||||
}
|
||||
|
||||
dbStatements := utils.NormalizeStatements(dump)
|
||||
fileStatements := utils.NormalizeStatements(string(schemaBytes))
|
||||
|
||||
onlyDB, onlyFile := utils.CompareStatements(dbStatements, fileStatements)
|
||||
|
||||
if len(onlyDB) == 0 && len(onlyFile) == 0 {
|
||||
fmt.Println("✅ Database schema matches schema.sql")
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(onlyFile) > 0 {
|
||||
fmt.Println("⚠️ Statements missing from database (present in schema.sql):")
|
||||
for _, stmt := range onlyFile {
|
||||
fmt.Printf(" + %s;\n", stmt)
|
||||
}
|
||||
}
|
||||
|
||||
if len(onlyDB) > 0 {
|
||||
fmt.Println("⚠️ Extra statements found in database:")
|
||||
for _, stmt := range onlyDB {
|
||||
fmt.Printf(" - %s;\n", stmt)
|
||||
}
|
||||
}
|
||||
|
||||
return errors.New("schema mismatch detected")
|
||||
}
|
||||
132
account/internal/utils/diff.go
Normal file
132
account/internal/utils/diff.go
Normal file
@ -0,0 +1,132 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// NormalizeStatements extracts relevant DDL statements from the provided dump.
|
||||
// Comments, empty lines and helper SET statements are ignored. Whitespace is
|
||||
// normalised to make comparisons deterministic across environments.
|
||||
func NormalizeStatements(dump string) []string {
|
||||
var statements []string
|
||||
var builder strings.Builder
|
||||
|
||||
flush := func() {
|
||||
stmt := strings.TrimSpace(builder.String())
|
||||
if stmt == "" {
|
||||
builder.Reset()
|
||||
return
|
||||
}
|
||||
stmt = strings.TrimSuffix(stmt, ";")
|
||||
stmt = strings.TrimSpace(stmt)
|
||||
if isRelevantStatement(stmt) {
|
||||
normalized := collapseWhitespace(stmt)
|
||||
statements = append(statements, normalized)
|
||||
}
|
||||
builder.Reset()
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(strings.NewReader(dump))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "--") {
|
||||
continue
|
||||
}
|
||||
if shouldSkipLine(line) {
|
||||
continue
|
||||
}
|
||||
|
||||
builder.WriteString(line)
|
||||
if strings.HasSuffix(line, ";") {
|
||||
flush()
|
||||
} else {
|
||||
builder.WriteString(" ")
|
||||
}
|
||||
}
|
||||
|
||||
flush()
|
||||
|
||||
sort.Strings(statements)
|
||||
return statements
|
||||
}
|
||||
|
||||
// CompareStatements returns the statements that are present only on the left
|
||||
// or only on the right collection.
|
||||
func CompareStatements(left, right []string) (onlyLeft, onlyRight []string) {
|
||||
leftSet := make(map[string]struct{}, len(left))
|
||||
for _, stmt := range left {
|
||||
if strings.Contains(stmt, "pglogical") {
|
||||
continue
|
||||
}
|
||||
leftSet[stmt] = struct{}{}
|
||||
}
|
||||
|
||||
rightSet := make(map[string]struct{}, len(right))
|
||||
for _, stmt := range right {
|
||||
if strings.Contains(stmt, "pglogical") {
|
||||
continue
|
||||
}
|
||||
rightSet[stmt] = struct{}{}
|
||||
}
|
||||
|
||||
for stmt := range leftSet {
|
||||
if _, ok := rightSet[stmt]; !ok {
|
||||
onlyLeft = append(onlyLeft, stmt)
|
||||
}
|
||||
}
|
||||
|
||||
for stmt := range rightSet {
|
||||
if _, ok := leftSet[stmt]; !ok {
|
||||
onlyRight = append(onlyRight, stmt)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(onlyLeft)
|
||||
sort.Strings(onlyRight)
|
||||
return
|
||||
}
|
||||
|
||||
func shouldSkipLine(line string) bool {
|
||||
lower := strings.ToLower(line)
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "set "):
|
||||
return true
|
||||
case strings.HasPrefix(lower, "select pg_catalog.set_config"):
|
||||
return true
|
||||
case strings.HasPrefix(lower, "reset "):
|
||||
return true
|
||||
case strings.HasPrefix(line, "\\connect "):
|
||||
return true
|
||||
case strings.HasPrefix(lower, "lock table"):
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isRelevantStatement(stmt string) bool {
|
||||
lower := strings.ToLower(stmt)
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "create table"):
|
||||
return true
|
||||
case strings.HasPrefix(lower, "alter table"):
|
||||
return true
|
||||
case strings.HasPrefix(lower, "create index"):
|
||||
return true
|
||||
case strings.HasPrefix(lower, "alter index"):
|
||||
return true
|
||||
case strings.HasPrefix(lower, "comment on table"):
|
||||
return true
|
||||
case strings.HasPrefix(lower, "comment on column"):
|
||||
return true
|
||||
case strings.HasPrefix(lower, "grant "):
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func collapseWhitespace(input string) string {
|
||||
fields := strings.Fields(input)
|
||||
return strings.Join(fields, " ")
|
||||
}
|
||||
56
account/internal/utils/exec.go
Normal file
56
account/internal/utils/exec.go
Normal file
@ -0,0 +1,56 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CommandResult captures stdout and stderr from an executed command.
|
||||
type CommandResult struct {
|
||||
Stdout string
|
||||
Stderr string
|
||||
}
|
||||
|
||||
// RunCommand executes the provided command with context awareness and
|
||||
// returns the collected stdout/stderr output. Errors include contextual
|
||||
// information as well as stderr to make troubleshooting easier.
|
||||
func RunCommand(ctx context.Context, name string, args ...string) (*CommandResult, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Env = os.Environ()
|
||||
|
||||
var stdoutBuf, stderrBuf bytes.Buffer
|
||||
cmd.Stdout = &stdoutBuf
|
||||
cmd.Stderr = &stderrBuf
|
||||
|
||||
err := cmd.Run()
|
||||
result := &CommandResult{Stdout: stdoutBuf.String(), Stderr: stderrBuf.String()}
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("command %s %s failed: %w\n%s", name, strings.Join(args, " "), err, result.Stderr)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// RunPgDump executes pg_dump with the flags required for schema comparison
|
||||
// and returns the textual dump. The pglogical schema is excluded to avoid
|
||||
// touching logical replication internals.
|
||||
func RunPgDump(ctx context.Context, dsn string) (string, error) {
|
||||
args := []string{
|
||||
"--schema-only",
|
||||
"--no-owner",
|
||||
"--no-privileges",
|
||||
"--exclude-schema=pglogical",
|
||||
"--dbname", dsn,
|
||||
}
|
||||
|
||||
result, err := RunCommand(ctx, "pg_dump", args...)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return result.Stdout, nil
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
-- No-op down migration; manual rollback required.
|
||||
1
account/sql/migrations/202510070000_to_rbac.down.sql
Normal file
1
account/sql/migrations/202510070000_to_rbac.down.sql
Normal file
@ -0,0 +1 @@
|
||||
-- No-op down migration; manual rollback required.
|
||||
@ -0,0 +1 @@
|
||||
-- No-op down migration; manual rollback required.
|
||||
97
account/sql/migrations/202510080100_generated_columns.up.sql
Normal file
97
account/sql/migrations/202510080100_generated_columns.up.sql
Normal file
@ -0,0 +1,97 @@
|
||||
-- =========================================
|
||||
-- 20251008-fix-generated-columns.sql
|
||||
-- Migration: Remove generated columns incompatible with pglogical
|
||||
-- Safe to re-run (幂等)
|
||||
-- =========================================
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- 尝试确保当前用户对 pglogical schema 有访问权限
|
||||
DO $grant$
|
||||
BEGIN
|
||||
BEGIN
|
||||
EXECUTE 'GRANT USAGE ON SCHEMA pglogical TO ' || current_user;
|
||||
EXECUTE 'GRANT ALL ON ALL TABLES IN SCHEMA pglogical TO ' || current_user;
|
||||
EXECUTE 'GRANT ALL ON ALL SEQUENCES IN SCHEMA pglogical TO ' || current_user;
|
||||
EXECUTE 'GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pglogical TO ' || current_user;
|
||||
RAISE NOTICE 'Granted pglogical schema permissions to current_user: %', current_user;
|
||||
EXCEPTION WHEN others THEN
|
||||
RAISE NOTICE 'Skipping GRANT for schema pglogical (possibly already granted or insufficient privilege)';
|
||||
END;
|
||||
END;
|
||||
$grant$;
|
||||
|
||||
-- =========================================
|
||||
-- Step 1. 检查并删除旧的 generated column
|
||||
-- =========================================
|
||||
DO $drop$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'users'
|
||||
AND column_name = 'email_verified'
|
||||
AND is_generated = 'ALWAYS'
|
||||
) THEN
|
||||
RAISE NOTICE 'Detected generated column: users.email_verified — dropping it for pglogical compatibility';
|
||||
EXECUTE 'ALTER TABLE public.users DROP COLUMN email_verified';
|
||||
ELSE
|
||||
RAISE NOTICE 'No generated column detected: users.email_verified is safe';
|
||||
END IF;
|
||||
END;
|
||||
$drop$;
|
||||
|
||||
-- =========================================
|
||||
-- Step 2. 添加普通布尔列 (safe re-run)
|
||||
-- =========================================
|
||||
DO $add$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'users'
|
||||
AND column_name = 'email_verified'
|
||||
) THEN
|
||||
RAISE NOTICE 'Adding normal column: users.email_verified (BOOLEAN DEFAULT false)';
|
||||
EXECUTE 'ALTER TABLE public.users ADD COLUMN email_verified BOOLEAN DEFAULT false NOT NULL';
|
||||
ELSE
|
||||
RAISE NOTICE 'Column users.email_verified already exists, skipping ADD COLUMN';
|
||||
END IF;
|
||||
END;
|
||||
$add$;
|
||||
|
||||
-- =========================================
|
||||
-- Step 3. 创建维护触发器
|
||||
-- =========================================
|
||||
DROP FUNCTION IF EXISTS maintain_email_verified() CASCADE;
|
||||
|
||||
CREATE FUNCTION maintain_email_verified() RETURNS trigger
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
NEW.email_verified := (NEW.email_verified_at IS NOT NULL);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_users_maintain_email_verified ON public.users;
|
||||
|
||||
CREATE TRIGGER trg_users_maintain_email_verified
|
||||
BEFORE INSERT OR UPDATE ON public.users
|
||||
FOR EACH ROW EXECUTE FUNCTION maintain_email_verified();
|
||||
|
||||
-- =========================================
|
||||
-- Step 4. 修正现有数据
|
||||
-- =========================================
|
||||
UPDATE public.users
|
||||
SET email_verified = (email_verified_at IS NOT NULL)
|
||||
WHERE email_verified IS DISTINCT FROM (email_verified_at IS NOT NULL);
|
||||
|
||||
COMMIT;
|
||||
|
||||
-- =========================================
|
||||
-- End of migration
|
||||
-- =========================================
|
||||
|
||||
@ -0,0 +1 @@
|
||||
-- No-op down migration; manual rollback required.
|
||||
86
account/sql/migrations/202510080200_email_verified.up.sql
Normal file
86
account/sql/migrations/202510080200_email_verified.up.sql
Normal file
@ -0,0 +1,86 @@
|
||||
-- =========================================
|
||||
-- 20251008.migrate.generated-columns.sql
|
||||
-- Migration: Convert generated columns to normal + trigger
|
||||
-- Safe to re-run (幂等)
|
||||
-- =========================================
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- 1️⃣ 兼容 pglogical:确保当前用户可访问 schema
|
||||
DO $$
|
||||
BEGIN
|
||||
BEGIN
|
||||
GRANT USAGE ON SCHEMA pglogical TO CURRENT_USER;
|
||||
RAISE NOTICE 'Granted USAGE on schema pglogical to current user';
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
RAISE NOTICE 'Skipping GRANT for schema pglogical (possibly already granted or insufficient privilege)';
|
||||
END;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- 2️⃣ 删除不兼容的 generated column
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'users'
|
||||
AND column_name = 'email_verified'
|
||||
AND is_generated = 'ALWAYS'
|
||||
) THEN
|
||||
RAISE NOTICE 'Detected generated column: users.email_verified — dropping it for pglogical compatibility';
|
||||
ALTER TABLE public.users DROP COLUMN email_verified;
|
||||
ELSE
|
||||
RAISE NOTICE 'No generated column detected: users.email_verified is safe';
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- 3️⃣ 确保存在普通列 email_verified
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'users'
|
||||
AND column_name = 'email_verified'
|
||||
) THEN
|
||||
RAISE NOTICE 'Adding normal column: users.email_verified (BOOLEAN DEFAULT false)';
|
||||
ALTER TABLE public.users ADD COLUMN email_verified BOOLEAN DEFAULT false NOT NULL;
|
||||
ELSE
|
||||
RAISE NOTICE 'Column users.email_verified already exists, skipping ADD COLUMN';
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- 4️⃣ 定义幂等触发器函数
|
||||
DROP FUNCTION IF EXISTS public.maintain_email_verified() CASCADE;
|
||||
|
||||
CREATE FUNCTION public.maintain_email_verified() RETURNS trigger
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
NEW.email_verified := (NEW.email_verified_at IS NOT NULL);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- 5️⃣ 触发器同步 email_verified 与 email_verified_at
|
||||
DROP TRIGGER IF EXISTS trg_users_maintain_email_verified ON public.users;
|
||||
|
||||
CREATE TRIGGER trg_users_maintain_email_verified
|
||||
BEFORE INSERT OR UPDATE ON public.users
|
||||
FOR EACH ROW EXECUTE FUNCTION public.maintain_email_verified();
|
||||
|
||||
-- 6️⃣ 修正历史数据
|
||||
UPDATE public.users
|
||||
SET email_verified = (email_verified_at IS NOT NULL)
|
||||
WHERE email_verified IS DISTINCT FROM (email_verified_at IS NOT NULL);
|
||||
|
||||
COMMIT;
|
||||
|
||||
-- =========================================
|
||||
-- End of migration
|
||||
-- =========================================
|
||||
|
||||
53
account/sql/readme.md
Normal file
53
account/sql/readme.md
Normal file
@ -0,0 +1,53 @@
|
||||
使用新的 `migratectl` CLI 可以在不同环境下快速执行迁移、校验和重置操作:
|
||||
|
||||
```bash
|
||||
go run ./cmd/migratectl/main.go migrate --dsn "$DB_URL"
|
||||
go run ./cmd/migratectl/main.go check --cn "$CN_DSN" --global "$GLOBAL_DSN"
|
||||
```
|
||||
|
||||
以下命令展示了如何授予 pglogical schema 访问权限:
|
||||
|
||||
sudo -u postgres psql -d account -c "GRANT USAGE ON SCHEMA pglogical TO PUBLIC;"
|
||||
|
||||
-- 登录 postgres
|
||||
sudo -u postgres psql -d account
|
||||
|
||||
-- 授权 shenlan 对 public schema 全权限
|
||||
ALTER SCHEMA public OWNER TO shenlan;
|
||||
GRANT ALL ON SCHEMA public TO shenlan;
|
||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO shenlan;
|
||||
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO shenlan;
|
||||
GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public TO shenlan;
|
||||
|
||||
-- 授权 pglogical schema 使用权限(防止混用)
|
||||
GRANT USAGE ON SCHEMA pglogical TO shenlan;
|
||||
|
||||
|
||||
|
||||
\q
|
||||
|
||||
|
||||
执行顺序建议
|
||||
步骤 节点 脚本 说明
|
||||
1️⃣ Global schema_base.sql 创建业务结构
|
||||
2️⃣ CN schema_base.sql 创建相同业务结构
|
||||
3️⃣ Global schema_pglogical_region_global.sql 定义 Global provider + 订阅 CN
|
||||
4️⃣ CN schema_pglogical_region_cn.sql 定义 CN provider + 订阅 Global
|
||||
🧩 验证同步状态
|
||||
SELECT * FROM pglogical.show_subscription_status();
|
||||
|
||||
如果输出中:
|
||||
|
||||
status = 'replicating'
|
||||
|
||||
即表示 双向复制同步正常。
|
||||
|
||||
🚀 双向同步特性汇总
|
||||
特性 实现机制
|
||||
双主写入 两端都是 Provider + Subscriber
|
||||
唯一性保障 所有主键为 gen_random_uuid(),避免冲突
|
||||
邮箱唯一 lower(email) 唯一索引
|
||||
异步复制 WAL 级逻辑同步,自动断点续传
|
||||
结构一致性 schema_base.sql 保证完全相同
|
||||
幂等可重建 全部 IF NOT EXISTS,可重复执行
|
||||
可扩展性 可新增字段或表,通过 replication_set_add_table() 同步
|
||||
@ -1,30 +1,34 @@
|
||||
-- =========================================
|
||||
-- PostgreSQL schema initialization script
|
||||
-- Safe to re-run (幂等)
|
||||
-- schema_base.sql
|
||||
-- Shared schema for pglogical bidirectional sync
|
||||
-- PostgreSQL 16 + gen_random_uuid()
|
||||
-- =========================================
|
||||
|
||||
SET statement_timeout = 0;
|
||||
SET lock_timeout = 0;
|
||||
SET client_encoding = 'UTF8';
|
||||
SET standard_conforming_strings = on;
|
||||
SET row_security = off;
|
||||
SET search_path = public;
|
||||
DROP SCHEMA IF EXISTS public CASCADE;
|
||||
CREATE SCHEMA public AUTHORIZATION postgres;
|
||||
|
||||
-- 必要扩展
|
||||
-- =========================================
|
||||
-- Extensions
|
||||
-- =========================================
|
||||
CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA public;
|
||||
CREATE EXTENSION IF NOT EXISTS pglogical WITH SCHEMA pglogical;
|
||||
|
||||
-- =========================================
|
||||
-- Function: set_updated_at()
|
||||
-- Functions
|
||||
-- =========================================
|
||||
DROP FUNCTION IF EXISTS set_updated_at() CASCADE;
|
||||
|
||||
CREATE FUNCTION set_updated_at() RETURNS trigger
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
CREATE OR REPLACE FUNCTION public.set_updated_at() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
NEW.updated_at := now();
|
||||
RETURN NEW;
|
||||
NEW.updated_at := now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.maintain_email_verified() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
NEW.email_verified := (NEW.email_verified_at IS NOT NULL);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
@ -32,84 +36,84 @@ $$;
|
||||
-- Tables
|
||||
-- =========================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
username TEXT NOT NULL,
|
||||
password TEXT NOT NULL,
|
||||
email TEXT,
|
||||
level INTEGER DEFAULT 20 NOT NULL,
|
||||
role TEXT DEFAULT 'user' NOT NULL,
|
||||
groups JSONB DEFAULT '[]'::jsonb NOT NULL,
|
||||
permissions JSONB DEFAULT '[]'::jsonb NOT NULL,
|
||||
created_at TIMESTAMPTZ DEFAULT now(),
|
||||
uuid UUID DEFAULT gen_random_uuid() NOT NULL,
|
||||
mfa_totp_secret TEXT,
|
||||
mfa_enabled BOOLEAN DEFAULT false NOT NULL,
|
||||
mfa_secret_issued_at TIMESTAMPTZ,
|
||||
mfa_confirmed_at TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ DEFAULT now(),
|
||||
email_verified_at TIMESTAMPTZ,
|
||||
email_verified BOOLEAN GENERATED ALWAYS AS ((email_verified_at IS NOT NULL)) STORED,
|
||||
CONSTRAINT users_pkey PRIMARY KEY (uuid),
|
||||
CONSTRAINT users_username_uk UNIQUE (username),
|
||||
CONSTRAINT users_email_uk UNIQUE (email)
|
||||
CREATE TABLE public.users (
|
||||
uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
username TEXT NOT NULL,
|
||||
password TEXT NOT NULL,
|
||||
email TEXT,
|
||||
role TEXT NOT NULL DEFAULT 'user',
|
||||
level INTEGER NOT NULL DEFAULT 20,
|
||||
groups JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
permissions JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
mfa_totp_secret TEXT,
|
||||
mfa_enabled BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
mfa_secret_issued_at TIMESTAMPTZ,
|
||||
mfa_confirmed_at TIMESTAMPTZ,
|
||||
email_verified_at TIMESTAMPTZ,
|
||||
email_verified BOOLEAN GENERATED ALWAYS AS ((email_verified_at IS NOT NULL)) STORED
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS users_username_lower_uk ON users (lower(username));
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS users_email_lower_uk ON users (lower(email)) WHERE email IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS identities (
|
||||
provider TEXT NOT NULL,
|
||||
external_id TEXT NOT NULL,
|
||||
uuid UUID DEFAULT gen_random_uuid() NOT NULL,
|
||||
user_uuid UUID NOT NULL,
|
||||
CONSTRAINT identities_pkey PRIMARY KEY (uuid),
|
||||
CONSTRAINT identities_provider_external_id_uk UNIQUE (provider, external_id),
|
||||
CONSTRAINT identities_user_fk FOREIGN KEY (user_uuid)
|
||||
REFERENCES users(uuid) ON DELETE CASCADE
|
||||
CREATE TABLE public.identities (
|
||||
uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
provider TEXT NOT NULL,
|
||||
external_id TEXT NOT NULL,
|
||||
user_uuid UUID NOT NULL REFERENCES public.users(uuid) ON DELETE CASCADE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT identities_provider_external_id_uk UNIQUE (provider, external_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token TEXT NOT NULL,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
uuid UUID DEFAULT gen_random_uuid() NOT NULL,
|
||||
user_uuid UUID NOT NULL,
|
||||
CONSTRAINT sessions_pkey PRIMARY KEY (uuid),
|
||||
CONSTRAINT sessions_user_fk FOREIGN KEY (user_uuid)
|
||||
REFERENCES users(uuid) ON DELETE CASCADE
|
||||
CREATE TABLE public.sessions (
|
||||
uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
token TEXT NOT NULL,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
user_uuid UUID NOT NULL REFERENCES public.users(uuid) ON DELETE CASCADE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS admin_settings (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
module_key TEXT NOT NULL,
|
||||
role TEXT NOT NULL,
|
||||
enabled BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
version BIGINT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT admin_settings_module_role_uk UNIQUE (module_key, role)
|
||||
CREATE TABLE public.admin_settings (
|
||||
uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
module_key TEXT NOT NULL,
|
||||
role TEXT NOT NULL,
|
||||
enabled BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
version BIGINT NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT admin_settings_module_role_uk UNIQUE (module_key, role)
|
||||
);
|
||||
|
||||
-- =========================================
|
||||
-- Indexes
|
||||
-- =========================================
|
||||
CREATE INDEX IF NOT EXISTS idx_identities_provider ON identities (provider);
|
||||
CREATE INDEX IF NOT EXISTS idx_identities_user_uuid ON identities (user_uuid);
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_user_uuid ON sessions (user_uuid);
|
||||
CREATE INDEX IF NOT EXISTS idx_admin_settings_version ON admin_settings (version);
|
||||
CREATE UNIQUE INDEX users_username_lower_uk ON public.users (lower(username));
|
||||
CREATE UNIQUE INDEX users_email_lower_uk ON public.users (lower(email)) WHERE email IS NOT NULL;
|
||||
CREATE INDEX idx_identities_user_uuid ON public.identities (user_uuid);
|
||||
CREATE INDEX idx_sessions_user_uuid ON public.sessions (user_uuid);
|
||||
CREATE INDEX idx_admin_settings_version ON public.admin_settings (version);
|
||||
|
||||
-- =========================================
|
||||
-- Trigger
|
||||
-- Triggers
|
||||
-- =========================================
|
||||
DROP TRIGGER IF EXISTS trg_users_set_updated_at ON users;
|
||||
CREATE TRIGGER trg_users_set_updated_at
|
||||
BEFORE UPDATE ON users
|
||||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||||
BEFORE UPDATE ON public.users
|
||||
FOR EACH ROW EXECUTE FUNCTION public.set_updated_at();
|
||||
|
||||
CREATE TRIGGER trg_users_maintain_email_verified
|
||||
BEFORE INSERT OR UPDATE ON public.users
|
||||
FOR EACH ROW EXECUTE FUNCTION public.maintain_email_verified();
|
||||
|
||||
CREATE TRIGGER trg_identities_set_updated_at
|
||||
BEFORE UPDATE ON public.identities
|
||||
FOR EACH ROW EXECUTE FUNCTION public.set_updated_at();
|
||||
|
||||
CREATE TRIGGER trg_sessions_set_updated_at
|
||||
BEFORE UPDATE ON public.sessions
|
||||
FOR EACH ROW EXECUTE FUNCTION public.set_updated_at();
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_admin_settings_set_updated_at ON admin_settings;
|
||||
CREATE TRIGGER trg_admin_settings_set_updated_at
|
||||
BEFORE UPDATE ON admin_settings
|
||||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||||
BEFORE UPDATE ON public.admin_settings
|
||||
FOR EACH ROW EXECUTE FUNCTION public.set_updated_at();
|
||||
|
||||
-- =========================================
|
||||
-- End of schema.sql
|
||||
-- =========================================
|
||||
|
||||
@ -1,151 +0,0 @@
|
||||
# ==============================================
|
||||
# XControl Homepage — Production (Dynamic Render)
|
||||
# Structure: Nginx → Node.js(3000) → Go(8080/8090)
|
||||
# ==============================================
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name www.svc.plus global-homepage.svc.plus;
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name www.svc.plus global-homepage.svc.plus;
|
||||
|
||||
ssl_certificate /etc/ssl/svc.plus.pem;
|
||||
ssl_certificate_key /etc/ssl/svc.plus.rsa.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
ssl_prefer_server_ciphers on;
|
||||
|
||||
# ===============================
|
||||
# 基本安全头(推荐保持)
|
||||
# ===============================
|
||||
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||
add_header X-Frame-Options "SAMEORIGIN";
|
||||
add_header X-Content-Type-Options "nosniff";
|
||||
add_header Referrer-Policy "strict-origin-when-cross-origin";
|
||||
add_header X-XSS-Protection "1; mode=block";
|
||||
|
||||
# ===============================
|
||||
# API 反向代理
|
||||
# ===============================
|
||||
# 优先匹配认证服务
|
||||
location ^~ /api/auth/ {
|
||||
proxy_pass http://127.0.0.1:8080;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# 其他 /api/ 请求转 server 服务
|
||||
location ^~ /api/ {
|
||||
proxy_pass http://127.0.0.1:8090;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# AskAI 接口限流 (保持 Lua 逻辑不变)
|
||||
location = /api/askai {
|
||||
access_by_lua_block {
|
||||
local redis = require "resty.redis"
|
||||
local r = redis:new()
|
||||
r:set_timeout(200)
|
||||
local ok, err = r:connect("127.0.0.1", 6379)
|
||||
if not ok then
|
||||
ngx.log(ngx.ERR, "Redis connect error: ", err)
|
||||
return ngx.exit(500)
|
||||
end
|
||||
|
||||
local user = ngx.var.arg_user or ngx.var.remote_addr
|
||||
local today = os.date("%Y%m%d")
|
||||
local key = "limit:user:" .. user .. ":" .. today
|
||||
|
||||
local count, err = r:incr(key)
|
||||
if count == 1 then r:expire(key, 86400) end
|
||||
if count > 200 then
|
||||
ngx.status = 429
|
||||
ngx.header["Content-Type"] = "text/plain; charset=utf-8"
|
||||
ngx.say("Too Many Requests: daily limit reached")
|
||||
return ngx.exit(429)
|
||||
end
|
||||
}
|
||||
|
||||
proxy_pass http://127.0.0.1:8090;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# ===============================
|
||||
# Next.js 动态页面代理
|
||||
# ===============================
|
||||
location ^~ /_next/ {
|
||||
proxy_pass http://127.0.0.1:3000;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
access_log off;
|
||||
expires 7d;
|
||||
add_header Cache-Control "public, max-age=31536000, immutable";
|
||||
}
|
||||
|
||||
location = /favicon.ico {
|
||||
proxy_pass http://127.0.0.1:3000;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:3000;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_set_header Cookie $http_cookie;
|
||||
proxy_pass_header Set-Cookie;
|
||||
|
||||
# WebSocket 支持 (Next.js HMR / API routes)
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
|
||||
# 可选:微缓存 SSR 页面(减少 Node 负载)
|
||||
# proxy_cache ssr_cache;
|
||||
# proxy_cache_key $scheme$host$request_uri;
|
||||
# proxy_cache_valid 200 10s;
|
||||
# proxy_cache_valid 404 1s;
|
||||
# add_header X-Cache $upstream_cache_status;
|
||||
}
|
||||
|
||||
# ===============================
|
||||
# 隐藏文件保护
|
||||
# ===============================
|
||||
location ~ /\. {
|
||||
deny all;
|
||||
}
|
||||
|
||||
# ===============================
|
||||
# 性能优化:gzip 压缩
|
||||
# ===============================
|
||||
gzip on;
|
||||
gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml;
|
||||
gzip_min_length 1024;
|
||||
gzip_proxied any;
|
||||
gzip_comp_level 6;
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
-- init.sql
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS documents (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
chunk_id INT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
embedding VECTOR(1024),
|
||||
metadata JSONB
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS documents_embedding_idx
|
||||
ON documents USING hnsw (embedding vector_cosine_ops);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_documents_metadata
|
||||
ON documents USING gin (metadata);
|
||||
@ -1,77 +0,0 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name dev.svc.plus dev-homepage.svc.plus;
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name dev.svc.plus dev-homepage.svc.plus;
|
||||
|
||||
ssl_certificate /etc/ssl/svc.plus.pem;
|
||||
ssl_certificate_key /etc/ssl/svc.plus.rsa.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
|
||||
# ================================
|
||||
# Next.js 静态资源 (_next/static/*)
|
||||
# ================================
|
||||
location /_next/static/ {
|
||||
proxy_pass http://127.0.0.1:3001;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# 静态文件可缓存一年
|
||||
access_log off;
|
||||
expires 7d;
|
||||
add_header Cache-Control "public, max-age=31536000, immutable";
|
||||
}
|
||||
|
||||
# ================================
|
||||
# Public 静态资源 (favicon, images)
|
||||
# ================================
|
||||
location /public/ {
|
||||
alias /var/www/XControl/ui/homepage/public/;
|
||||
access_log off;
|
||||
expires 30d;
|
||||
add_header Cache-Control "public, max-age=2592000";
|
||||
}
|
||||
|
||||
# ================================
|
||||
# API / SSR 页面
|
||||
# ================================
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:3001;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_set_header Cookie $http_cookie;
|
||||
proxy_pass_header Set-Cookie;
|
||||
|
||||
# WebSocket 支持 (Next.js HMR, API)
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
}
|
||||
|
||||
# 禁止访问隐藏文件
|
||||
location ~ /\. {
|
||||
deny all;
|
||||
}
|
||||
|
||||
# ================================
|
||||
# 性能优化:开启 gzip 压缩
|
||||
# ================================
|
||||
gzip on;
|
||||
gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml;
|
||||
gzip_min_length 1024;
|
||||
gzip_proxied any;
|
||||
gzip_comp_level 6;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,49 +0,0 @@
|
||||
global:
|
||||
redis:
|
||||
addr: "127.0.0.1:6379"
|
||||
password: ""
|
||||
vectordb:
|
||||
pgurl: "postgres://shenlan:password@127.0.0.1:5432/shenlan"
|
||||
datasources:
|
||||
- name: Xstream
|
||||
repo: https://github.com/svc-design/Xstream
|
||||
path: docs
|
||||
- name: documents
|
||||
repo: https://github.com/svc-design/documents
|
||||
path: /
|
||||
- name: XControl
|
||||
repo: https://github.com/svc-design/XControl
|
||||
path: docs
|
||||
sync:
|
||||
repo:
|
||||
proxy: socks5://127.0.0.1:1080 # 仅在同步仓库时使用代理
|
||||
|
||||
models:
|
||||
embedder:
|
||||
provider: "huggingface_hub"
|
||||
models: "bge-m3"
|
||||
endpoint: "http://127.0.0.1:9000/v1/embeddings"
|
||||
generator:
|
||||
provider: "ollama"
|
||||
models:
|
||||
- 'gemma3:4b'
|
||||
endpoint: "http://127.0.0.1:11434/v1/chat/completions"
|
||||
|
||||
embedding:
|
||||
max_batch: 64
|
||||
dimension: 1024 #维度
|
||||
max_chars: 8000
|
||||
rate_limit_tpm: 120000
|
||||
|
||||
chunking:
|
||||
embed_toc: true
|
||||
max_tokens: 800
|
||||
overlap_tokens: 80
|
||||
prefer_heading_split: true
|
||||
include_exts: [".md", ".mdx"]
|
||||
ignore_dirs: [".git", "node_modules", "dist", "build"]
|
||||
|
||||
api:
|
||||
askai:
|
||||
timeout: 100
|
||||
retries: 3
|
||||
@ -1,51 +0,0 @@
|
||||
global:
|
||||
redis:
|
||||
addr: "127.0.0.1:6379"
|
||||
password: ""
|
||||
vectordb:
|
||||
pgurl: "postgres://shenlan:password@127.0.0.1:5432/postgres"
|
||||
datasources:
|
||||
- name: Xstream
|
||||
repo: https://github.com/svc-design/Xstream
|
||||
path: docs
|
||||
- name: XControl
|
||||
repo: https://github.com/svc-design/XControl
|
||||
path: docs
|
||||
- name: documents
|
||||
repo: https://github.com/svc-design/documents
|
||||
path: /
|
||||
sync:
|
||||
repo:
|
||||
proxy: socks5://127.0.0.1:1080 # 仅在同步仓库时使用代理
|
||||
|
||||
provider:
|
||||
- name: ollama
|
||||
endpoint: http://localhost:11434
|
||||
models:
|
||||
- 'gpt-oss:20b'
|
||||
- name: chutes
|
||||
endpoint: https://llm.chutes.ai/v1
|
||||
token: "cpk_xxxxxxxxxxxxxxxxxx"
|
||||
models:
|
||||
- 'moonshotai/Kimi-K2-Instruct'
|
||||
|
||||
embedding:
|
||||
endpoint: https://chutes-baai-bge-m3.chutes.ai/embed
|
||||
token: "cpk_xxxxxxxxxxxxxxxxxx"
|
||||
dimension: 0 # 0 = 首次响应自动探测维度
|
||||
rate_limit_tpm: 120000
|
||||
max_batch: 64
|
||||
max_chars: 8000
|
||||
|
||||
chunking:
|
||||
max_tokens: 800
|
||||
overlap_tokens: 80
|
||||
prefer_heading_split: true
|
||||
include_exts: [".md", ".mdx"]
|
||||
ignore_dirs: [".git", "node_modules", "dist", "build"]
|
||||
embed_toc: true
|
||||
|
||||
api:
|
||||
askai:
|
||||
timeout: 100
|
||||
retries: 3
|
||||
5
go.mod
5
go.mod
@ -8,6 +8,7 @@ require (
|
||||
github.com/gin-contrib/cors v1.6.0
|
||||
github.com/gin-gonic/gin v1.9.1
|
||||
github.com/go-git/go-git/v5 v5.16.2
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.7.5
|
||||
github.com/pgvector/pgvector-go v0.3.0
|
||||
@ -45,6 +46,8 @@ require (
|
||||
github.com/go-playground/validator/v10 v10.19.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
@ -72,6 +75,6 @@ require (
|
||||
golang.org/x/sync v0.13.0 // indirect
|
||||
golang.org/x/sys v0.32.0 // indirect
|
||||
golang.org/x/text v0.24.0 // indirect
|
||||
google.golang.org/protobuf v1.34.1 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
)
|
||||
|
||||
11
go.sum
11
go.sum
@ -76,6 +76,8 @@ github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn
|
||||
github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
@ -83,6 +85,11 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
@ -226,8 +233,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
|
||||
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
|
||||
0
go1.24.5.linux-amd64.tar.gz
Normal file
0
go1.24.5.linux-amd64.tar.gz
Normal file
21
scripts/export_schema_clean.sh
Normal file
21
scripts/export_schema_clean.sh
Normal file
@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
DB_URL="postgres://shenlan:password@127.0.0.1:5432/account?sslmode=disable"
|
||||
OUT="/tmp/schema_clean.sql"
|
||||
|
||||
echo ">>> Exporting clean schema (PostgreSQL 16 compatible)"
|
||||
pg_dump \
|
||||
--schema-only \
|
||||
--no-owner \
|
||||
--no-privileges \
|
||||
--exclude-schema=pglogical \
|
||||
"$DB_URL" \
|
||||
| grep -v -i "EXTENSION pglogical" \
|
||||
| grep -v -i "COMMENT ON EXTENSION pglogical" \
|
||||
| grep -v -i "SCHEMA pglogical" \
|
||||
> "$OUT"
|
||||
|
||||
echo "✅ Schema exported to $OUT"
|
||||
grep -i pglogical "$OUT" || echo "✅ pglogical completely removed"
|
||||
|
||||
114
scripts/generate-postgres-tls.sh
Normal file
114
scripts/generate-postgres-tls.sh
Normal file
@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
# ============================================================
|
||||
# PostgreSQL 专用 TLS 证书生成脚本(含 *.svc.plus + 双 IP)
|
||||
# 作者:SVC.PLUS PostgreSQL Server TLS Generator
|
||||
# ============================================================
|
||||
|
||||
TLS_DIR="/etc/postgres-tls"
|
||||
CA_DIR="$TLS_DIR/ca"
|
||||
SERVER_DIR="$TLS_DIR/server"
|
||||
|
||||
echo ">>> [1/6] 创建目录结构 ..."
|
||||
sudo mkdir -p "$CA_DIR" "$SERVER_DIR"
|
||||
cd "$TLS_DIR"
|
||||
|
||||
# ============================================================
|
||||
# 1. 创建私有 CA 根证书
|
||||
# ============================================================
|
||||
echo ">>> [2/6] 生成 PostgreSQL 专用私有 CA ..."
|
||||
sudo openssl genrsa -aes256 -out "$CA_DIR/ca.key.pem" 4096
|
||||
sudo chmod 600 "$CA_DIR/ca.key.pem"
|
||||
|
||||
sudo openssl req -x509 -new -nodes -key "$CA_DIR/ca.key.pem" -sha256 -days 3650 \
|
||||
-subj "/C=CN/O=SVC.PLUS PostgreSQL Authority/OU=DB Security/CN=SVC.PLUS PostgreSQL Root CA" \
|
||||
-out "$CA_DIR/ca.cert.pem"
|
||||
|
||||
# ============================================================
|
||||
# 2. 生成服务器证书
|
||||
# ============================================================
|
||||
echo ">>> [3/6] 生成服务器私钥与 CSR ..."
|
||||
sudo openssl genrsa -out "$SERVER_DIR/server.key.pem" 2048
|
||||
sudo chmod 600 "$SERVER_DIR/server.key.pem"
|
||||
|
||||
sudo openssl req -new -key "$SERVER_DIR/server.key.pem" \
|
||||
-subj "/C=CN/O=SVC.PLUS PostgreSQL Server/OU=DB/CN=global-homepage.svc.plus" \
|
||||
-out "$SERVER_DIR/server.csr.pem"
|
||||
|
||||
# SAN 扩展配置
|
||||
cat <<EOF | sudo tee "$SERVER_DIR/server.ext" >/dev/null
|
||||
basicConstraints=CA:FALSE
|
||||
keyUsage = critical, digitalSignature, keyEncipherment
|
||||
extendedKeyUsage = serverAuth
|
||||
subjectAltName = @alt_names
|
||||
|
||||
[alt_names]
|
||||
DNS.1 = *.svc.plus
|
||||
DNS.2 = svc.plus
|
||||
DNS.3 = global-homepage.svc.plus
|
||||
DNS.4 = cn-homepage.svc.plus
|
||||
IP.1 = 167.179.72.223
|
||||
IP.2 = 47.120.61.35
|
||||
EOF
|
||||
|
||||
# 签发服务器证书(2年有效)
|
||||
echo ">>> [4/6] 使用 SVC.PLUS PostgreSQL Root CA 签发服务器证书 ..."
|
||||
sudo openssl x509 -req -in "$SERVER_DIR/server.csr.pem" \
|
||||
-CA "$CA_DIR/ca.cert.pem" -CAkey "$CA_DIR/ca.key.pem" \
|
||||
-CAcreateserial -out "$SERVER_DIR/server.cert.pem" \
|
||||
-days 730 -sha256 -extfile "$SERVER_DIR/server.ext"
|
||||
|
||||
# fullchain
|
||||
sudo cat "$SERVER_DIR/server.cert.pem" "$CA_DIR/ca.cert.pem" | sudo tee "$SERVER_DIR/server.fullchain.pem" >/dev/null
|
||||
|
||||
# ============================================================
|
||||
# 3. 安装到 PostgreSQL 标准路径
|
||||
# ============================================================
|
||||
echo ">>> [5/6] 安装证书到 PostgreSQL SSL 目录 ..."
|
||||
sudo install -o postgres -g postgres -m 600 "$SERVER_DIR/server.key.pem" /etc/ssl/private/svc.plus-postgres.key
|
||||
sudo install -o postgres -g postgres -m 644 "$SERVER_DIR/server.fullchain.pem" /etc/ssl/certs/svc.plus-postgres.crt
|
||||
sudo install -o postgres -g postgres -m 644 "$CA_DIR/ca.cert.pem" /etc/ssl/certs/svc.plus-postgres-ca.crt
|
||||
|
||||
# ============================================================
|
||||
# 4. 输出后续操作提示
|
||||
# ============================================================
|
||||
echo "==============================================================="
|
||||
echo "✅ [SVC.PLUS PostgreSQL TLS] 已生成并安装完成"
|
||||
echo ""
|
||||
echo "请在 /etc/postgresql/16/main/postgresql.conf 中添加或确认以下配置:"
|
||||
echo ""
|
||||
echo " ssl = on"
|
||||
echo " ssl_cert_file = '/etc/ssl/certs/svc.plus-postgres.crt'"
|
||||
echo " ssl_key_file = '/etc/ssl/private/svc.plus-postgres.key'"
|
||||
echo " ssl_ca_file = '/etc/ssl/certs/svc.plus-postgres-ca.crt'"
|
||||
echo ""
|
||||
echo "⚙️ 然后执行: sudo systemctl restart postgresql"
|
||||
echo ""
|
||||
echo "📦 客户端(订阅端)请复制 CA 根证书:"
|
||||
echo " /etc/postgres-tls/ca/ca.cert.pem"
|
||||
echo "至客户端路径:"
|
||||
echo " /var/lib/postgresql/.postgresql/root.crt"
|
||||
echo "(权限:600,属主 postgres)"
|
||||
echo ""
|
||||
echo "🔍 验证命令示例:"
|
||||
echo " openssl s_client -connect 167.179.72.223:5432 -starttls postgres -servername global-homepage.svc.plus"
|
||||
echo ""
|
||||
echo "👑 证书主题:SVC.PLUS PostgreSQL Server"
|
||||
echo "包含 SAN: *.svc.plus, global-homepage, cn-homepage, IP(167.179.72.223, 47.120.61.35)"
|
||||
echo "==============================================================="
|
||||
|
||||
sudo chown postgres:postgres /etc/ssl/private/svc.plus-postgres.key
|
||||
sudo chmod 600 /etc/ssl/private/svc.plus-postgres.key
|
||||
|
||||
sudo chown root:postgres /etc/ssl/private
|
||||
sudo chmod 750 /etc/ssl/private
|
||||
|
||||
|
||||
# 1️⃣ 创建目录
|
||||
sudo -u postgres mkdir -p /var/lib/postgresql/.postgresql
|
||||
# 2️⃣ 从 global-homepage 拉取服务器的 CA 根证书
|
||||
sudo scp root@167.179.72.223:/etc/ssl/certs/svc.plus-postgres-ca.crt /var/lib/postgresql/.postgresql/root.crt
|
||||
# 3️⃣ 设置权限
|
||||
sudo chown postgres:postgres /var/lib/postgresql/.postgresql/root.crt
|
||||
sudo chmod 600 /var/lib/postgresql/.postgresql/root.crt
|
||||
@ -1,40 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================
|
||||
# 🧹 git-branch-keeper.sh
|
||||
# 保留 main 与所有 release/* 分支
|
||||
# 清理多余本地和远程分支
|
||||
# =============================================
|
||||
|
||||
set -e
|
||||
|
||||
echo ">>> Fetching and pruning remote branches..."
|
||||
git fetch --all --prune
|
||||
|
||||
echo ">>> Cleaning local branches..."
|
||||
for branch in $(git branch | sed 's/*//'); do
|
||||
case "$branch" in
|
||||
main|HEAD|release/*)
|
||||
echo "✅ 保留本地分支:$branch"
|
||||
;;
|
||||
*)
|
||||
echo "🗑️ 删除本地分支:$branch"
|
||||
git branch -D "$branch"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ">>> Cleaning remote branches..."
|
||||
for branch in $(git branch -r | sed 's/origin\///'); do
|
||||
case "$branch" in
|
||||
HEAD|main|release/*)
|
||||
echo "✅ 保留远程分支:origin/$branch"
|
||||
;;
|
||||
*)
|
||||
echo "🗑️ 删除远程分支:origin/$branch"
|
||||
git push origin --delete "$branch" || true
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "✅ 分支清理完成!"
|
||||
|
||||
39
scripts/import_schema_clean.sh
Normal file
39
scripts/import_schema_clean.sh
Normal file
@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# scripts/import_schema_clean.sh
|
||||
# ---------------------------------------------
|
||||
# Import a cleaned schema.sql file into PostgreSQL.
|
||||
# Safe for re-run (幂等导入)
|
||||
# ---------------------------------------------
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ====== Configuration ======
|
||||
DB_URL=${1:-"postgres://shenlan:password@127.0.0.1:5432/account?sslmode=disable"}
|
||||
IN_FILE=${2:-"/tmp/schema_clean.sql"}
|
||||
|
||||
# ====== Validation ======
|
||||
if [ ! -f "$IN_FILE" ]; then
|
||||
echo "❌ File not found: $IN_FILE"
|
||||
echo "💡 请先运行 export_schema_clean.sh 导出 schema"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v psql >/dev/null 2>&1; then
|
||||
echo "❌ 未检测到 psql,请先安装 PostgreSQL 客户端"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ====== Import schema ======
|
||||
echo ">>> Importing schema into database"
|
||||
echo "---------------------------------------------"
|
||||
echo "Database: $DB_URL"
|
||||
echo "Schema: $IN_FILE"
|
||||
echo "---------------------------------------------"
|
||||
|
||||
psql "$DB_URL" -v ON_ERROR_STOP=1 -f "$IN_FILE"
|
||||
|
||||
echo ""
|
||||
echo "✅ Schema import completed successfully"
|
||||
echo "---------------------------------------------"
|
||||
|
||||
@ -98,16 +98,9 @@ install-postgresql() {
|
||||
echo "=== 安装 PostgreSQL ${PG_MAJOR} ==="
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y wget curl gnupg lsb-release ca-certificates
|
||||
if ! grep -q "apt.postgresql.org" /etc/apt/sources.list.d/pgdg.list 2>/dev/null; then
|
||||
curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc -o /tmp/pgdg.asc || \
|
||||
curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xB97B0E2D95A5761FB72B0C18ACCC4CF8" -o /tmp/pgdg.asc
|
||||
sudo gpg --dearmor -o /usr/share/keyrings/postgresql.gpg /tmp/pgdg.asc
|
||||
echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
|
||||
| sudo tee /etc/apt/sources.list.d/pgdg.list
|
||||
sudo apt-get update
|
||||
fi
|
||||
sudo apt-get install -y "postgresql-${PG_MAJOR}" "postgresql-client-${PG_MAJOR}" \
|
||||
"postgresql-contrib-${PG_MAJOR}" "postgresql-server-dev-${PG_MAJOR}"
|
||||
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 2F59B5F99B1BE0B4
|
||||
echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list
|
||||
sudo apt-get install -y "postgresql-${PG_MAJOR}" "postgresql-client-${PG_MAJOR}" "postgresql-contrib-${PG_MAJOR}" "postgresql-server-dev-${PG_MAJOR}"
|
||||
sudo systemctl enable --now postgresql
|
||||
}
|
||||
|
||||
@ -123,8 +116,7 @@ install-pgvector() {
|
||||
sudo apt-get install -y git make gcc
|
||||
tmp_dir=$(mktemp -d)
|
||||
cd "$tmp_dir"
|
||||
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git || \
|
||||
git clone https://ghproxy.com/https://github.com/pgvector/pgvector.git
|
||||
git clone git@github.com:svc-design/pgvector.git
|
||||
cd pgvector
|
||||
make && sudo make install
|
||||
cd /
|
||||
@ -138,8 +130,7 @@ install-zhparser() {
|
||||
# 编译安装 scws v1.2.3
|
||||
tmp_dir=$(mktemp -d)
|
||||
cd "$tmp_dir"
|
||||
git clone https://github.com/hightman/scws.git || \
|
||||
git clone https://ghproxy.com/https://github.com/hightman/scws.git
|
||||
git clone git@github.com:svc-design/scws.git
|
||||
cd scws
|
||||
|
||||
# 修掉 automake 不兼容的注释
|
||||
@ -162,8 +153,7 @@ install-zhparser() {
|
||||
# 编译安装 zhparser
|
||||
tmp_dir=$(mktemp -d)
|
||||
cd "$tmp_dir"
|
||||
git clone https://github.com/amutu/zhparser.git || \
|
||||
git clone https://ghproxy.com/https://github.com/amutu/zhparser.git
|
||||
git clone git@github.com:svc-design/zhparser.git
|
||||
cd zhparser
|
||||
pg_config_path="$(command -v pg_config || echo "/usr/lib/postgresql/${PG_MAJOR}/bin/pg_config")"
|
||||
make SCWS_HOME=/usr PG_CONFIG="${pg_config_path}"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user