merged: docs/pglogical.md

This commit is contained in:
Haitao Pan 2025-10-09 16:35:57 +08:00
commit 2f47c1cfdd
37 changed files with 1420 additions and 2858 deletions

View File

@ -3,7 +3,8 @@ MAIN_FILE := ./cmd/accountsvc/main.go
PORT ?= 8080
OS := $(shell uname -s)
SCHEMA_FILE := ./sql/schema.sql
MIGRATION_FILE := ./sql/20251007-migrate_to_rbac.sql
MIGRATION_FILES := $(shell ls -1 sql/migrations/*.up.sql 2>/dev/null | sort)
DB_NAME := account
DB_USER := shenlan
@ -59,18 +60,24 @@ init-db:
@echo "使用数据库连接: $(DB_URL)"
@psql "$(DB_URL)" -v ON_ERROR_STOP=1 -f $(SCHEMA_FILE)
# =========================================
# migrate-db target
# =========================================
verify-db:
@echo ">>> 执行数据库 schema verify (golang-migrate)"
@go run ./cmd/migratectl/main.go verify --schema sql/schema.sql --dsn "$(DB_URL)" --dir account/sql/migrations
# =========================================
# migrate-db target
# =========================================
migrate-db:
@echo ">>> 执行数据库 schema 迁移 (level/role metadata & MFA/email verification)"
@if [ ! -f $(MIGRATION_FILE) ]; then \
echo "未找到迁移 SQL 文件: $(MIGRATION_FILE)"; \
exit 1; \
fi
@if ! command -v psql >/dev/null 2>&1; then \
echo "未检测到 psql请先安装 PostgreSQL 客户端"; \
exit 1; \
fi
@echo ">>> 执行数据库 schema 迁移 (golang-migrate)"
@echo "--------------------------------------------"
@echo "使用数据库连接: $(DB_URL)"
@psql "$(DB_URL)" -v ON_ERROR_STOP=1 -f $(MIGRATION_FILE)
@if ! command -v go >/dev/null 2>&1; then \
echo "未检测到 go请先安装 Go 环境"; \
exit 1; \
fi
@go run ./cmd/migratectl/main.go migrate --dsn "$(DB_URL)" --dir sql/migrations
# 删除数据库(安全防呆)
drop-db:

View File

@ -0,0 +1,181 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"time"
"github.com/spf13/cobra"
"xcontrol/account/internal/migrate"
)
const (
defaultMigrationDir = "account/sql/migrations"
defaultSchemaFile = "account/sql/schema.sql"
)
func main() {
ctx := context.Background()
rootCmd := newRootCmd()
if err := rootCmd.ExecuteContext(ctx); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func newRootCmd() *cobra.Command {
var migrationDir string
cmd := &cobra.Command{
Use: "migratectl",
Short: "XControl database migration orchestrator",
}
migrationDir = defaultMigrationDir
cmd.PersistentFlags().StringVar(&migrationDir, "dir", migrationDir, "directory containing migration files")
cmd.AddCommand(newMigrateCmd(&migrationDir))
cmd.AddCommand(newCleanCmd())
cmd.AddCommand(newCheckCmd())
cmd.AddCommand(newVerifyCmd())
cmd.AddCommand(newResetCmd(&migrationDir))
cmd.AddCommand(newVersionCmd(&migrationDir))
return cmd
}
func newMigrateCmd(dir *string) *cobra.Command {
var dsn string
cmd := &cobra.Command{
Use: "migrate",
Short: "Apply database migrations",
RunE: func(cmd *cobra.Command, args []string) error {
if dsn == "" {
return errors.New("--dsn is required")
}
runner := migrate.NewRunner(*dir)
ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
defer cancel()
return runner.Up(ctx, dsn)
},
}
cmd.Flags().StringVar(&dsn, "dsn", "", "PostgreSQL connection string")
return cmd
}
func newCleanCmd() *cobra.Command {
var (
dsn string
force bool
)
cmd := &cobra.Command{
Use: "clean",
Short: "Clean leftover database structures",
RunE: func(cmd *cobra.Command, args []string) error {
if dsn == "" {
return errors.New("--dsn is required")
}
cleaner := migrate.NewCleaner()
ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
defer cancel()
return cleaner.Clean(ctx, dsn, force)
},
}
cmd.Flags().StringVar(&dsn, "dsn", "", "PostgreSQL connection string")
cmd.Flags().BoolVar(&force, "force", false, "Confirm clean-up actions")
return cmd
}
func newCheckCmd() *cobra.Command {
var (
cnDSN string
globalDSN string
autoFix bool
)
cmd := &cobra.Command{
Use: "check",
Short: "Compare CN and Global schemas",
RunE: func(cmd *cobra.Command, args []string) error {
checker := migrate.NewChecker()
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Minute)
defer cancel()
return checker.Check(ctx, cnDSN, globalDSN, autoFix)
},
}
cmd.Flags().StringVar(&cnDSN, "cn", "", "CN region PostgreSQL DSN")
cmd.Flags().StringVar(&globalDSN, "global", "", "Global region PostgreSQL DSN")
cmd.Flags().BoolVar(&autoFix, "auto-fix", false, "Automatically apply missing statements to CN")
return cmd
}
func newVerifyCmd() *cobra.Command {
var (
dsn string
schemaPath string
)
cmd := &cobra.Command{
Use: "verify",
Short: "Verify that the database matches schema.sql",
RunE: func(cmd *cobra.Command, args []string) error {
if dsn == "" {
return errors.New("--dsn is required")
}
if schemaPath == "" {
schemaPath = defaultSchemaFile
}
verifier := migrate.NewVerifier()
ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
defer cancel()
return verifier.Verify(ctx, dsn, schemaPath)
},
}
cmd.Flags().StringVar(&dsn, "dsn", "", "PostgreSQL connection string")
cmd.Flags().StringVar(&schemaPath, "schema", defaultSchemaFile, "Path to schema.sql reference file")
return cmd
}
func newResetCmd(dir *string) *cobra.Command {
var dsn string
cmd := &cobra.Command{
Use: "reset",
Short: "Drop public schema and re-run migrations",
RunE: func(cmd *cobra.Command, args []string) error {
if dsn == "" {
return errors.New("--dsn is required")
}
runner := migrate.NewRunner(*dir)
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Minute)
defer cancel()
return runner.Reset(ctx, dsn)
},
}
cmd.Flags().StringVar(&dsn, "dsn", "", "PostgreSQL connection string")
return cmd
}
func newVersionCmd(dir *string) *cobra.Command {
var dsn string
cmd := &cobra.Command{
Use: "version",
Short: "Show current migration version",
RunE: func(cmd *cobra.Command, args []string) error {
if dsn == "" {
return errors.New("--dsn is required")
}
runner := migrate.NewRunner(*dir)
version, dirty, err := runner.Version(dsn)
if err != nil {
return err
}
if dirty {
fmt.Printf("Current migration version: %d (dirty)\n", version)
} else {
fmt.Printf("Current migration version: %d\n", version)
}
return nil
},
}
cmd.Flags().StringVar(&dsn, "dsn", "", "PostgreSQL connection string")
return cmd
}

View File

@ -0,0 +1,102 @@
package migrate
import (
"context"
"errors"
"fmt"
"strings"
"xcontrol/account/internal/utils"
)
// Checker compares two PostgreSQL schemas (CN vs Global) and optionally fixes
// missing structures on the CN side.
type Checker struct{}
func NewChecker() *Checker {
return &Checker{}
}
func (c *Checker) Check(ctx context.Context, cnDSN, globalDSN string, autoFix bool) error {
if cnDSN == "" || globalDSN == "" {
return errors.New("both --cn and --global DSNs are required")
}
cnDump, err := utils.RunPgDump(ctx, cnDSN)
if err != nil {
return fmt.Errorf("dump CN schema: %w", err)
}
globalDump, err := utils.RunPgDump(ctx, globalDSN)
if err != nil {
return fmt.Errorf("dump Global schema: %w", err)
}
cnStatements := utils.NormalizeStatements(cnDump)
globalStatements := utils.NormalizeStatements(globalDump)
onlyCN, onlyGlobal := utils.CompareStatements(cnStatements, globalStatements)
if len(onlyCN) == 0 && len(onlyGlobal) == 0 {
fmt.Println("✅ CN and Global schemas are consistent")
return nil
}
if len(onlyGlobal) > 0 {
fmt.Println("⚠️ Statements missing on CN (present on Global):")
for _, stmt := range onlyGlobal {
fmt.Printf(" + %s;\n", stmt)
}
}
if len(onlyCN) > 0 {
fmt.Println("⚠️ Statements only found on CN:")
for _, stmt := range onlyCN {
fmt.Printf(" - %s;\n", stmt)
}
}
if autoFix && len(onlyGlobal) > 0 {
if err := applyStatements(ctx, cnDSN, onlyGlobal); err != nil {
return fmt.Errorf("apply auto-fix: %w", err)
}
fmt.Println("✅ Auto-fix applied on CN database — please re-run check to confirm")
// After auto-fix we still return an error if CN has extra statements.
if len(onlyCN) > 0 {
return errors.New("schema differences remain on CN after auto-fix")
}
return nil
}
return errors.New("schema differences detected")
}
func applyStatements(ctx context.Context, dsn string, statements []string) error {
db, err := openDB(ctx, dsn)
if err != nil {
return err
}
defer db.Close()
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
for _, stmt := range statements {
if strings.Contains(stmt, "pglogical") {
continue
}
sql := stmt
if !strings.HasSuffix(sql, ";") {
sql += ";"
}
fmt.Printf("→ Applying fix: %s\n", sql)
if _, err := tx.ExecContext(ctx, sql); err != nil {
return err
}
fmt.Printf("✅ Applied fix\n")
}
return tx.Commit()
}

View File

@ -0,0 +1,138 @@
package migrate
import (
"context"
"database/sql"
"errors"
"fmt"
)
// Cleaner removes leftover structures such as invalid indexes or disabled
// triggers while keeping pglogical untouched.
type Cleaner struct{}
func NewCleaner() *Cleaner {
return &Cleaner{}
}
func (c *Cleaner) Clean(ctx context.Context, dsn string, force bool) error {
if !force {
return errors.New("clean requires --force confirmation")
}
db, err := openDB(ctx, dsn)
if err != nil {
return err
}
defer db.Close()
tx, err := db.BeginTx(ctx, &sql.TxOptions{})
if err != nil {
return err
}
defer tx.Rollback()
if err := dropInvalidIndexes(ctx, tx); err != nil {
return err
}
if err := dropDisabledTriggers(ctx, tx); err != nil {
return err
}
if err := dropTemporaryTables(ctx, tx); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return err
}
fmt.Println("✅ Database clean-up completed")
return nil
}
func dropInvalidIndexes(ctx context.Context, tx *sql.Tx) error {
rows, err := tx.QueryContext(ctx, `
SELECT quote_ident(n.nspname) || '.' || quote_ident(c.relname)
FROM pg_index i
JOIN pg_class c ON c.oid = i.indexrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname NOT IN ('pglogical', 'pg_catalog', 'information_schema')
AND (NOT i.indisvalid OR NOT i.indisready)
`)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var identifier string
if err := rows.Scan(&identifier); err != nil {
return err
}
fmt.Printf("→ Dropping invalid index %s\n", identifier)
if _, err := tx.ExecContext(ctx, fmt.Sprintf("DROP INDEX IF EXISTS %s", identifier)); err != nil {
return err
}
fmt.Printf("✅ Dropped index %s\n", identifier)
}
return rows.Err()
}
func dropDisabledTriggers(ctx context.Context, tx *sql.Tx) error {
rows, err := tx.QueryContext(ctx, `
SELECT quote_ident(n.nspname) || '.' || quote_ident(rel.relname) AS tbl,
quote_ident(t.tgname) AS trigger_name
FROM pg_trigger t
JOIN pg_class rel ON rel.oid = t.tgrelid
JOIN pg_namespace n ON n.oid = rel.relnamespace
WHERE t.tgenabled = 'D'
AND t.tgisinternal = false
AND n.nspname NOT IN ('pglogical', 'pg_catalog', 'information_schema')
`)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var table, trigger string
if err := rows.Scan(&table, &trigger); err != nil {
return err
}
fmt.Printf("→ Dropping disabled trigger %s on %s\n", trigger, table)
if _, err := tx.ExecContext(ctx, fmt.Sprintf("DROP TRIGGER IF EXISTS %s ON %s", trigger, table)); err != nil {
return err
}
fmt.Printf("✅ Dropped trigger %s on %s\n", trigger, table)
}
return rows.Err()
}
func dropTemporaryTables(ctx context.Context, tx *sql.Tx) error {
rows, err := tx.QueryContext(ctx, `
SELECT quote_ident(table_schema) || '.' || quote_ident(table_name)
FROM information_schema.tables
WHERE table_schema NOT IN ('pglogical', 'pg_catalog', 'information_schema')
AND (table_name LIKE 'tmp_%' OR table_name LIKE 'temp_%' OR table_name LIKE 'backup_%')
`)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var table string
if err := rows.Scan(&table); err != nil {
return err
}
fmt.Printf("→ Dropping temporary table %s\n", table)
if _, err := tx.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", table)); err != nil {
return err
}
fmt.Printf("✅ Dropped table %s\n", table)
}
return rows.Err()
}

View File

@ -0,0 +1,28 @@
package migrate
import (
"context"
"database/sql"
"time"
_ "github.com/jackc/pgx/v5/stdlib"
)
func openDB(ctx context.Context, dsn string) (*sql.DB, error) {
db, err := sql.Open("pgx", dsn)
if err != nil {
return nil, err
}
db.SetConnMaxLifetime(0)
db.SetConnMaxIdleTime(5 * time.Minute)
db.SetMaxIdleConns(5)
db.SetMaxOpenConns(10)
if err := db.PingContext(ctx); err != nil {
db.Close()
return nil, err
}
return db, nil
}

View File

@ -0,0 +1,183 @@
package migrate
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"github.com/golang-migrate/migrate/v4"
)
const defaultDir = "account/sql/migrations"
// Runner coordinates golang-migrate operations.
type Runner struct {
Dir string
}
// NewRunner creates a runner that reads migration files from dir. When dir is
// empty, the default directory under account/sql/migrations is used.
func NewRunner(dir string) *Runner {
if dir == "" {
dir = defaultDir
}
return &Runner{Dir: dir}
}
// Up executes all migrations that have not been applied yet. Each step logs
// its outcome to provide clear visibility.
func (r *Runner) Up(ctx context.Context, dsn string) error {
absDir, err := filepath.Abs(r.Dir)
if err != nil {
return err
}
migrations, err := r.loadMigrations(absDir)
if err != nil {
return err
}
m, err := migrate.New(fmt.Sprintf("file://%s", absDir), dsn)
if err != nil {
return err
}
defer closeMigrator(m)
currentVersion, dirty, err := m.Version()
if err != nil {
if errors.Is(err, migrate.ErrNilVersion) {
currentVersion = 0
} else {
return fmt.Errorf("fetch current version: %w", err)
}
}
if dirty {
return fmt.Errorf("database is in a dirty state at version %d; please fix manually", currentVersion)
}
applied := false
for _, migration := range migrations {
if migration.version <= currentVersion {
continue
}
fmt.Printf("→ Applying migration %s ...\n", migration.name)
if err := m.Migrate(migration.version); err != nil {
if errors.Is(err, migrate.ErrNoChange) {
fmt.Printf("✅ Migration %s already applied\n", migration.name)
continue
}
return fmt.Errorf("apply migration %s: %w", migration.name, err)
}
applied = true
fmt.Printf("✅ Migration %s applied\n", migration.name)
}
if !applied {
fmt.Println("✅ Database schema already up-to-date")
}
return nil
}
// Version reports the current schema version tracked by golang-migrate.
func (r *Runner) Version(dsn string) (uint, bool, error) {
absDir, err := filepath.Abs(r.Dir)
if err != nil {
return 0, false, err
}
m, err := migrate.New(fmt.Sprintf("file://%s", absDir), dsn)
if err != nil {
return 0, false, err
}
defer closeMigrator(m)
version, dirty, err := m.Version()
if err != nil {
if errors.Is(err, migrate.ErrNilVersion) {
return 0, false, nil
}
return 0, false, err
}
return version, dirty, nil
}
// Reset drops the public schema before replaying all migrations.
func (r *Runner) Reset(ctx context.Context, dsn string) error {
db, err := openDB(ctx, dsn)
if err != nil {
return err
}
defer db.Close()
fmt.Println("⚠️ Dropping public schema (preserving pglogical)...")
if _, err := db.ExecContext(ctx, "DROP SCHEMA IF EXISTS public CASCADE"); err != nil {
return fmt.Errorf("drop public schema: %w", err)
}
if _, err := db.ExecContext(ctx, "CREATE SCHEMA IF NOT EXISTS public"); err != nil {
return fmt.Errorf("recreate public schema: %w", err)
}
return r.Up(ctx, dsn)
}
func (r *Runner) loadMigrations(absDir string) ([]*migrationFile, error) {
entries, err := os.ReadDir(absDir)
if err != nil {
return nil, err
}
migrationMap := make(map[uint]*migrationFile)
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if !strings.HasSuffix(name, ".up.sql") {
continue
}
parts := strings.SplitN(name, "_", 2)
if len(parts) != 2 {
continue
}
version, err := strconv.ParseUint(parts[0], 10, 64)
if err != nil {
continue
}
migrationMap[uint(version)] = &migrationFile{
version: uint(version),
name: name,
}
}
var migrations []*migrationFile
for _, m := range migrationMap {
migrations = append(migrations, m)
}
sort.Slice(migrations, func(i, j int) bool {
return migrations[i].version < migrations[j].version
})
return migrations, nil
}
func closeMigrator(m *migrate.Migrate) {
if m == nil {
return
}
_, _ = m.Close()
}
type migrationFile struct {
version uint
name string
}

View File

@ -0,0 +1,64 @@
package migrate
import (
"context"
"errors"
"fmt"
"os"
"xcontrol/account/internal/utils"
)
const defaultSchemaPath = "account/sql/schema.sql"
// Verifier validates that the live database matches the canonical schema.sql.
type Verifier struct{}
func NewVerifier() *Verifier {
return &Verifier{}
}
func (v *Verifier) Verify(ctx context.Context, dsn, schemaPath string) error {
if dsn == "" {
return errors.New("--dsn is required")
}
if schemaPath == "" {
schemaPath = defaultSchemaPath
}
dump, err := utils.RunPgDump(ctx, dsn)
if err != nil {
return fmt.Errorf("dump database schema: %w", err)
}
schemaBytes, err := os.ReadFile(schemaPath)
if err != nil {
return fmt.Errorf("read schema file: %w", err)
}
dbStatements := utils.NormalizeStatements(dump)
fileStatements := utils.NormalizeStatements(string(schemaBytes))
onlyDB, onlyFile := utils.CompareStatements(dbStatements, fileStatements)
if len(onlyDB) == 0 && len(onlyFile) == 0 {
fmt.Println("✅ Database schema matches schema.sql")
return nil
}
if len(onlyFile) > 0 {
fmt.Println("⚠️ Statements missing from database (present in schema.sql):")
for _, stmt := range onlyFile {
fmt.Printf(" + %s;\n", stmt)
}
}
if len(onlyDB) > 0 {
fmt.Println("⚠️ Extra statements found in database:")
for _, stmt := range onlyDB {
fmt.Printf(" - %s;\n", stmt)
}
}
return errors.New("schema mismatch detected")
}

View File

@ -0,0 +1,132 @@
package utils
import (
"bufio"
"sort"
"strings"
)
// NormalizeStatements extracts relevant DDL statements from the provided dump.
// Comments, empty lines and helper SET statements are ignored. Whitespace is
// normalised to make comparisons deterministic across environments.
func NormalizeStatements(dump string) []string {
var statements []string
var builder strings.Builder
flush := func() {
stmt := strings.TrimSpace(builder.String())
if stmt == "" {
builder.Reset()
return
}
stmt = strings.TrimSuffix(stmt, ";")
stmt = strings.TrimSpace(stmt)
if isRelevantStatement(stmt) {
normalized := collapseWhitespace(stmt)
statements = append(statements, normalized)
}
builder.Reset()
}
scanner := bufio.NewScanner(strings.NewReader(dump))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "--") {
continue
}
if shouldSkipLine(line) {
continue
}
builder.WriteString(line)
if strings.HasSuffix(line, ";") {
flush()
} else {
builder.WriteString(" ")
}
}
flush()
sort.Strings(statements)
return statements
}
// CompareStatements returns the statements that are present only on the left
// or only on the right collection.
func CompareStatements(left, right []string) (onlyLeft, onlyRight []string) {
leftSet := make(map[string]struct{}, len(left))
for _, stmt := range left {
if strings.Contains(stmt, "pglogical") {
continue
}
leftSet[stmt] = struct{}{}
}
rightSet := make(map[string]struct{}, len(right))
for _, stmt := range right {
if strings.Contains(stmt, "pglogical") {
continue
}
rightSet[stmt] = struct{}{}
}
for stmt := range leftSet {
if _, ok := rightSet[stmt]; !ok {
onlyLeft = append(onlyLeft, stmt)
}
}
for stmt := range rightSet {
if _, ok := leftSet[stmt]; !ok {
onlyRight = append(onlyRight, stmt)
}
}
sort.Strings(onlyLeft)
sort.Strings(onlyRight)
return
}
func shouldSkipLine(line string) bool {
lower := strings.ToLower(line)
switch {
case strings.HasPrefix(lower, "set "):
return true
case strings.HasPrefix(lower, "select pg_catalog.set_config"):
return true
case strings.HasPrefix(lower, "reset "):
return true
case strings.HasPrefix(line, "\\connect "):
return true
case strings.HasPrefix(lower, "lock table"):
return true
}
return false
}
func isRelevantStatement(stmt string) bool {
lower := strings.ToLower(stmt)
switch {
case strings.HasPrefix(lower, "create table"):
return true
case strings.HasPrefix(lower, "alter table"):
return true
case strings.HasPrefix(lower, "create index"):
return true
case strings.HasPrefix(lower, "alter index"):
return true
case strings.HasPrefix(lower, "comment on table"):
return true
case strings.HasPrefix(lower, "comment on column"):
return true
case strings.HasPrefix(lower, "grant "):
return true
}
return false
}
func collapseWhitespace(input string) string {
fields := strings.Fields(input)
return strings.Join(fields, " ")
}

View File

@ -0,0 +1,56 @@
package utils
import (
"bytes"
"context"
"fmt"
"os"
"os/exec"
"strings"
)
// CommandResult captures stdout and stderr from an executed command.
type CommandResult struct {
Stdout string
Stderr string
}
// RunCommand executes the provided command with context awareness and
// returns the collected stdout/stderr output. Errors include contextual
// information as well as stderr to make troubleshooting easier.
func RunCommand(ctx context.Context, name string, args ...string) (*CommandResult, error) {
cmd := exec.CommandContext(ctx, name, args...)
cmd.Env = os.Environ()
var stdoutBuf, stderrBuf bytes.Buffer
cmd.Stdout = &stdoutBuf
cmd.Stderr = &stderrBuf
err := cmd.Run()
result := &CommandResult{Stdout: stdoutBuf.String(), Stderr: stderrBuf.String()}
if err != nil {
return result, fmt.Errorf("command %s %s failed: %w\n%s", name, strings.Join(args, " "), err, result.Stderr)
}
return result, nil
}
// RunPgDump executes pg_dump with the flags required for schema comparison
// and returns the textual dump. The pglogical schema is excluded to avoid
// touching logical replication internals.
func RunPgDump(ctx context.Context, dsn string) (string, error) {
args := []string{
"--schema-only",
"--no-owner",
"--no-privileges",
"--exclude-schema=pglogical",
"--dbname", dsn,
}
result, err := RunCommand(ctx, "pg_dump", args...)
if err != nil {
return "", err
}
return result.Stdout, nil
}

View File

@ -0,0 +1 @@
-- No-op down migration; manual rollback required.

View File

@ -0,0 +1 @@
-- No-op down migration; manual rollback required.

View File

@ -0,0 +1 @@
-- No-op down migration; manual rollback required.

View File

@ -0,0 +1,97 @@
-- =========================================
-- 20251008-fix-generated-columns.sql
-- Migration: Remove generated columns incompatible with pglogical
-- Safe to re-run (幂等)
-- =========================================
BEGIN;
-- 尝试确保当前用户对 pglogical schema 有访问权限
DO $grant$
BEGIN
BEGIN
EXECUTE 'GRANT USAGE ON SCHEMA pglogical TO ' || current_user;
EXECUTE 'GRANT ALL ON ALL TABLES IN SCHEMA pglogical TO ' || current_user;
EXECUTE 'GRANT ALL ON ALL SEQUENCES IN SCHEMA pglogical TO ' || current_user;
EXECUTE 'GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pglogical TO ' || current_user;
RAISE NOTICE 'Granted pglogical schema permissions to current_user: %', current_user;
EXCEPTION WHEN others THEN
RAISE NOTICE 'Skipping GRANT for schema pglogical (possibly already granted or insufficient privilege)';
END;
END;
$grant$;
-- =========================================
-- Step 1. 检查并删除旧的 generated column
-- =========================================
DO $drop$
BEGIN
IF EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'users'
AND column_name = 'email_verified'
AND is_generated = 'ALWAYS'
) THEN
RAISE NOTICE 'Detected generated column: users.email_verified — dropping it for pglogical compatibility';
EXECUTE 'ALTER TABLE public.users DROP COLUMN email_verified';
ELSE
RAISE NOTICE 'No generated column detected: users.email_verified is safe';
END IF;
END;
$drop$;
-- =========================================
-- Step 2. 添加普通布尔列 (safe re-run)
-- =========================================
DO $add$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'users'
AND column_name = 'email_verified'
) THEN
RAISE NOTICE 'Adding normal column: users.email_verified (BOOLEAN DEFAULT false)';
EXECUTE 'ALTER TABLE public.users ADD COLUMN email_verified BOOLEAN DEFAULT false NOT NULL';
ELSE
RAISE NOTICE 'Column users.email_verified already exists, skipping ADD COLUMN';
END IF;
END;
$add$;
-- =========================================
-- Step 3. 创建维护触发器
-- =========================================
DROP FUNCTION IF EXISTS maintain_email_verified() CASCADE;
CREATE FUNCTION maintain_email_verified() RETURNS trigger
LANGUAGE plpgsql
AS $$
BEGIN
NEW.email_verified := (NEW.email_verified_at IS NOT NULL);
RETURN NEW;
END;
$$;
DROP TRIGGER IF EXISTS trg_users_maintain_email_verified ON public.users;
CREATE TRIGGER trg_users_maintain_email_verified
BEFORE INSERT OR UPDATE ON public.users
FOR EACH ROW EXECUTE FUNCTION maintain_email_verified();
-- =========================================
-- Step 4. 修正现有数据
-- =========================================
UPDATE public.users
SET email_verified = (email_verified_at IS NOT NULL)
WHERE email_verified IS DISTINCT FROM (email_verified_at IS NOT NULL);
COMMIT;
-- =========================================
-- End of migration
-- =========================================

View File

@ -0,0 +1 @@
-- No-op down migration; manual rollback required.

View File

@ -0,0 +1,86 @@
-- =========================================
-- 20251008.migrate.generated-columns.sql
-- Migration: Convert generated columns to normal + trigger
-- Safe to re-run (幂等)
-- =========================================
BEGIN;
-- 1⃣ 兼容 pglogical确保当前用户可访问 schema
DO $$
BEGIN
BEGIN
GRANT USAGE ON SCHEMA pglogical TO CURRENT_USER;
RAISE NOTICE 'Granted USAGE on schema pglogical to current user';
EXCEPTION WHEN OTHERS THEN
RAISE NOTICE 'Skipping GRANT for schema pglogical (possibly already granted or insufficient privilege)';
END;
END;
$$;
-- 2⃣ 删除不兼容的 generated column
DO $$
BEGIN
IF EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'users'
AND column_name = 'email_verified'
AND is_generated = 'ALWAYS'
) THEN
RAISE NOTICE 'Detected generated column: users.email_verified — dropping it for pglogical compatibility';
ALTER TABLE public.users DROP COLUMN email_verified;
ELSE
RAISE NOTICE 'No generated column detected: users.email_verified is safe';
END IF;
END;
$$;
-- 3⃣ 确保存在普通列 email_verified
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'users'
AND column_name = 'email_verified'
) THEN
RAISE NOTICE 'Adding normal column: users.email_verified (BOOLEAN DEFAULT false)';
ALTER TABLE public.users ADD COLUMN email_verified BOOLEAN DEFAULT false NOT NULL;
ELSE
RAISE NOTICE 'Column users.email_verified already exists, skipping ADD COLUMN';
END IF;
END;
$$;
-- 4⃣ 定义幂等触发器函数
DROP FUNCTION IF EXISTS public.maintain_email_verified() CASCADE;
CREATE FUNCTION public.maintain_email_verified() RETURNS trigger
LANGUAGE plpgsql
AS $$
BEGIN
NEW.email_verified := (NEW.email_verified_at IS NOT NULL);
RETURN NEW;
END;
$$;
-- 5⃣ 触发器同步 email_verified 与 email_verified_at
DROP TRIGGER IF EXISTS trg_users_maintain_email_verified ON public.users;
CREATE TRIGGER trg_users_maintain_email_verified
BEFORE INSERT OR UPDATE ON public.users
FOR EACH ROW EXECUTE FUNCTION public.maintain_email_verified();
-- 6⃣ 修正历史数据
UPDATE public.users
SET email_verified = (email_verified_at IS NOT NULL)
WHERE email_verified IS DISTINCT FROM (email_verified_at IS NOT NULL);
COMMIT;
-- =========================================
-- End of migration
-- =========================================

53
account/sql/readme.md Normal file
View File

@ -0,0 +1,53 @@
使用新的 `migratectl` CLI 可以在不同环境下快速执行迁移、校验和重置操作:
```bash
go run ./cmd/migratectl/main.go migrate --dsn "$DB_URL"
go run ./cmd/migratectl/main.go check --cn "$CN_DSN" --global "$GLOBAL_DSN"
```
以下命令展示了如何授予 pglogical schema 访问权限:
sudo -u postgres psql -d account -c "GRANT USAGE ON SCHEMA pglogical TO PUBLIC;"
-- 登录 postgres
sudo -u postgres psql -d account
-- 授权 shenlan 对 public schema 全权限
ALTER SCHEMA public OWNER TO shenlan;
GRANT ALL ON SCHEMA public TO shenlan;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO shenlan;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO shenlan;
GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public TO shenlan;
-- 授权 pglogical schema 使用权限(防止混用)
GRANT USAGE ON SCHEMA pglogical TO shenlan;
\q
执行顺序建议
步骤 节点 脚本 说明
1⃣ Global schema_base.sql 创建业务结构
2⃣ CN schema_base.sql 创建相同业务结构
3⃣ Global schema_pglogical_region_global.sql 定义 Global provider + 订阅 CN
4⃣ CN schema_pglogical_region_cn.sql 定义 CN provider + 订阅 Global
🧩 验证同步状态
SELECT * FROM pglogical.show_subscription_status();
如果输出中:
status = 'replicating'
即表示 双向复制同步正常。
🚀 双向同步特性汇总
特性 实现机制
双主写入 两端都是 Provider + Subscriber
唯一性保障 所有主键为 gen_random_uuid(),避免冲突
邮箱唯一 lower(email) 唯一索引
异步复制 WAL 级逻辑同步,自动断点续传
结构一致性 schema_base.sql 保证完全相同
幂等可重建 全部 IF NOT EXISTS可重复执行
可扩展性 可新增字段或表,通过 replication_set_add_table() 同步

View File

@ -1,30 +1,34 @@
-- =========================================
-- PostgreSQL schema initialization script
-- Safe to re-run (幂等)
-- schema_base.sql
-- Shared schema for pglogical bidirectional sync
-- PostgreSQL 16 + gen_random_uuid()
-- =========================================
SET statement_timeout = 0;
SET lock_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SET row_security = off;
SET search_path = public;
DROP SCHEMA IF EXISTS public CASCADE;
CREATE SCHEMA public AUTHORIZATION postgres;
-- 必要扩展
-- =========================================
-- Extensions
-- =========================================
CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;
CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA public;
CREATE EXTENSION IF NOT EXISTS pglogical WITH SCHEMA pglogical;
-- =========================================
-- Function: set_updated_at()
-- Functions
-- =========================================
DROP FUNCTION IF EXISTS set_updated_at() CASCADE;
CREATE FUNCTION set_updated_at() RETURNS trigger
LANGUAGE plpgsql
AS $$
CREATE OR REPLACE FUNCTION public.set_updated_at() RETURNS trigger
LANGUAGE plpgsql AS $$
BEGIN
NEW.updated_at := now();
RETURN NEW;
NEW.updated_at := now();
RETURN NEW;
END;
$$;
CREATE OR REPLACE FUNCTION public.maintain_email_verified() RETURNS trigger
LANGUAGE plpgsql AS $$
BEGIN
NEW.email_verified := (NEW.email_verified_at IS NOT NULL);
RETURN NEW;
END;
$$;
@ -32,84 +36,84 @@ $$;
-- Tables
-- =========================================
CREATE TABLE IF NOT EXISTS users (
username TEXT NOT NULL,
password TEXT NOT NULL,
email TEXT,
level INTEGER DEFAULT 20 NOT NULL,
role TEXT DEFAULT 'user' NOT NULL,
groups JSONB DEFAULT '[]'::jsonb NOT NULL,
permissions JSONB DEFAULT '[]'::jsonb NOT NULL,
created_at TIMESTAMPTZ DEFAULT now(),
uuid UUID DEFAULT gen_random_uuid() NOT NULL,
mfa_totp_secret TEXT,
mfa_enabled BOOLEAN DEFAULT false NOT NULL,
mfa_secret_issued_at TIMESTAMPTZ,
mfa_confirmed_at TIMESTAMPTZ,
updated_at TIMESTAMPTZ DEFAULT now(),
email_verified_at TIMESTAMPTZ,
email_verified BOOLEAN GENERATED ALWAYS AS ((email_verified_at IS NOT NULL)) STORED,
CONSTRAINT users_pkey PRIMARY KEY (uuid),
CONSTRAINT users_username_uk UNIQUE (username),
CONSTRAINT users_email_uk UNIQUE (email)
CREATE TABLE public.users (
uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
username TEXT NOT NULL,
password TEXT NOT NULL,
email TEXT,
role TEXT NOT NULL DEFAULT 'user',
level INTEGER NOT NULL DEFAULT 20,
groups JSONB NOT NULL DEFAULT '[]'::jsonb,
permissions JSONB NOT NULL DEFAULT '[]'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
mfa_totp_secret TEXT,
mfa_enabled BOOLEAN NOT NULL DEFAULT FALSE,
mfa_secret_issued_at TIMESTAMPTZ,
mfa_confirmed_at TIMESTAMPTZ,
email_verified_at TIMESTAMPTZ,
email_verified BOOLEAN GENERATED ALWAYS AS ((email_verified_at IS NOT NULL)) STORED
);
CREATE UNIQUE INDEX IF NOT EXISTS users_username_lower_uk ON users (lower(username));
CREATE UNIQUE INDEX IF NOT EXISTS users_email_lower_uk ON users (lower(email)) WHERE email IS NOT NULL;
CREATE TABLE IF NOT EXISTS identities (
provider TEXT NOT NULL,
external_id TEXT NOT NULL,
uuid UUID DEFAULT gen_random_uuid() NOT NULL,
user_uuid UUID NOT NULL,
CONSTRAINT identities_pkey PRIMARY KEY (uuid),
CONSTRAINT identities_provider_external_id_uk UNIQUE (provider, external_id),
CONSTRAINT identities_user_fk FOREIGN KEY (user_uuid)
REFERENCES users(uuid) ON DELETE CASCADE
CREATE TABLE public.identities (
uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
provider TEXT NOT NULL,
external_id TEXT NOT NULL,
user_uuid UUID NOT NULL REFERENCES public.users(uuid) ON DELETE CASCADE,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT identities_provider_external_id_uk UNIQUE (provider, external_id)
);
CREATE TABLE IF NOT EXISTS sessions (
token TEXT NOT NULL,
expires_at TIMESTAMPTZ NOT NULL,
uuid UUID DEFAULT gen_random_uuid() NOT NULL,
user_uuid UUID NOT NULL,
CONSTRAINT sessions_pkey PRIMARY KEY (uuid),
CONSTRAINT sessions_user_fk FOREIGN KEY (user_uuid)
REFERENCES users(uuid) ON DELETE CASCADE
CREATE TABLE public.sessions (
uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
token TEXT NOT NULL,
expires_at TIMESTAMPTZ NOT NULL,
user_uuid UUID NOT NULL REFERENCES public.users(uuid) ON DELETE CASCADE,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS admin_settings (
id BIGSERIAL PRIMARY KEY,
module_key TEXT NOT NULL,
role TEXT NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT FALSE,
version BIGINT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT admin_settings_module_role_uk UNIQUE (module_key, role)
CREATE TABLE public.admin_settings (
uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
module_key TEXT NOT NULL,
role TEXT NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT FALSE,
version BIGINT NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT admin_settings_module_role_uk UNIQUE (module_key, role)
);
-- =========================================
-- Indexes
-- =========================================
CREATE INDEX IF NOT EXISTS idx_identities_provider ON identities (provider);
CREATE INDEX IF NOT EXISTS idx_identities_user_uuid ON identities (user_uuid);
CREATE INDEX IF NOT EXISTS idx_sessions_user_uuid ON sessions (user_uuid);
CREATE INDEX IF NOT EXISTS idx_admin_settings_version ON admin_settings (version);
CREATE UNIQUE INDEX users_username_lower_uk ON public.users (lower(username));
CREATE UNIQUE INDEX users_email_lower_uk ON public.users (lower(email)) WHERE email IS NOT NULL;
CREATE INDEX idx_identities_user_uuid ON public.identities (user_uuid);
CREATE INDEX idx_sessions_user_uuid ON public.sessions (user_uuid);
CREATE INDEX idx_admin_settings_version ON public.admin_settings (version);
-- =========================================
-- Trigger
-- Triggers
-- =========================================
DROP TRIGGER IF EXISTS trg_users_set_updated_at ON users;
CREATE TRIGGER trg_users_set_updated_at
BEFORE UPDATE ON users
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
BEFORE UPDATE ON public.users
FOR EACH ROW EXECUTE FUNCTION public.set_updated_at();
CREATE TRIGGER trg_users_maintain_email_verified
BEFORE INSERT OR UPDATE ON public.users
FOR EACH ROW EXECUTE FUNCTION public.maintain_email_verified();
CREATE TRIGGER trg_identities_set_updated_at
BEFORE UPDATE ON public.identities
FOR EACH ROW EXECUTE FUNCTION public.set_updated_at();
CREATE TRIGGER trg_sessions_set_updated_at
BEFORE UPDATE ON public.sessions
FOR EACH ROW EXECUTE FUNCTION public.set_updated_at();
DROP TRIGGER IF EXISTS trg_admin_settings_set_updated_at ON admin_settings;
CREATE TRIGGER trg_admin_settings_set_updated_at
BEFORE UPDATE ON admin_settings
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
BEFORE UPDATE ON public.admin_settings
FOR EACH ROW EXECUTE FUNCTION public.set_updated_at();
-- =========================================
-- End of schema.sql
-- =========================================

View File

@ -1,151 +0,0 @@
# ==============================================
# XControl Homepage — Production (Dynamic Render)
# Structure: Nginx → Node.js(3000) → Go(8080/8090)
# ==============================================
server {
listen 80;
server_name www.svc.plus global-homepage.svc.plus;
return 301 https://$host$request_uri;
}
server {
listen 443 ssl http2;
server_name www.svc.plus global-homepage.svc.plus;
ssl_certificate /etc/ssl/svc.plus.pem;
ssl_certificate_key /etc/ssl/svc.plus.rsa.key;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
# ===============================
# 基本安全头(推荐保持)
# ===============================
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
add_header X-Frame-Options "SAMEORIGIN";
add_header X-Content-Type-Options "nosniff";
add_header Referrer-Policy "strict-origin-when-cross-origin";
add_header X-XSS-Protection "1; mode=block";
# ===============================
# API 反向代理
# ===============================
# 优先匹配认证服务
location ^~ /api/auth/ {
proxy_pass http://127.0.0.1:8080;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# 其他 /api/ 请求转 server 服务
location ^~ /api/ {
proxy_pass http://127.0.0.1:8090;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# AskAI 接口限流 (保持 Lua 逻辑不变)
location = /api/askai {
access_by_lua_block {
local redis = require "resty.redis"
local r = redis:new()
r:set_timeout(200)
local ok, err = r:connect("127.0.0.1", 6379)
if not ok then
ngx.log(ngx.ERR, "Redis connect error: ", err)
return ngx.exit(500)
end
local user = ngx.var.arg_user or ngx.var.remote_addr
local today = os.date("%Y%m%d")
local key = "limit:user:" .. user .. ":" .. today
local count, err = r:incr(key)
if count == 1 then r:expire(key, 86400) end
if count > 200 then
ngx.status = 429
ngx.header["Content-Type"] = "text/plain; charset=utf-8"
ngx.say("Too Many Requests: daily limit reached")
return ngx.exit(429)
end
}
proxy_pass http://127.0.0.1:8090;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# ===============================
# Next.js 动态页面代理
# ===============================
location ^~ /_next/ {
proxy_pass http://127.0.0.1:3000;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
access_log off;
expires 7d;
add_header Cache-Control "public, max-age=31536000, immutable";
}
location = /favicon.ico {
proxy_pass http://127.0.0.1:3000;
}
location / {
proxy_pass http://127.0.0.1:3000;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Cookie $http_cookie;
proxy_pass_header Set-Cookie;
# WebSocket 支持 (Next.js HMR / API routes)
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
# 可选:微缓存 SSR 页面(减少 Node 负载)
# proxy_cache ssr_cache;
# proxy_cache_key $scheme$host$request_uri;
# proxy_cache_valid 200 10s;
# proxy_cache_valid 404 1s;
# add_header X-Cache $upstream_cache_status;
}
# ===============================
# 隐藏文件保护
# ===============================
location ~ /\. {
deny all;
}
# ===============================
# 性能优化gzip 压缩
# ===============================
gzip on;
gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml;
gzip_min_length 1024;
gzip_proxied any;
gzip_comp_level 6;
}

View File

@ -1,18 +0,0 @@
-- init.sql
CREATE EXTENSION IF NOT EXISTS vector;
CREATE TABLE IF NOT EXISTS documents (
id BIGSERIAL PRIMARY KEY,
repo TEXT NOT NULL,
path TEXT NOT NULL,
chunk_id INT NOT NULL,
content TEXT NOT NULL,
embedding VECTOR(1024),
metadata JSONB
);
CREATE INDEX IF NOT EXISTS documents_embedding_idx
ON documents USING hnsw (embedding vector_cosine_ops);
CREATE INDEX IF NOT EXISTS idx_documents_metadata
ON documents USING gin (metadata);

View File

@ -1,77 +0,0 @@
server {
listen 80;
server_name dev.svc.plus dev-homepage.svc.plus;
return 301 https://$host$request_uri;
}
server {
listen 443 ssl http2;
server_name dev.svc.plus dev-homepage.svc.plus;
ssl_certificate /etc/ssl/svc.plus.pem;
ssl_certificate_key /etc/ssl/svc.plus.rsa.key;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers HIGH:!aNULL:!MD5;
# ================================
# Next.js 静态资源 (_next/static/*)
# ================================
location /_next/static/ {
proxy_pass http://127.0.0.1:3001;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# 静态文件可缓存一年
access_log off;
expires 7d;
add_header Cache-Control "public, max-age=31536000, immutable";
}
# ================================
# Public 静态资源 (favicon, images)
# ================================
location /public/ {
alias /var/www/XControl/ui/homepage/public/;
access_log off;
expires 30d;
add_header Cache-Control "public, max-age=2592000";
}
# ================================
# API / SSR 页面
# ================================
location / {
proxy_pass http://127.0.0.1:3001;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Cookie $http_cookie;
proxy_pass_header Set-Cookie;
# WebSocket 支持 (Next.js HMR, API)
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
# 禁止访问隐藏文件
location ~ /\. {
deny all;
}
# ================================
# 性能优化:开启 gzip 压缩
# ================================
gzip on;
gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml;
gzip_min_length 1024;
gzip_proxied any;
gzip_comp_level 6;
}

File diff suppressed because it is too large Load Diff

View File

@ -1,49 +0,0 @@
global:
redis:
addr: "127.0.0.1:6379"
password: ""
vectordb:
pgurl: "postgres://shenlan:password@127.0.0.1:5432/shenlan"
datasources:
- name: Xstream
repo: https://github.com/svc-design/Xstream
path: docs
- name: documents
repo: https://github.com/svc-design/documents
path: /
- name: XControl
repo: https://github.com/svc-design/XControl
path: docs
sync:
repo:
proxy: socks5://127.0.0.1:1080 # 仅在同步仓库时使用代理
models:
embedder:
provider: "huggingface_hub"
models: "bge-m3"
endpoint: "http://127.0.0.1:9000/v1/embeddings"
generator:
provider: "ollama"
models:
- 'gemma3:4b'
endpoint: "http://127.0.0.1:11434/v1/chat/completions"
embedding:
max_batch: 64
dimension: 1024 #维度
max_chars: 8000
rate_limit_tpm: 120000
chunking:
embed_toc: true
max_tokens: 800
overlap_tokens: 80
prefer_heading_split: true
include_exts: [".md", ".mdx"]
ignore_dirs: [".git", "node_modules", "dist", "build"]
api:
askai:
timeout: 100
retries: 3

View File

@ -1,51 +0,0 @@
global:
redis:
addr: "127.0.0.1:6379"
password: ""
vectordb:
pgurl: "postgres://shenlan:password@127.0.0.1:5432/postgres"
datasources:
- name: Xstream
repo: https://github.com/svc-design/Xstream
path: docs
- name: XControl
repo: https://github.com/svc-design/XControl
path: docs
- name: documents
repo: https://github.com/svc-design/documents
path: /
sync:
repo:
proxy: socks5://127.0.0.1:1080 # 仅在同步仓库时使用代理
provider:
- name: ollama
endpoint: http://localhost:11434
models:
- 'gpt-oss:20b'
- name: chutes
endpoint: https://llm.chutes.ai/v1
token: "cpk_xxxxxxxxxxxxxxxxxx"
models:
- 'moonshotai/Kimi-K2-Instruct'
embedding:
endpoint: https://chutes-baai-bge-m3.chutes.ai/embed
token: "cpk_xxxxxxxxxxxxxxxxxx"
dimension: 0 # 0 = 首次响应自动探测维度
rate_limit_tpm: 120000
max_batch: 64
max_chars: 8000
chunking:
max_tokens: 800
overlap_tokens: 80
prefer_heading_split: true
include_exts: [".md", ".mdx"]
ignore_dirs: [".git", "node_modules", "dist", "build"]
embed_toc: true
api:
askai:
timeout: 100
retries: 3

5
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/gin-contrib/cors v1.6.0
github.com/gin-gonic/gin v1.9.1
github.com/go-git/go-git/v5 v5.16.2
github.com/golang-migrate/migrate/v4 v4.19.0
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.7.5
github.com/pgvector/pgvector-go v0.3.0
@ -45,6 +46,8 @@ require (
github.com/go-playground/validator/v10 v10.19.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
@ -72,6 +75,6 @@ require (
golang.org/x/sync v0.13.0 // indirect
golang.org/x/sys v0.32.0 // indirect
golang.org/x/text v0.24.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
)

11
go.sum
View File

@ -76,6 +76,8 @@ github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn
github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@ -83,6 +85,11 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
@ -226,8 +233,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

View File

View File

@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -euo pipefail
DB_URL="postgres://shenlan:password@127.0.0.1:5432/account?sslmode=disable"
OUT="/tmp/schema_clean.sql"
echo ">>> Exporting clean schema (PostgreSQL 16 compatible)"
pg_dump \
--schema-only \
--no-owner \
--no-privileges \
--exclude-schema=pglogical \
"$DB_URL" \
| grep -v -i "EXTENSION pglogical" \
| grep -v -i "COMMENT ON EXTENSION pglogical" \
| grep -v -i "SCHEMA pglogical" \
> "$OUT"
echo "✅ Schema exported to $OUT"
grep -i pglogical "$OUT" || echo "✅ pglogical completely removed"

View File

@ -0,0 +1,114 @@
#!/usr/bin/env bash
set -e
# ============================================================
# PostgreSQL 专用 TLS 证书生成脚本(含 *.svc.plus + 双 IP
# 作者SVC.PLUS PostgreSQL Server TLS Generator
# ============================================================
TLS_DIR="/etc/postgres-tls"
CA_DIR="$TLS_DIR/ca"
SERVER_DIR="$TLS_DIR/server"
echo ">>> [1/6] 创建目录结构 ..."
sudo mkdir -p "$CA_DIR" "$SERVER_DIR"
cd "$TLS_DIR"
# ============================================================
# 1. 创建私有 CA 根证书
# ============================================================
echo ">>> [2/6] 生成 PostgreSQL 专用私有 CA ..."
sudo openssl genrsa -aes256 -out "$CA_DIR/ca.key.pem" 4096
sudo chmod 600 "$CA_DIR/ca.key.pem"
sudo openssl req -x509 -new -nodes -key "$CA_DIR/ca.key.pem" -sha256 -days 3650 \
-subj "/C=CN/O=SVC.PLUS PostgreSQL Authority/OU=DB Security/CN=SVC.PLUS PostgreSQL Root CA" \
-out "$CA_DIR/ca.cert.pem"
# ============================================================
# 2. 生成服务器证书
# ============================================================
echo ">>> [3/6] 生成服务器私钥与 CSR ..."
sudo openssl genrsa -out "$SERVER_DIR/server.key.pem" 2048
sudo chmod 600 "$SERVER_DIR/server.key.pem"
sudo openssl req -new -key "$SERVER_DIR/server.key.pem" \
-subj "/C=CN/O=SVC.PLUS PostgreSQL Server/OU=DB/CN=global-homepage.svc.plus" \
-out "$SERVER_DIR/server.csr.pem"
# SAN 扩展配置
cat <<EOF | sudo tee "$SERVER_DIR/server.ext" >/dev/null
basicConstraints=CA:FALSE
keyUsage = critical, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = *.svc.plus
DNS.2 = svc.plus
DNS.3 = global-homepage.svc.plus
DNS.4 = cn-homepage.svc.plus
IP.1 = 167.179.72.223
IP.2 = 47.120.61.35
EOF
# 签发服务器证书2年有效
echo ">>> [4/6] 使用 SVC.PLUS PostgreSQL Root CA 签发服务器证书 ..."
sudo openssl x509 -req -in "$SERVER_DIR/server.csr.pem" \
-CA "$CA_DIR/ca.cert.pem" -CAkey "$CA_DIR/ca.key.pem" \
-CAcreateserial -out "$SERVER_DIR/server.cert.pem" \
-days 730 -sha256 -extfile "$SERVER_DIR/server.ext"
# fullchain
sudo cat "$SERVER_DIR/server.cert.pem" "$CA_DIR/ca.cert.pem" | sudo tee "$SERVER_DIR/server.fullchain.pem" >/dev/null
# ============================================================
# 3. 安装到 PostgreSQL 标准路径
# ============================================================
echo ">>> [5/6] 安装证书到 PostgreSQL SSL 目录 ..."
sudo install -o postgres -g postgres -m 600 "$SERVER_DIR/server.key.pem" /etc/ssl/private/svc.plus-postgres.key
sudo install -o postgres -g postgres -m 644 "$SERVER_DIR/server.fullchain.pem" /etc/ssl/certs/svc.plus-postgres.crt
sudo install -o postgres -g postgres -m 644 "$CA_DIR/ca.cert.pem" /etc/ssl/certs/svc.plus-postgres-ca.crt
# ============================================================
# 4. 输出后续操作提示
# ============================================================
echo "==============================================================="
echo "✅ [SVC.PLUS PostgreSQL TLS] 已生成并安装完成"
echo ""
echo "请在 /etc/postgresql/16/main/postgresql.conf 中添加或确认以下配置:"
echo ""
echo " ssl = on"
echo " ssl_cert_file = '/etc/ssl/certs/svc.plus-postgres.crt'"
echo " ssl_key_file = '/etc/ssl/private/svc.plus-postgres.key'"
echo " ssl_ca_file = '/etc/ssl/certs/svc.plus-postgres-ca.crt'"
echo ""
echo "⚙️ 然后执行: sudo systemctl restart postgresql"
echo ""
echo "📦 客户端(订阅端)请复制 CA 根证书:"
echo " /etc/postgres-tls/ca/ca.cert.pem"
echo "至客户端路径:"
echo " /var/lib/postgresql/.postgresql/root.crt"
echo "权限600属主 postgres"
echo ""
echo "🔍 验证命令示例:"
echo " openssl s_client -connect 167.179.72.223:5432 -starttls postgres -servername global-homepage.svc.plus"
echo ""
echo "👑 证书主题SVC.PLUS PostgreSQL Server"
echo "包含 SAN: *.svc.plus, global-homepage, cn-homepage, IP(167.179.72.223, 47.120.61.35)"
echo "==============================================================="
sudo chown postgres:postgres /etc/ssl/private/svc.plus-postgres.key
sudo chmod 600 /etc/ssl/private/svc.plus-postgres.key
sudo chown root:postgres /etc/ssl/private
sudo chmod 750 /etc/ssl/private
# 1⃣ 创建目录
sudo -u postgres mkdir -p /var/lib/postgresql/.postgresql
# 2⃣ 从 global-homepage 拉取服务器的 CA 根证书
sudo scp root@167.179.72.223:/etc/ssl/certs/svc.plus-postgres-ca.crt /var/lib/postgresql/.postgresql/root.crt
# 3⃣ 设置权限
sudo chown postgres:postgres /var/lib/postgresql/.postgresql/root.crt
sudo chmod 600 /var/lib/postgresql/.postgresql/root.crt

View File

@ -1,40 +0,0 @@
#!/usr/bin/env bash
# =============================================
# 🧹 git-branch-keeper.sh
# 保留 main 与所有 release/* 分支
# 清理多余本地和远程分支
# =============================================
set -e
echo ">>> Fetching and pruning remote branches..."
git fetch --all --prune
echo ">>> Cleaning local branches..."
for branch in $(git branch | sed 's/*//'); do
case "$branch" in
main|HEAD|release/*)
echo "✅ 保留本地分支:$branch"
;;
*)
echo "🗑️ 删除本地分支:$branch"
git branch -D "$branch"
;;
esac
done
echo ">>> Cleaning remote branches..."
for branch in $(git branch -r | sed 's/origin\///'); do
case "$branch" in
HEAD|main|release/*)
echo "✅ 保留远程分支origin/$branch"
;;
*)
echo "🗑️ 删除远程分支origin/$branch"
git push origin --delete "$branch" || true
;;
esac
done
echo "✅ 分支清理完成!"

View File

@ -0,0 +1,39 @@
#!/usr/bin/env bash
#
# scripts/import_schema_clean.sh
# ---------------------------------------------
# Import a cleaned schema.sql file into PostgreSQL.
# Safe for re-run (幂等导入)
# ---------------------------------------------
set -euo pipefail
# ====== Configuration ======
DB_URL=${1:-"postgres://shenlan:password@127.0.0.1:5432/account?sslmode=disable"}
IN_FILE=${2:-"/tmp/schema_clean.sql"}
# ====== Validation ======
if [ ! -f "$IN_FILE" ]; then
echo "❌ File not found: $IN_FILE"
echo "💡 请先运行 export_schema_clean.sh 导出 schema"
exit 1
fi
if ! command -v psql >/dev/null 2>&1; then
echo "❌ 未检测到 psql请先安装 PostgreSQL 客户端"
exit 1
fi
# ====== Import schema ======
echo ">>> Importing schema into database"
echo "---------------------------------------------"
echo "Database: $DB_URL"
echo "Schema: $IN_FILE"
echo "---------------------------------------------"
psql "$DB_URL" -v ON_ERROR_STOP=1 -f "$IN_FILE"
echo ""
echo "✅ Schema import completed successfully"
echo "---------------------------------------------"

View File

@ -98,16 +98,9 @@ install-postgresql() {
echo "=== 安装 PostgreSQL ${PG_MAJOR} ==="
sudo apt-get update
sudo apt-get install -y wget curl gnupg lsb-release ca-certificates
if ! grep -q "apt.postgresql.org" /etc/apt/sources.list.d/pgdg.list 2>/dev/null; then
curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc -o /tmp/pgdg.asc || \
curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xB97B0E2D95A5761FB72B0C18ACCC4CF8" -o /tmp/pgdg.asc
sudo gpg --dearmor -o /usr/share/keyrings/postgresql.gpg /tmp/pgdg.asc
echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
| sudo tee /etc/apt/sources.list.d/pgdg.list
sudo apt-get update
fi
sudo apt-get install -y "postgresql-${PG_MAJOR}" "postgresql-client-${PG_MAJOR}" \
"postgresql-contrib-${PG_MAJOR}" "postgresql-server-dev-${PG_MAJOR}"
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 2F59B5F99B1BE0B4
echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list
sudo apt-get install -y "postgresql-${PG_MAJOR}" "postgresql-client-${PG_MAJOR}" "postgresql-contrib-${PG_MAJOR}" "postgresql-server-dev-${PG_MAJOR}"
sudo systemctl enable --now postgresql
}
@ -123,8 +116,7 @@ install-pgvector() {
sudo apt-get install -y git make gcc
tmp_dir=$(mktemp -d)
cd "$tmp_dir"
git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git || \
git clone https://ghproxy.com/https://github.com/pgvector/pgvector.git
git clone git@github.com:svc-design/pgvector.git
cd pgvector
make && sudo make install
cd /
@ -138,8 +130,7 @@ install-zhparser() {
# 编译安装 scws v1.2.3
tmp_dir=$(mktemp -d)
cd "$tmp_dir"
git clone https://github.com/hightman/scws.git || \
git clone https://ghproxy.com/https://github.com/hightman/scws.git
git clone git@github.com:svc-design/scws.git
cd scws
# 修掉 automake 不兼容的注释
@ -162,8 +153,7 @@ install-zhparser() {
# 编译安装 zhparser
tmp_dir=$(mktemp -d)
cd "$tmp_dir"
git clone https://github.com/amutu/zhparser.git || \
git clone https://ghproxy.com/https://github.com/amutu/zhparser.git
git clone git@github.com:svc-design/zhparser.git
cd zhparser
pg_config_path="$(command -v pg_config || echo "/usr/lib/postgresql/${PG_MAJOR}/bin/pg_config")"
make SCWS_HOME=/usr PG_CONFIG="${pg_config_path}"