From e7e53c5c7ebbb4fc07259d90ae7d58b8ce293676 Mon Sep 17 00:00:00 2001 From: Haitao Pan Date: Sun, 1 Feb 2026 20:51:51 +0800 Subject: [PATCH] Remove sensitive files --- .gitignore | 49 + CLAUDE.md | 496 +++ KEYS | 52 + LICENSE | 5 +- Makefile | 696 +++ NOTICE | 8 + README.md | 686 ++- ansible.cfg | 13 + app.yml | 136 + app/README.md | 153 + app/bytebase/.env | 3 + app/bytebase/Makefile | 53 + app/bytebase/README.md | 66 + app/bytebase/docker-compose.yml | 13 + app/dify/Makefile | 39 + app/dify/docker-compose.yml | 851 ++++ app/dify/nginx/conf.d/default.conf.template | 48 + app/dify/nginx/docker-entrypoint.sh | 42 + app/dify/nginx/https.conf.template | 9 + app/dify/nginx/nginx.conf.template | 34 + app/dify/nginx/proxy.conf.template | 11 + app/dify/ssrf_proxy/docker-entrypoint.sh | 42 + app/dify/ssrf_proxy/squid.conf.template | 51 + app/dify/volumes/sandbox/conf/config.yaml | 31 + .../volumes/sandbox/conf/config.yaml.example | 35 + .../dependencies/python-requirements.txt | 1 + app/electric/Makefile | 41 + app/electric/README.md | 6 + app/electric/docker-compose.yml | 14 + app/ferretdb/.env | 2 + app/ferretdb/Makefile | 34 + app/ferretdb/README.md | 93 + app/ferretdb/docker-compose.yml | 13 + app/gitea/.env | 15 + app/gitea/Makefile | 43 + app/gitea/README.md | 78 + app/gitea/docker-compose.yml | 32 + app/jupyter/.env | 21 + app/jupyter/Makefile | 46 + app/jupyter/README.md | 81 + app/jupyter/docker-compose.yml | 15 + app/kong/.env | 9 + app/kong/Makefile | 36 + app/kong/README.md | 58 + app/kong/docker-compose.yml | 42 + app/mattermost/.env | 48 + app/mattermost/Makefile | 44 + app/mattermost/docker-compose.yml | 42 + app/metabase/.env | 9 + app/metabase/Makefile | 45 + app/metabase/README.md | 62 + app/metabase/docker-compose.yml | 14 + app/minio/Makefile | 40 + app/minio/README.md | 32 + app/minio/docker-compose.yml | 14 + app/nocodb/.env | 5 + app/nocodb/Makefile | 42 + app/nocodb/README.md | 68 + app/nocodb/docker-compose.yml | 12 + app/odoo/.env | 11 + app/odoo/Makefile | 47 + app/odoo/README.md | 111 + app/odoo/docker-compose.yml | 47 + app/pg_exporter/.env | 16 + app/pg_exporter/Makefile | 37 + app/pg_exporter/README.md | 43 + app/pg_exporter/docker-compose.yml | 37 + app/pgadmin/.env | 9 + app/pgadmin/Makefile | 60 + app/pgadmin/README.md | 86 + app/pgadmin/docker-compose.yml | 13 + app/pgweb/Makefile | 38 + app/pgweb/README.md | 33 + app/pgweb/docker-compose.yml | 8 + app/postgrest/.env | 6 + app/postgrest/Makefile | 43 + app/postgrest/README.md | 46 + app/postgrest/docker-compose.yml | 15 + app/registry/Makefile | 91 + app/registry/README.md | 254 ++ app/registry/config.yml | 49 + app/registry/docker-compose.yml | 42 + app/supabase/Makefile | 50 + app/supabase/README.md | 48 + app/supabase/bin/dump | 33 + app/supabase/bin/load | 34 + app/supabase/docker-compose.yml | 392 ++ app/supabase/volumes/api/kong.yml | 283 ++ app/supabase/volumes/functions/hello/index.ts | 16 + app/supabase/volumes/functions/main/index.ts | 94 + app/supabase/volumes/logs/vector.yml | 253 ++ app/teable/.env | 28 + app/teable/Makefile | 37 + app/teable/README.md | 18 + app/teable/docker-compose.yml | 15 + app/wiki/.env | 8 + app/wiki/Makefile | 34 + app/wiki/README.md | 68 + app/wiki/docker-compose.yml | 15 + bin/boot-pkg | 88 + bin/etcd-add | 163 + bin/etcd-rm | 170 + bin/get-pkg | 190 + bin/install | 160 + bin/inventory_cmdb | 45 + bin/inventory_conf | 31 + bin/inventory_load | 255 ++ bin/node-add | 71 + bin/node-rm | 72 + bin/parse_nginx.py | 119 + bin/pglog-summary | 29 + bin/pgmon-add | 75 + bin/pgmon-rm | 80 + bin/pgsql-add | 163 + bin/pgsql-db | 80 + bin/pgsql-ext | 110 + bin/pgsql-hba | 122 + bin/pgsql-rm | 147 + bin/pgsql-svc | 119 + bin/pgsql-user | 76 + bin/profile | 47 + bin/psql.py | 221 + bin/redis-add | 138 + bin/redis-rm | 137 + bin/release | 84 + bin/repo-add | 77 + bin/schemaspy | 33 + bin/stackcollapse-perf.pl | 435 ++ bin/svg | 42 + bin/validate | 544 +++ bootstrap | 794 ++++ cache.yml | 137 + cert.yml | 124 + conf/README.md | 109 + conf/app/dify.yml | 128 + conf/app/maybe.yml | 115 + conf/app/odoo.yml | 109 + conf/app/registry.yml | 131 + conf/app/teable.yml | 116 + conf/demo/bare.yml | 11 + conf/demo/debian.yml | 839 ++++ conf/demo/demo.yml | 154 + conf/demo/el.yml | 846 ++++ conf/demo/kernel.yml | 129 + conf/demo/kernels.yml | 544 +++ conf/demo/minio.yml | 98 + conf/demo/redis.yml | 78 + conf/demo/remote.yml | 87 + conf/demo/saas.yml | 175 + conf/demo/wool.yml | 65 + conf/docker.yml | 108 + conf/fat.yml | 309 ++ conf/ha/citus.yml | 128 + conf/ha/dual.yml | 88 + conf/ha/full.yml | 177 + conf/ha/safe.yml | 193 + conf/ha/simu.yml | 370 ++ conf/ha/trio.yml | 140 + conf/infra.yml | 48 + conf/ivory.yml | 86 + conf/meta.yml | 220 + conf/mongo.yml | 101 + conf/mssql.yml | 96 + conf/mysql.yml | 87 + conf/oriole.yml | 85 + conf/pgsql.yml | 85 + conf/pgtde.yml | 88 + conf/polar.yml | 96 + conf/rich.yml | 346 ++ conf/slim.yml | 75 + conf/vibe.yml | 121 + configure | 809 ++++ deploy.yml | 141 + docker.yml | 59 + docker/.env | 5 + docker/Dockerfile | 117 + docker/Makefile | 281 ++ docker/README.md | 137 + docker/docker-compose.yml | 38 + etcd-rm.yml | 57 + etcd.yml | 60 + files/cmdb.sql | 1661 ++++++++ files/grafana/README.md | 199 + files/grafana/app/pglog-overview.json | 463 ++ files/grafana/app/pglog-session.json | 239 ++ files/grafana/grafana.py | 405 ++ .../grafana/infra/alertmanager-instance.json | 1573 +++++++ files/grafana/infra/etcd-overview.json | 1279 ++++++ files/grafana/infra/grafana-instance.json | 565 +++ files/grafana/infra/infra-overview.json | 1045 +++++ files/grafana/infra/inventory-cmdb.json | 924 ++++ files/grafana/infra/logs-instance.json | 143 + files/grafana/infra/nginx-instance.json | 573 +++ files/grafana/infra/vlogs-instance.json | 2610 ++++++++++++ files/grafana/infra/vmalert-instance.json | 1682 ++++++++ files/grafana/infra/vmetrics-instance.json | 2706 ++++++++++++ files/grafana/infra/vtraces-instance.json | 2736 ++++++++++++ files/grafana/minio/minio-instance.json | 739 ++++ files/grafana/minio/minio-overview.json | 1293 ++++++ files/grafana/mongo/mongo-overview.json | 332 ++ files/grafana/node/claude-code.json | 1916 +++++++++ files/grafana/node/node-alert.json | 785 ++++ files/grafana/node/node-cluster.json | 3444 +++++++++++++++ files/grafana/node/node-disk.json | 932 ++++ files/grafana/node/node-haproxy.json | 2652 ++++++++++++ files/grafana/node/node-instance.json | 3730 +++++++++++++++++ files/grafana/node/node-juice.json | 1236 ++++++ files/grafana/node/node-overview.json | 959 +++++ files/grafana/node/node-vector.json | 758 ++++ files/grafana/node/node-vip.json | 1260 ++++++ files/grafana/pgsql/pgcat-database.json | 3708 ++++++++++++++++ files/grafana/pgsql/pgcat-instance.json | 2667 ++++++++++++ files/grafana/pgsql/pgcat-locks.json | 1360 ++++++ files/grafana/pgsql/pgcat-query.json | 1408 +++++++ files/grafana/pgsql/pgcat-schema.json | 1348 ++++++ files/grafana/pgsql/pgcat-table.json | 1403 +++++++ files/grafana/pgsql/pgrds-cluster.json | 3300 +++++++++++++++ files/grafana/pgsql/pgrds-instance.json | 2987 +++++++++++++ files/grafana/pgsql/pgsql-activity.json | 1446 +++++++ files/grafana/pgsql/pgsql-alert.json | 1013 +++++ files/grafana/pgsql/pgsql-cluster.json | 2544 +++++++++++ files/grafana/pgsql/pgsql-database.json | 1838 ++++++++ files/grafana/pgsql/pgsql-databases.json | 1575 +++++++ files/grafana/pgsql/pgsql-exporter.json | 1096 +++++ files/grafana/pgsql/pgsql-instance.json | 1655 ++++++++ files/grafana/pgsql/pgsql-overview.json | 1176 ++++++ files/grafana/pgsql/pgsql-patroni.json | 764 ++++ files/grafana/pgsql/pgsql-persist.json | 1860 ++++++++ files/grafana/pgsql/pgsql-pgbouncer.json | 1569 +++++++ files/grafana/pgsql/pgsql-pitr.json | 1068 +++++ files/grafana/pgsql/pgsql-proxy.json | 469 +++ files/grafana/pgsql/pgsql-query.json | 1595 +++++++ files/grafana/pgsql/pgsql-replication.json | 1106 +++++ files/grafana/pgsql/pgsql-service.json | 723 ++++ files/grafana/pgsql/pgsql-session.json | 2067 +++++++++ files/grafana/pgsql/pgsql-shard.json | 1097 +++++ files/grafana/pgsql/pgsql-table.json | 1484 +++++++ files/grafana/pgsql/pgsql-tables.json | 1741 ++++++++ files/grafana/pgsql/pgsql-xacts.json | 1276 ++++++ files/grafana/pigsty.json | 302 ++ files/grafana/redis/redis-cluster.json | 1979 +++++++++ files/grafana/redis/redis-instance.json | 1788 ++++++++ files/grafana/redis/redis-overview.json | 1770 ++++++++ files/migration/pg-meta.yml | 38 + files/mssql.sql | 8 + files/postgres/pg-backup | 84 + files/postgres/pg-badger | 67 + files/postgres/pg-basebackup | 410 ++ files/postgres/pg-cluster | 34 + files/postgres/pg-drop-role | 753 ++++ files/postgres/pg-failover-callback | 151 + files/postgres/pg-fork | 424 ++ files/postgres/pg-heartbeat | 18 + files/postgres/pg-pass-add | 71 + files/postgres/pg-pitr | 378 ++ files/postgres/pg-repack | 315 ++ files/postgres/pg-role | 173 + files/postgres/pg-tune-hugepage | 182 + files/postgres/pg-vacuum | 278 ++ files/postgres/pgb-user | 102 + files/postgres/supa-kick | 16 + files/supabase.sql | 2218 ++++++++++ files/victoria/bin/check | 12 + files/victoria/bin/new | 12 + files/victoria/bin/reload | 12 + files/victoria/bin/status | 15 + files/victoria/rules/etcd.yml | 125 + files/victoria/rules/infra.yml | 92 + files/victoria/rules/kafka.yml | 73 + files/victoria/rules/minio.yml | 65 + files/victoria/rules/mysql.yml | 42 + files/victoria/rules/node.yml | 964 +++++ files/victoria/rules/pgsql.yml | 1599 +++++++ files/victoria/rules/redis.yml | 422 ++ infra-rm.yml | 134 + infra.yml | 104 + juice.yml | 67 + minio-rm.yml | 49 + minio.yml | 50 + mongo.yml | 60 + node-rm.yml | 56 + node.yml | 115 + pgsql-db.yml | 182 + pgsql-migration.yml | 50 + pgsql-monitor.yml | 91 + pgsql-pitr.yml | 86 + pgsql-rm.yml | 78 + pgsql-user.yml | 170 + pgsql.yml | 133 + pigsty.yml | 224 + redis-rm.yml | 65 + redis.yml | 180 + roles/ca/README.md | 280 ++ roles/ca/defaults/main.yml | 8 + roles/ca/meta/main.yml | 18 + roles/ca/tasks/main.yml | 101 + roles/cache/README.md | 259 ++ roles/cache/defaults/main.yml | 28 + roles/cache/meta/main.yml | 18 + roles/cache/tasks/main.yml | 124 + roles/docker/README.md | 180 + roles/docker/defaults/main.yml | 18 + roles/docker/meta/main.yml | 18 + roles/docker/tasks/main.yml | 103 + roles/docker/templates/daemon.json.j2 | 35 + roles/etcd/README.md | 152 + roles/etcd/defaults/main.yml | 26 + roles/etcd/meta/main.yml | 18 + roles/etcd/tasks/config.yml | 79 + roles/etcd/tasks/main.yml | 134 + roles/etcd/templates/etcd.conf | 47 + roles/etcd/templates/etcd.pass | 1 + roles/etcd/templates/etcd.svc | 22 + roles/etcd/templates/etcdctl.sh | 15 + roles/etcd_remove/README.md | 119 + roles/etcd_remove/defaults/main.yml | 16 + roles/etcd_remove/meta/main.yml | 18 + roles/etcd_remove/tasks/leave.yml | 51 + roles/etcd_remove/tasks/main.yml | 94 + roles/ferret/README.md | 153 + roles/ferret/defaults/main.yml | 19 + roles/ferret/meta/main.yml | 20 + roles/ferret/tasks/cert.yml | 55 + roles/ferret/tasks/main.yml | 130 + roles/ferret/templates/ferretdb.env | 13 + roles/ferret/templates/ferretdb.svc | 37 + roles/haproxy/README.md | 230 + roles/haproxy/defaults/main.yml | 20 + roles/haproxy/files/haproxy.svc | 23 + roles/haproxy/meta/main.yml | 18 + roles/haproxy/tasks/main.yml | 112 + roles/haproxy/templates/haproxy.cfg.j2 | 59 + roles/haproxy/templates/service.j2 | 20 + roles/infra/README.md | 230 + roles/infra/defaults/main.yml | 184 + roles/infra/meta/main.yml | 24 + roles/infra/tasks/alertmanager.yml | 27 + roles/infra/tasks/blackbox.yml | 33 + roles/infra/tasks/cert.yml | 54 + roles/infra/tasks/dir.yml | 78 + roles/infra/tasks/dns.yml | 74 + roles/infra/tasks/env.yml | 148 + roles/infra/tasks/grafana.yml | 118 + roles/infra/tasks/main.yml | 92 + roles/infra/tasks/nginx.yml | 282 ++ roles/infra/tasks/pkg.yml | 11 + roles/infra/tasks/register.yml | 141 + roles/infra/tasks/user.yml | 38 + roles/infra/tasks/victoria.yml | 146 + roles/infra/templates/dns/dnsmasq.conf | 697 +++ roles/infra/templates/dns/dnsmasq.svc | 14 + roles/infra/templates/env/patronictl.yml.j2 | 63 + roles/infra/templates/env/pg_service.conf.j2 | 58 + roles/infra/templates/env/pigsty.j2 | 17 + roles/infra/templates/env/servers.j2 | 20 + .../infra/templates/grafana/dashboard.yml.j2 | 23 + .../infra/templates/grafana/datasource.yml.j2 | 84 + roles/infra/templates/grafana/grafana.svc | 54 + roles/infra/templates/nginx/404.html.j2 | 292 ++ roles/infra/templates/nginx/endpoint.conf | 86 + roles/infra/templates/nginx/home.conf | 187 + roles/infra/templates/nginx/htpasswd.j2 | 4 + roles/infra/templates/nginx/index.html.j2 | 1607 +++++++ roles/infra/templates/nginx/index.zh.html.j2 | 1617 +++++++ roles/infra/templates/nginx/link-cert.j2 | 43 + roles/infra/templates/nginx/nginx.conf.j2 | 28 + .../infra/templates/nginx/nginx_exporter.env | 3 + .../infra/templates/nginx/nginx_exporter.svc | 15 + roles/infra/templates/nginx/path.conf | 79 + roles/infra/templates/nginx/sign-cert.j2 | 70 + .../templates/prometheus/alertmanager.env | 3 + .../templates/prometheus/alertmanager.svc | 17 + .../templates/prometheus/alertmanager.yml | 205 + roles/infra/templates/prometheus/blackbox.env | 3 + roles/infra/templates/prometheus/blackbox.yml | 28 + .../prometheus/blackbox_exporter.svc | 17 + roles/infra/templates/prometheus/infra.yml.j2 | 66 + roles/infra/templates/vector/grafana.yaml | 45 + roles/infra/templates/vector/nginx.yaml | 93 + roles/infra/templates/victoria/agent.yml | 65 + roles/infra/templates/victoria/prometheus.yml | 221 + roles/infra/templates/victoria/vlogs.env | 3 + roles/infra/templates/victoria/vlogs.svc | 21 + roles/infra/templates/victoria/vmalert.env | 3 + roles/infra/templates/victoria/vmalert.svc | 22 + roles/infra/templates/victoria/vmetrics.env | 3 + roles/infra/templates/victoria/vmetrics.svc | 21 + roles/infra/templates/victoria/vtraces.env | 3 + roles/infra/templates/victoria/vtraces.svc | 21 + roles/juice/README.md | 88 + roles/juice/defaults/main.yml | 23 + roles/juice/meta/main.yml | 21 + roles/juice/tasks/clean.yml | 29 + roles/juice/tasks/instance.yml | 65 + roles/juice/tasks/main.yml | 108 + roles/juice/templates/juice.env | 2 + roles/juice/templates/juice.svc | 16 + roles/minio/README.md | 190 + roles/minio/defaults/main.yml | 42 + roles/minio/handlers/main.yml | 6 + roles/minio/meta/main.yml | 18 + roles/minio/tasks/config.yml | 121 + roles/minio/tasks/install.yml | 50 + roles/minio/tasks/main.yml | 131 + roles/minio/tasks/provision.yml | 98 + roles/minio/templates/minio.env | 10 + roles/minio/templates/minio.svc | 33 + roles/minio/templates/policy.json | 4 + roles/minio_remove/README.md | 122 + roles/minio_remove/defaults/main.yml | 15 + roles/minio_remove/meta/main.yml | 18 + roles/minio_remove/tasks/main.yml | 137 + roles/node/README.md | 263 ++ roles/node/defaults/main.yml | 122 + roles/node/files/disable_numa.sh | 7 + roles/node/files/disable_thp.sh | 14 + roles/node/files/disk_prefetch.sh | 11 + roles/node/files/limits.conf | 81 + roles/node/files/node.sh | 193 + roles/node/files/policy-rc.d | 13 + roles/node/files/ssh.config | 8 + roles/node/files/static_network.sh | 10 + roles/node/meta/main.yml | 18 + roles/node/tasks/admin.yml | 135 + roles/node/tasks/cert.yml | 45 + roles/node/tasks/dns.yml | 69 + roles/node/tasks/main.yml | 60 + roles/node/tasks/pkg.yml | 149 + roles/node/tasks/sec.yml | 46 + roles/node/tasks/time.yml | 79 + roles/node/tasks/tune.yml | 111 + roles/node/tasks/vip.yml | 49 + roles/node/templates/chrony.conf.j2 | 45 + roles/node/templates/keepalived.conf.j2 | 50 + roles/node/templates/tuned-crit.conf | 180 + roles/node/templates/tuned-olap.conf | 182 + roles/node/templates/tuned-oltp.conf | 181 + roles/node/templates/tuned-tiny.conf | 156 + roles/node_id/README.md | 170 + roles/node_id/defaults/main.yml | 10 + roles/node_id/meta/main.yml | 18 + roles/node_id/tasks/main.yml | 105 + roles/node_id/vars/d12.aarch64.yml | 745 ++++ roles/node_id/vars/d12.x86_64.yml | 745 ++++ roles/node_id/vars/d13.aarch64.yml | 745 ++++ roles/node_id/vars/d13.x86_64.yml | 745 ++++ roles/node_id/vars/el10.aarch64.yml | 749 ++++ roles/node_id/vars/el10.x86_64.yml | 750 ++++ roles/node_id/vars/el7.x86_64.yml | 621 +++ roles/node_id/vars/el8.aarch64.yml | 749 ++++ roles/node_id/vars/el8.x86_64.yml | 750 ++++ roles/node_id/vars/el9.aarch64.yml | 749 ++++ roles/node_id/vars/el9.x86_64.yml | 750 ++++ roles/node_id/vars/u22.aarch64.yml | 745 ++++ roles/node_id/vars/u22.x86_64.yml | 745 ++++ roles/node_id/vars/u24.aarch64.yml | 745 ++++ roles/node_id/vars/u24.x86_64.yml | 745 ++++ roles/node_monitor/README.md | 196 + roles/node_monitor/defaults/main.yml | 48 + .../files/keepalived_exporter.svc | 17 + roles/node_monitor/files/node_exporter.svc | 17 + roles/node_monitor/meta/main.yml | 18 + roles/node_monitor/tasks/main.yml | 207 + roles/node_monitor/tasks/vector.yml | 67 + roles/node_monitor/templates/node.yaml | 30 + roles/node_monitor/templates/vector.env | 1 + roles/node_monitor/templates/vector.svc | 17 + roles/node_monitor/templates/vector.yaml | 62 + roles/node_remove/README.md | 104 + roles/node_remove/defaults/main.yml | 21 + roles/node_remove/meta/main.yml | 19 + roles/node_remove/tasks/main.yml | 137 + roles/node_remove/tasks/nginx.yml | 29 + roles/pg_exporters/README.md | 183 + roles/pg_exporters/defaults/main.yml | 50 + roles/pg_exporters/meta/main.yml | 18 + roles/pg_exporters/tasks/main.yml | 76 + roles/pg_exporters/tasks/pg_exporter.yml | 46 + roles/pg_exporters/tasks/register_grafana.yml | 12 + .../tasks/register_grafana_db.yml | 73 + roles/pg_exporters/templates/pg_exporter.env | 22 + roles/pg_exporters/templates/pg_exporter.svc | 18 + roles/pg_exporters/templates/pg_exporter.yml | 2335 +++++++++++ roles/pg_id/README.md | 177 + roles/pg_id/defaults/main.yml | 21 + roles/pg_id/meta/main.yml | 19 + roles/pg_id/tasks/main.yml | 51 + roles/pg_migration/README.md | 220 + roles/pg_migration/defaults/main.yml | 43 + roles/pg_migration/meta/main.yml | 19 + roles/pg_migration/tasks/main.yml | 71 + roles/pg_migration/templates/activate | 57 + roles/pg_migration/templates/check-db | 89 + roles/pg_migration/templates/check-hba | 60 + roles/pg_migration/templates/check-misc | 59 + roles/pg_migration/templates/check-repl | 98 + roles/pg_migration/templates/check-user | 109 + roles/pg_migration/templates/copy-diff | 80 + roles/pg_migration/templates/copy-progress | 68 + roles/pg_migration/templates/copy-schema | 83 + roles/pg_migration/templates/copy-seq | 134 + roles/pg_migration/templates/create-pub | 79 + roles/pg_migration/templates/create-sub | 73 + roles/pg_migration/templates/drop-pub | 72 + roles/pg_migration/templates/drop-sub | 73 + roles/pg_migration/templates/manual.md | 51 + roles/pg_monitor/README.md | 209 + roles/pg_monitor/defaults/main.yml | 82 + roles/pg_monitor/handlers/main.yml | 19 + roles/pg_monitor/meta/main.yml | 22 + roles/pg_monitor/tasks/main.yml | 50 + roles/pg_monitor/tasks/pg_exporter.yml | 32 + .../pg_monitor/tasks/pgbackrest_exporter.yml | 26 + roles/pg_monitor/tasks/pgbouncer_exporter.yml | 31 + roles/pg_monitor/tasks/register_grafana.yml | 77 + roles/pg_monitor/tasks/register_vector.yml | 15 + roles/pg_monitor/tasks/register_victoria.yml | 78 + roles/pg_monitor/templates/pg_exporter.env | 24 + roles/pg_monitor/templates/pg_exporter.svc | 16 + roles/pg_monitor/templates/pg_exporter.yml | 2335 +++++++++++ roles/pg_monitor/templates/pgbackrest.yaml | 43 + .../templates/pgbackrest_exporter.env | 16 + .../templates/pgbackrest_exporter.svc | 14 + .../templates/pgbouncer_exporter.env | 15 + .../templates/pgbouncer_exporter.svc | 16 + .../templates/pgbouncer_exporter.yml | 321 ++ roles/pg_monitor/templates/postgres.yaml | 64 + roles/pg_pitr/README.md | 243 ++ roles/pg_pitr/defaults/main.yml | 78 + roles/pg_pitr/meta/main.yml | 21 + roles/pg_pitr/tasks/etcd.yml | 32 + roles/pg_pitr/tasks/main.yml | 41 + roles/pg_pitr/tasks/pause.yml | 31 + roles/pg_pitr/tasks/pitr.yml | 120 + roles/pg_pitr/tasks/print.yml | 17 + roles/pg_pitr/tasks/start.yml | 109 + roles/pg_pitr/tasks/stop.yml | 28 + roles/pg_pitr/templates/command | 82 + roles/pg_pitr/templates/pg-restore | 218 + roles/pg_pitr/templates/pitr.conf | 77 + roles/pg_remove/README.md | 178 + roles/pg_remove/defaults/main.yml | 67 + roles/pg_remove/meta/main.yml | 19 + roles/pg_remove/tasks/dns.yml | 19 + roles/pg_remove/tasks/grafana.yml | 23 + roles/pg_remove/tasks/main.yml | 129 + roles/pg_remove/tasks/pg_exporter.yml | 19 + roles/pg_remove/tasks/pg_service.yml | 59 + roles/pg_remove/tasks/pgbackrest.yml | 42 + roles/pg_remove/tasks/pgbackrest_exporter.yml | 16 + roles/pg_remove/tasks/pgbouncer.yml | 16 + roles/pg_remove/tasks/pgbouncer_exporter.yml | 19 + roles/pg_remove/tasks/postgres.yml | 93 + roles/pg_remove/tasks/uninstall.yml | 46 + roles/pg_remove/tasks/victoria.yml | 35 + roles/pg_remove/tasks/vip.yml | 17 + roles/pgsql/README.md | 276 ++ roles/pgsql/defaults/main.yml | 263 ++ roles/pgsql/handlers/main.yml | 42 + roles/pgsql/meta/main.yml | 22 + roles/pgsql/tasks/cert.yml | 82 + roles/pgsql/tasks/config.yml | 108 + roles/pgsql/tasks/crontab.yml | 21 + roles/pgsql/tasks/database.yml | 108 + roles/pgsql/tasks/dbsu.yml | 85 + roles/pgsql/tasks/dns.yml | 68 + roles/pgsql/tasks/install.yml | 205 + roles/pgsql/tasks/main.yml | 124 + roles/pgsql/tasks/patroni.yml | 290 ++ roles/pgsql/tasks/pgbackrest.yml | 52 + roles/pgsql/tasks/pgbouncer.yml | 82 + roles/pgsql/tasks/service_local.yml | 42 + roles/pgsql/tasks/service_remote.yml | 55 + roles/pgsql/tasks/user.yml | 68 + roles/pgsql/tasks/vip.yml | 23 + roles/pgsql/templates/crit.yml | 580 +++ roles/pgsql/templates/olap.yml | 575 +++ roles/pgsql/templates/oltp.yml | 581 +++ roles/pgsql/templates/patroni.svc | 68 + roles/pgsql/templates/pg-alias | 131 + roles/pgsql/templates/pg-db.sql | 226 + roles/pgsql/templates/pg-init | 40 + roles/pgsql/templates/pg-init-roles.sql | 180 + roles/pgsql/templates/pg-init-template.sql | 693 +++ roles/pgsql/templates/pg-user.sql | 168 + roles/pgsql/templates/pg_hba.conf | 92 + roles/pgsql/templates/pgbackrest.conf | 98 + roles/pgsql/templates/pgbouncer.database | 13 + roles/pgsql/templates/pgbouncer.hba | 96 + roles/pgsql/templates/pgbouncer.ini | 60 + roles/pgsql/templates/pgbouncer.svc | 32 + roles/pgsql/templates/pgbouncer.useropts | 7 + roles/pgsql/templates/pgsodium_getkey | 15 + roles/pgsql/templates/postgres.svc | 47 + roles/pgsql/templates/psqlrc | 56 + roles/pgsql/templates/service.cfg | 42 + roles/pgsql/templates/tiny.yml | 570 +++ roles/pgsql/templates/vip-manager.svc | 15 + roles/pgsql/templates/vip-manager.yml | 76 + roles/pgsql/vars/main.yml | 4 + roles/redis/README.md | 181 + roles/redis/defaults/main.yml | 33 + roles/redis/meta/main.yml | 18 + roles/redis/tasks/exporter.yml | 54 + roles/redis/tasks/instance.yml | 42 + roles/redis/tasks/main.yml | 94 + roles/redis/tasks/node.yml | 62 + roles/redis/templates/redis-sentinel.conf | 353 ++ roles/redis/templates/redis.conf | 2097 +++++++++ roles/redis/templates/redis.svc | 24 + roles/redis/templates/redis.yaml | 29 + roles/redis_remove/README.md | 123 + roles/redis_remove/defaults/main.yml | 18 + roles/redis_remove/meta/main.yml | 19 + roles/redis_remove/tasks/main.yml | 138 + roles/repo/README.md | 212 + roles/repo/defaults/main.yml | 41 + roles/repo/meta/main.yml | 19 + roles/repo/tasks/build.yml | 287 ++ roles/repo/tasks/main.yml | 114 + roles/repo/tasks/nginx.yml | 61 + roles/repo/templates/default.conf.j2 | 20 + roles/repo/templates/index.html.j2 | 411 ++ roles/repo/templates/nginx.conf.j2 | 28 + roles/vibe/README.md | 169 + roles/vibe/defaults/main.yml | 46 + roles/vibe/meta/main.yml | 22 + roles/vibe/tasks/claude.yml | 68 + roles/vibe/tasks/code.yml | 74 + roles/vibe/tasks/jupyter.yml | 76 + roles/vibe/tasks/main.yml | 52 + roles/vibe/tasks/nodejs.yml | 45 + roles/vibe/templates/CLAUDE.md | 120 + roles/vibe/templates/code.env | 7 + roles/vibe/templates/code.svc | 20 + roles/vibe/templates/code.yml | 5 + roles/vibe/templates/jupyter.env | 3 + roles/vibe/templates/jupyter.svc | 20 + roles/vibe/templates/jupyter_config.py | 23 + slim.yml | 79 + templates/crit.yml | 1 + templates/distro | 1 + templates/nginx | 1 + templates/olap.yml | 1 + templates/oltp.yml | 1 + templates/pg-db.sql | 1 + templates/pg-user.sql | 1 + templates/tiny.yml | 1 + terraform/Makefile | 38 + terraform/README.md | 304 ++ terraform/spec/aws-cn.tf | 231 + terraform/spec/aws.tf | 269 ++ terraform/spec/azure.tf | 280 ++ terraform/spec/digitalocean.tf | 182 + terraform/spec/gcp.tf | 254 ++ terraform/spec/hetzner.tf | 199 + terraform/spec/linode.tf | 169 + terraform/spec/qcloud.tf | 235 ++ terraform/spec/vultr.tf | 205 + terraform/ssh | 40 + vagrant/Makefile | 395 ++ vagrant/README.md | 163 + vagrant/Vagrantfile.libvirt | 83 + vagrant/Vagrantfile.virtualbox | 105 + vagrant/config | 172 + vagrant/dns | 96 + vagrant/nuke | 39 + vagrant/spec/all.rb | 19 + vagrant/spec/citus.rb | 27 + vagrant/spec/deb.rb | 14 + vagrant/spec/deci.rb | 16 + vagrant/spec/dual.rb | 8 + vagrant/spec/example/check.rb | 36 + vagrant/spec/example/citus.rb | 10 + vagrant/spec/example/el7.rb | 8 + vagrant/spec/example/el8.rb | 8 + vagrant/spec/example/el9.rb | 8 + vagrant/spec/example/os.rb | 20 + vagrant/spec/example/ubuntu.rb | 13 + vagrant/spec/full.rb | 10 + vagrant/spec/meta.rb | 7 + vagrant/spec/minio.rb | 15 + vagrant/spec/old.rb | 10 + vagrant/spec/oss.rb | 14 + vagrant/spec/pro.rb | 20 + vagrant/spec/rpm.rb | 10 + vagrant/spec/simu.rb | 36 + vagrant/spec/trio.rb | 9 + vagrant/ssh | 122 + vibe.yml | 81 + 690 files changed, 176345 insertions(+), 34 deletions(-) create mode 100644 .gitignore create mode 100644 CLAUDE.md create mode 100644 KEYS create mode 100644 Makefile create mode 100644 NOTICE create mode 100644 ansible.cfg create mode 100755 app.yml create mode 100644 app/README.md create mode 100644 app/bytebase/.env create mode 100644 app/bytebase/Makefile create mode 100644 app/bytebase/README.md create mode 100644 app/bytebase/docker-compose.yml create mode 100644 app/dify/Makefile create mode 100644 app/dify/docker-compose.yml create mode 100644 app/dify/nginx/conf.d/default.conf.template create mode 100755 app/dify/nginx/docker-entrypoint.sh create mode 100644 app/dify/nginx/https.conf.template create mode 100644 app/dify/nginx/nginx.conf.template create mode 100644 app/dify/nginx/proxy.conf.template create mode 100755 app/dify/ssrf_proxy/docker-entrypoint.sh create mode 100644 app/dify/ssrf_proxy/squid.conf.template create mode 100644 app/dify/volumes/sandbox/conf/config.yaml create mode 100644 app/dify/volumes/sandbox/conf/config.yaml.example create mode 100644 app/dify/volumes/sandbox/dependencies/python-requirements.txt create mode 100644 app/electric/Makefile create mode 100644 app/electric/README.md create mode 100644 app/electric/docker-compose.yml create mode 100644 app/ferretdb/.env create mode 100644 app/ferretdb/Makefile create mode 100644 app/ferretdb/README.md create mode 100644 app/ferretdb/docker-compose.yml create mode 100644 app/gitea/.env create mode 100644 app/gitea/Makefile create mode 100644 app/gitea/README.md create mode 100644 app/gitea/docker-compose.yml create mode 100644 app/jupyter/.env create mode 100644 app/jupyter/Makefile create mode 100644 app/jupyter/README.md create mode 100644 app/jupyter/docker-compose.yml create mode 100644 app/kong/.env create mode 100644 app/kong/Makefile create mode 100644 app/kong/README.md create mode 100644 app/kong/docker-compose.yml create mode 100644 app/mattermost/.env create mode 100644 app/mattermost/Makefile create mode 100644 app/mattermost/docker-compose.yml create mode 100644 app/metabase/.env create mode 100644 app/metabase/Makefile create mode 100644 app/metabase/README.md create mode 100644 app/metabase/docker-compose.yml create mode 100644 app/minio/Makefile create mode 100644 app/minio/README.md create mode 100644 app/minio/docker-compose.yml create mode 100644 app/nocodb/.env create mode 100644 app/nocodb/Makefile create mode 100644 app/nocodb/README.md create mode 100644 app/nocodb/docker-compose.yml create mode 100644 app/odoo/.env create mode 100644 app/odoo/Makefile create mode 100644 app/odoo/README.md create mode 100644 app/odoo/docker-compose.yml create mode 100644 app/pg_exporter/.env create mode 100644 app/pg_exporter/Makefile create mode 100644 app/pg_exporter/README.md create mode 100644 app/pg_exporter/docker-compose.yml create mode 100644 app/pgadmin/.env create mode 100644 app/pgadmin/Makefile create mode 100644 app/pgadmin/README.md create mode 100644 app/pgadmin/docker-compose.yml create mode 100644 app/pgweb/Makefile create mode 100644 app/pgweb/README.md create mode 100644 app/pgweb/docker-compose.yml create mode 100644 app/postgrest/.env create mode 100644 app/postgrest/Makefile create mode 100644 app/postgrest/README.md create mode 100644 app/postgrest/docker-compose.yml create mode 100644 app/registry/Makefile create mode 100644 app/registry/README.md create mode 100644 app/registry/config.yml create mode 100644 app/registry/docker-compose.yml create mode 100644 app/supabase/Makefile create mode 100644 app/supabase/README.md create mode 100755 app/supabase/bin/dump create mode 100755 app/supabase/bin/load create mode 100644 app/supabase/docker-compose.yml create mode 100644 app/supabase/volumes/api/kong.yml create mode 100644 app/supabase/volumes/functions/hello/index.ts create mode 100644 app/supabase/volumes/functions/main/index.ts create mode 100644 app/supabase/volumes/logs/vector.yml create mode 100644 app/teable/.env create mode 100644 app/teable/Makefile create mode 100644 app/teable/README.md create mode 100644 app/teable/docker-compose.yml create mode 100644 app/wiki/.env create mode 100644 app/wiki/Makefile create mode 100644 app/wiki/README.md create mode 100644 app/wiki/docker-compose.yml create mode 100755 bin/boot-pkg create mode 100755 bin/etcd-add create mode 100755 bin/etcd-rm create mode 100755 bin/get-pkg create mode 100755 bin/install create mode 100755 bin/inventory_cmdb create mode 100755 bin/inventory_conf create mode 100755 bin/inventory_load create mode 100755 bin/node-add create mode 100755 bin/node-rm create mode 100755 bin/parse_nginx.py create mode 100755 bin/pglog-summary create mode 100755 bin/pgmon-add create mode 100755 bin/pgmon-rm create mode 100755 bin/pgsql-add create mode 100755 bin/pgsql-db create mode 100755 bin/pgsql-ext create mode 100755 bin/pgsql-hba create mode 100755 bin/pgsql-rm create mode 100755 bin/pgsql-svc create mode 100755 bin/pgsql-user create mode 100755 bin/profile create mode 100755 bin/psql.py create mode 100755 bin/redis-add create mode 100755 bin/redis-rm create mode 100755 bin/release create mode 100755 bin/repo-add create mode 100755 bin/schemaspy create mode 100755 bin/stackcollapse-perf.pl create mode 100755 bin/svg create mode 100755 bin/validate create mode 100755 bootstrap create mode 100755 cache.yml create mode 100755 cert.yml create mode 100644 conf/README.md create mode 100644 conf/app/dify.yml create mode 100644 conf/app/maybe.yml create mode 100644 conf/app/odoo.yml create mode 100644 conf/app/registry.yml create mode 100644 conf/app/teable.yml create mode 100644 conf/demo/bare.yml create mode 100644 conf/demo/debian.yml create mode 100644 conf/demo/demo.yml create mode 100644 conf/demo/el.yml create mode 100644 conf/demo/kernel.yml create mode 100644 conf/demo/kernels.yml create mode 100644 conf/demo/minio.yml create mode 100644 conf/demo/redis.yml create mode 100644 conf/demo/remote.yml create mode 100644 conf/demo/saas.yml create mode 100644 conf/demo/wool.yml create mode 100644 conf/docker.yml create mode 100644 conf/fat.yml create mode 100644 conf/ha/citus.yml create mode 100644 conf/ha/dual.yml create mode 100644 conf/ha/full.yml create mode 100644 conf/ha/safe.yml create mode 100644 conf/ha/simu.yml create mode 100644 conf/ha/trio.yml create mode 100644 conf/infra.yml create mode 100644 conf/ivory.yml create mode 100644 conf/meta.yml create mode 100644 conf/mongo.yml create mode 100644 conf/mssql.yml create mode 100644 conf/mysql.yml create mode 100644 conf/oriole.yml create mode 100644 conf/pgsql.yml create mode 100644 conf/pgtde.yml create mode 100644 conf/polar.yml create mode 100644 conf/rich.yml create mode 100644 conf/slim.yml create mode 100644 conf/vibe.yml create mode 100755 configure create mode 100755 deploy.yml create mode 100755 docker.yml create mode 100644 docker/.env create mode 100644 docker/Dockerfile create mode 100644 docker/Makefile create mode 100644 docker/README.md create mode 100644 docker/docker-compose.yml create mode 100755 etcd-rm.yml create mode 100755 etcd.yml create mode 100644 files/cmdb.sql create mode 100644 files/grafana/README.md create mode 100644 files/grafana/app/pglog-overview.json create mode 100644 files/grafana/app/pglog-session.json create mode 100755 files/grafana/grafana.py create mode 100644 files/grafana/infra/alertmanager-instance.json create mode 100644 files/grafana/infra/etcd-overview.json create mode 100644 files/grafana/infra/grafana-instance.json create mode 100644 files/grafana/infra/infra-overview.json create mode 100644 files/grafana/infra/inventory-cmdb.json create mode 100644 files/grafana/infra/logs-instance.json create mode 100644 files/grafana/infra/nginx-instance.json create mode 100644 files/grafana/infra/vlogs-instance.json create mode 100644 files/grafana/infra/vmalert-instance.json create mode 100644 files/grafana/infra/vmetrics-instance.json create mode 100644 files/grafana/infra/vtraces-instance.json create mode 100644 files/grafana/minio/minio-instance.json create mode 100644 files/grafana/minio/minio-overview.json create mode 100644 files/grafana/mongo/mongo-overview.json create mode 100644 files/grafana/node/claude-code.json create mode 100644 files/grafana/node/node-alert.json create mode 100644 files/grafana/node/node-cluster.json create mode 100644 files/grafana/node/node-disk.json create mode 100644 files/grafana/node/node-haproxy.json create mode 100644 files/grafana/node/node-instance.json create mode 100644 files/grafana/node/node-juice.json create mode 100644 files/grafana/node/node-overview.json create mode 100644 files/grafana/node/node-vector.json create mode 100644 files/grafana/node/node-vip.json create mode 100644 files/grafana/pgsql/pgcat-database.json create mode 100644 files/grafana/pgsql/pgcat-instance.json create mode 100644 files/grafana/pgsql/pgcat-locks.json create mode 100644 files/grafana/pgsql/pgcat-query.json create mode 100644 files/grafana/pgsql/pgcat-schema.json create mode 100644 files/grafana/pgsql/pgcat-table.json create mode 100644 files/grafana/pgsql/pgrds-cluster.json create mode 100644 files/grafana/pgsql/pgrds-instance.json create mode 100644 files/grafana/pgsql/pgsql-activity.json create mode 100644 files/grafana/pgsql/pgsql-alert.json create mode 100644 files/grafana/pgsql/pgsql-cluster.json create mode 100644 files/grafana/pgsql/pgsql-database.json create mode 100644 files/grafana/pgsql/pgsql-databases.json create mode 100644 files/grafana/pgsql/pgsql-exporter.json create mode 100644 files/grafana/pgsql/pgsql-instance.json create mode 100644 files/grafana/pgsql/pgsql-overview.json create mode 100644 files/grafana/pgsql/pgsql-patroni.json create mode 100644 files/grafana/pgsql/pgsql-persist.json create mode 100644 files/grafana/pgsql/pgsql-pgbouncer.json create mode 100644 files/grafana/pgsql/pgsql-pitr.json create mode 100644 files/grafana/pgsql/pgsql-proxy.json create mode 100644 files/grafana/pgsql/pgsql-query.json create mode 100644 files/grafana/pgsql/pgsql-replication.json create mode 100644 files/grafana/pgsql/pgsql-service.json create mode 100644 files/grafana/pgsql/pgsql-session.json create mode 100644 files/grafana/pgsql/pgsql-shard.json create mode 100644 files/grafana/pgsql/pgsql-table.json create mode 100644 files/grafana/pgsql/pgsql-tables.json create mode 100644 files/grafana/pgsql/pgsql-xacts.json create mode 100644 files/grafana/pigsty.json create mode 100644 files/grafana/redis/redis-cluster.json create mode 100644 files/grafana/redis/redis-instance.json create mode 100644 files/grafana/redis/redis-overview.json create mode 100644 files/migration/pg-meta.yml create mode 100644 files/mssql.sql create mode 100755 files/postgres/pg-backup create mode 100755 files/postgres/pg-badger create mode 100755 files/postgres/pg-basebackup create mode 100755 files/postgres/pg-cluster create mode 100755 files/postgres/pg-drop-role create mode 100755 files/postgres/pg-failover-callback create mode 100755 files/postgres/pg-fork create mode 100755 files/postgres/pg-heartbeat create mode 100755 files/postgres/pg-pass-add create mode 100755 files/postgres/pg-pitr create mode 100755 files/postgres/pg-repack create mode 100755 files/postgres/pg-role create mode 100755 files/postgres/pg-tune-hugepage create mode 100755 files/postgres/pg-vacuum create mode 100755 files/postgres/pgb-user create mode 100755 files/postgres/supa-kick create mode 100644 files/supabase.sql create mode 100755 files/victoria/bin/check create mode 100755 files/victoria/bin/new create mode 100755 files/victoria/bin/reload create mode 100755 files/victoria/bin/status create mode 100644 files/victoria/rules/etcd.yml create mode 100644 files/victoria/rules/infra.yml create mode 100644 files/victoria/rules/kafka.yml create mode 100644 files/victoria/rules/minio.yml create mode 100644 files/victoria/rules/mysql.yml create mode 100644 files/victoria/rules/node.yml create mode 100644 files/victoria/rules/pgsql.yml create mode 100644 files/victoria/rules/redis.yml create mode 100755 infra-rm.yml create mode 100755 infra.yml create mode 100755 juice.yml create mode 100755 minio-rm.yml create mode 100755 minio.yml create mode 100755 mongo.yml create mode 100755 node-rm.yml create mode 100755 node.yml create mode 100755 pgsql-db.yml create mode 100755 pgsql-migration.yml create mode 100755 pgsql-monitor.yml create mode 100755 pgsql-pitr.yml create mode 100755 pgsql-rm.yml create mode 100755 pgsql-user.yml create mode 100755 pgsql.yml create mode 100644 pigsty.yml create mode 100755 redis-rm.yml create mode 100755 redis.yml create mode 100644 roles/ca/README.md create mode 100644 roles/ca/defaults/main.yml create mode 100644 roles/ca/meta/main.yml create mode 100644 roles/ca/tasks/main.yml create mode 100644 roles/cache/README.md create mode 100644 roles/cache/defaults/main.yml create mode 100644 roles/cache/meta/main.yml create mode 100644 roles/cache/tasks/main.yml create mode 100644 roles/docker/README.md create mode 100644 roles/docker/defaults/main.yml create mode 100644 roles/docker/meta/main.yml create mode 100644 roles/docker/tasks/main.yml create mode 100644 roles/docker/templates/daemon.json.j2 create mode 100644 roles/etcd/README.md create mode 100644 roles/etcd/defaults/main.yml create mode 100644 roles/etcd/meta/main.yml create mode 100644 roles/etcd/tasks/config.yml create mode 100644 roles/etcd/tasks/main.yml create mode 100644 roles/etcd/templates/etcd.conf create mode 100644 roles/etcd/templates/etcd.pass create mode 100644 roles/etcd/templates/etcd.svc create mode 100644 roles/etcd/templates/etcdctl.sh create mode 100644 roles/etcd_remove/README.md create mode 100644 roles/etcd_remove/defaults/main.yml create mode 100644 roles/etcd_remove/meta/main.yml create mode 100644 roles/etcd_remove/tasks/leave.yml create mode 100644 roles/etcd_remove/tasks/main.yml create mode 100644 roles/ferret/README.md create mode 100644 roles/ferret/defaults/main.yml create mode 100644 roles/ferret/meta/main.yml create mode 100644 roles/ferret/tasks/cert.yml create mode 100644 roles/ferret/tasks/main.yml create mode 100644 roles/ferret/templates/ferretdb.env create mode 100644 roles/ferret/templates/ferretdb.svc create mode 100644 roles/haproxy/README.md create mode 100644 roles/haproxy/defaults/main.yml create mode 100644 roles/haproxy/files/haproxy.svc create mode 100644 roles/haproxy/meta/main.yml create mode 100644 roles/haproxy/tasks/main.yml create mode 100644 roles/haproxy/templates/haproxy.cfg.j2 create mode 100644 roles/haproxy/templates/service.j2 create mode 100644 roles/infra/README.md create mode 100644 roles/infra/defaults/main.yml create mode 100644 roles/infra/meta/main.yml create mode 100644 roles/infra/tasks/alertmanager.yml create mode 100644 roles/infra/tasks/blackbox.yml create mode 100644 roles/infra/tasks/cert.yml create mode 100644 roles/infra/tasks/dir.yml create mode 100644 roles/infra/tasks/dns.yml create mode 100644 roles/infra/tasks/env.yml create mode 100644 roles/infra/tasks/grafana.yml create mode 100644 roles/infra/tasks/main.yml create mode 100644 roles/infra/tasks/nginx.yml create mode 100644 roles/infra/tasks/pkg.yml create mode 100644 roles/infra/tasks/register.yml create mode 100644 roles/infra/tasks/user.yml create mode 100644 roles/infra/tasks/victoria.yml create mode 100644 roles/infra/templates/dns/dnsmasq.conf create mode 100644 roles/infra/templates/dns/dnsmasq.svc create mode 100755 roles/infra/templates/env/patronictl.yml.j2 create mode 100755 roles/infra/templates/env/pg_service.conf.j2 create mode 100755 roles/infra/templates/env/pigsty.j2 create mode 100755 roles/infra/templates/env/servers.j2 create mode 100644 roles/infra/templates/grafana/dashboard.yml.j2 create mode 100644 roles/infra/templates/grafana/datasource.yml.j2 create mode 100644 roles/infra/templates/grafana/grafana.svc create mode 100644 roles/infra/templates/nginx/404.html.j2 create mode 100644 roles/infra/templates/nginx/endpoint.conf create mode 100644 roles/infra/templates/nginx/home.conf create mode 100644 roles/infra/templates/nginx/htpasswd.j2 create mode 100644 roles/infra/templates/nginx/index.html.j2 create mode 100644 roles/infra/templates/nginx/index.zh.html.j2 create mode 100644 roles/infra/templates/nginx/link-cert.j2 create mode 100644 roles/infra/templates/nginx/nginx.conf.j2 create mode 100644 roles/infra/templates/nginx/nginx_exporter.env create mode 100644 roles/infra/templates/nginx/nginx_exporter.svc create mode 100644 roles/infra/templates/nginx/path.conf create mode 100644 roles/infra/templates/nginx/sign-cert.j2 create mode 100644 roles/infra/templates/prometheus/alertmanager.env create mode 100644 roles/infra/templates/prometheus/alertmanager.svc create mode 100644 roles/infra/templates/prometheus/alertmanager.yml create mode 100644 roles/infra/templates/prometheus/blackbox.env create mode 100644 roles/infra/templates/prometheus/blackbox.yml create mode 100644 roles/infra/templates/prometheus/blackbox_exporter.svc create mode 100644 roles/infra/templates/prometheus/infra.yml.j2 create mode 100644 roles/infra/templates/vector/grafana.yaml create mode 100644 roles/infra/templates/vector/nginx.yaml create mode 100644 roles/infra/templates/victoria/agent.yml create mode 100644 roles/infra/templates/victoria/prometheus.yml create mode 100644 roles/infra/templates/victoria/vlogs.env create mode 100644 roles/infra/templates/victoria/vlogs.svc create mode 100644 roles/infra/templates/victoria/vmalert.env create mode 100644 roles/infra/templates/victoria/vmalert.svc create mode 100644 roles/infra/templates/victoria/vmetrics.env create mode 100644 roles/infra/templates/victoria/vmetrics.svc create mode 100644 roles/infra/templates/victoria/vtraces.env create mode 100644 roles/infra/templates/victoria/vtraces.svc create mode 100644 roles/juice/README.md create mode 100644 roles/juice/defaults/main.yml create mode 100644 roles/juice/meta/main.yml create mode 100644 roles/juice/tasks/clean.yml create mode 100644 roles/juice/tasks/instance.yml create mode 100644 roles/juice/tasks/main.yml create mode 100644 roles/juice/templates/juice.env create mode 100644 roles/juice/templates/juice.svc create mode 100644 roles/minio/README.md create mode 100644 roles/minio/defaults/main.yml create mode 100644 roles/minio/handlers/main.yml create mode 100644 roles/minio/meta/main.yml create mode 100644 roles/minio/tasks/config.yml create mode 100644 roles/minio/tasks/install.yml create mode 100644 roles/minio/tasks/main.yml create mode 100644 roles/minio/tasks/provision.yml create mode 100644 roles/minio/templates/minio.env create mode 100644 roles/minio/templates/minio.svc create mode 100644 roles/minio/templates/policy.json create mode 100644 roles/minio_remove/README.md create mode 100644 roles/minio_remove/defaults/main.yml create mode 100644 roles/minio_remove/meta/main.yml create mode 100644 roles/minio_remove/tasks/main.yml create mode 100644 roles/node/README.md create mode 100644 roles/node/defaults/main.yml create mode 100755 roles/node/files/disable_numa.sh create mode 100755 roles/node/files/disable_thp.sh create mode 100755 roles/node/files/disk_prefetch.sh create mode 100644 roles/node/files/limits.conf create mode 100644 roles/node/files/node.sh create mode 100644 roles/node/files/policy-rc.d create mode 100644 roles/node/files/ssh.config create mode 100755 roles/node/files/static_network.sh create mode 100644 roles/node/meta/main.yml create mode 100644 roles/node/tasks/admin.yml create mode 100644 roles/node/tasks/cert.yml create mode 100644 roles/node/tasks/dns.yml create mode 100644 roles/node/tasks/main.yml create mode 100644 roles/node/tasks/pkg.yml create mode 100644 roles/node/tasks/sec.yml create mode 100644 roles/node/tasks/time.yml create mode 100644 roles/node/tasks/tune.yml create mode 100644 roles/node/tasks/vip.yml create mode 100644 roles/node/templates/chrony.conf.j2 create mode 100644 roles/node/templates/keepalived.conf.j2 create mode 100644 roles/node/templates/tuned-crit.conf create mode 100644 roles/node/templates/tuned-olap.conf create mode 100644 roles/node/templates/tuned-oltp.conf create mode 100644 roles/node/templates/tuned-tiny.conf create mode 100644 roles/node_id/README.md create mode 100644 roles/node_id/defaults/main.yml create mode 100644 roles/node_id/meta/main.yml create mode 100644 roles/node_id/tasks/main.yml create mode 100644 roles/node_id/vars/d12.aarch64.yml create mode 100644 roles/node_id/vars/d12.x86_64.yml create mode 100644 roles/node_id/vars/d13.aarch64.yml create mode 100644 roles/node_id/vars/d13.x86_64.yml create mode 100644 roles/node_id/vars/el10.aarch64.yml create mode 100644 roles/node_id/vars/el10.x86_64.yml create mode 100644 roles/node_id/vars/el7.x86_64.yml create mode 100644 roles/node_id/vars/el8.aarch64.yml create mode 100644 roles/node_id/vars/el8.x86_64.yml create mode 100644 roles/node_id/vars/el9.aarch64.yml create mode 100644 roles/node_id/vars/el9.x86_64.yml create mode 100644 roles/node_id/vars/u22.aarch64.yml create mode 100644 roles/node_id/vars/u22.x86_64.yml create mode 100644 roles/node_id/vars/u24.aarch64.yml create mode 100644 roles/node_id/vars/u24.x86_64.yml create mode 100644 roles/node_monitor/README.md create mode 100644 roles/node_monitor/defaults/main.yml create mode 100644 roles/node_monitor/files/keepalived_exporter.svc create mode 100644 roles/node_monitor/files/node_exporter.svc create mode 100644 roles/node_monitor/meta/main.yml create mode 100644 roles/node_monitor/tasks/main.yml create mode 100644 roles/node_monitor/tasks/vector.yml create mode 100644 roles/node_monitor/templates/node.yaml create mode 100644 roles/node_monitor/templates/vector.env create mode 100644 roles/node_monitor/templates/vector.svc create mode 100644 roles/node_monitor/templates/vector.yaml create mode 100644 roles/node_remove/README.md create mode 100644 roles/node_remove/defaults/main.yml create mode 100644 roles/node_remove/meta/main.yml create mode 100644 roles/node_remove/tasks/main.yml create mode 100644 roles/node_remove/tasks/nginx.yml create mode 100644 roles/pg_exporters/README.md create mode 100644 roles/pg_exporters/defaults/main.yml create mode 100644 roles/pg_exporters/meta/main.yml create mode 100644 roles/pg_exporters/tasks/main.yml create mode 100644 roles/pg_exporters/tasks/pg_exporter.yml create mode 100644 roles/pg_exporters/tasks/register_grafana.yml create mode 100644 roles/pg_exporters/tasks/register_grafana_db.yml create mode 100644 roles/pg_exporters/templates/pg_exporter.env create mode 100644 roles/pg_exporters/templates/pg_exporter.svc create mode 100644 roles/pg_exporters/templates/pg_exporter.yml create mode 100644 roles/pg_id/README.md create mode 100644 roles/pg_id/defaults/main.yml create mode 100644 roles/pg_id/meta/main.yml create mode 100644 roles/pg_id/tasks/main.yml create mode 100644 roles/pg_migration/README.md create mode 100644 roles/pg_migration/defaults/main.yml create mode 100644 roles/pg_migration/meta/main.yml create mode 100644 roles/pg_migration/tasks/main.yml create mode 100644 roles/pg_migration/templates/activate create mode 100644 roles/pg_migration/templates/check-db create mode 100644 roles/pg_migration/templates/check-hba create mode 100644 roles/pg_migration/templates/check-misc create mode 100644 roles/pg_migration/templates/check-repl create mode 100644 roles/pg_migration/templates/check-user create mode 100644 roles/pg_migration/templates/copy-diff create mode 100644 roles/pg_migration/templates/copy-progress create mode 100644 roles/pg_migration/templates/copy-schema create mode 100644 roles/pg_migration/templates/copy-seq create mode 100644 roles/pg_migration/templates/create-pub create mode 100644 roles/pg_migration/templates/create-sub create mode 100644 roles/pg_migration/templates/drop-pub create mode 100644 roles/pg_migration/templates/drop-sub create mode 100644 roles/pg_migration/templates/manual.md create mode 100644 roles/pg_monitor/README.md create mode 100644 roles/pg_monitor/defaults/main.yml create mode 100644 roles/pg_monitor/handlers/main.yml create mode 100644 roles/pg_monitor/meta/main.yml create mode 100644 roles/pg_monitor/tasks/main.yml create mode 100644 roles/pg_monitor/tasks/pg_exporter.yml create mode 100644 roles/pg_monitor/tasks/pgbackrest_exporter.yml create mode 100644 roles/pg_monitor/tasks/pgbouncer_exporter.yml create mode 100644 roles/pg_monitor/tasks/register_grafana.yml create mode 100644 roles/pg_monitor/tasks/register_vector.yml create mode 100644 roles/pg_monitor/tasks/register_victoria.yml create mode 100644 roles/pg_monitor/templates/pg_exporter.env create mode 100644 roles/pg_monitor/templates/pg_exporter.svc create mode 100644 roles/pg_monitor/templates/pg_exporter.yml create mode 100644 roles/pg_monitor/templates/pgbackrest.yaml create mode 100644 roles/pg_monitor/templates/pgbackrest_exporter.env create mode 100644 roles/pg_monitor/templates/pgbackrest_exporter.svc create mode 100644 roles/pg_monitor/templates/pgbouncer_exporter.env create mode 100644 roles/pg_monitor/templates/pgbouncer_exporter.svc create mode 100644 roles/pg_monitor/templates/pgbouncer_exporter.yml create mode 100644 roles/pg_monitor/templates/postgres.yaml create mode 100644 roles/pg_pitr/README.md create mode 100644 roles/pg_pitr/defaults/main.yml create mode 100644 roles/pg_pitr/meta/main.yml create mode 100644 roles/pg_pitr/tasks/etcd.yml create mode 100644 roles/pg_pitr/tasks/main.yml create mode 100644 roles/pg_pitr/tasks/pause.yml create mode 100644 roles/pg_pitr/tasks/pitr.yml create mode 100644 roles/pg_pitr/tasks/print.yml create mode 100644 roles/pg_pitr/tasks/start.yml create mode 100644 roles/pg_pitr/tasks/stop.yml create mode 100644 roles/pg_pitr/templates/command create mode 100755 roles/pg_pitr/templates/pg-restore create mode 100644 roles/pg_pitr/templates/pitr.conf create mode 100644 roles/pg_remove/README.md create mode 100644 roles/pg_remove/defaults/main.yml create mode 100644 roles/pg_remove/meta/main.yml create mode 100644 roles/pg_remove/tasks/dns.yml create mode 100644 roles/pg_remove/tasks/grafana.yml create mode 100644 roles/pg_remove/tasks/main.yml create mode 100644 roles/pg_remove/tasks/pg_exporter.yml create mode 100644 roles/pg_remove/tasks/pg_service.yml create mode 100644 roles/pg_remove/tasks/pgbackrest.yml create mode 100644 roles/pg_remove/tasks/pgbackrest_exporter.yml create mode 100644 roles/pg_remove/tasks/pgbouncer.yml create mode 100644 roles/pg_remove/tasks/pgbouncer_exporter.yml create mode 100644 roles/pg_remove/tasks/postgres.yml create mode 100644 roles/pg_remove/tasks/uninstall.yml create mode 100644 roles/pg_remove/tasks/victoria.yml create mode 100644 roles/pg_remove/tasks/vip.yml create mode 100644 roles/pgsql/README.md create mode 100644 roles/pgsql/defaults/main.yml create mode 100644 roles/pgsql/handlers/main.yml create mode 100644 roles/pgsql/meta/main.yml create mode 100644 roles/pgsql/tasks/cert.yml create mode 100644 roles/pgsql/tasks/config.yml create mode 100644 roles/pgsql/tasks/crontab.yml create mode 100644 roles/pgsql/tasks/database.yml create mode 100644 roles/pgsql/tasks/dbsu.yml create mode 100644 roles/pgsql/tasks/dns.yml create mode 100644 roles/pgsql/tasks/install.yml create mode 100644 roles/pgsql/tasks/main.yml create mode 100644 roles/pgsql/tasks/patroni.yml create mode 100644 roles/pgsql/tasks/pgbackrest.yml create mode 100644 roles/pgsql/tasks/pgbouncer.yml create mode 100644 roles/pgsql/tasks/service_local.yml create mode 100644 roles/pgsql/tasks/service_remote.yml create mode 100644 roles/pgsql/tasks/user.yml create mode 100644 roles/pgsql/tasks/vip.yml create mode 100644 roles/pgsql/templates/crit.yml create mode 100644 roles/pgsql/templates/olap.yml create mode 100644 roles/pgsql/templates/oltp.yml create mode 100644 roles/pgsql/templates/patroni.svc create mode 100755 roles/pgsql/templates/pg-alias create mode 100644 roles/pgsql/templates/pg-db.sql create mode 100755 roles/pgsql/templates/pg-init create mode 100644 roles/pgsql/templates/pg-init-roles.sql create mode 100644 roles/pgsql/templates/pg-init-template.sql create mode 100644 roles/pgsql/templates/pg-user.sql create mode 100644 roles/pgsql/templates/pg_hba.conf create mode 100644 roles/pgsql/templates/pgbackrest.conf create mode 100644 roles/pgsql/templates/pgbouncer.database create mode 100644 roles/pgsql/templates/pgbouncer.hba create mode 100644 roles/pgsql/templates/pgbouncer.ini create mode 100644 roles/pgsql/templates/pgbouncer.svc create mode 100644 roles/pgsql/templates/pgbouncer.useropts create mode 100644 roles/pgsql/templates/pgsodium_getkey create mode 100644 roles/pgsql/templates/postgres.svc create mode 100644 roles/pgsql/templates/psqlrc create mode 100644 roles/pgsql/templates/service.cfg create mode 100644 roles/pgsql/templates/tiny.yml create mode 100644 roles/pgsql/templates/vip-manager.svc create mode 100644 roles/pgsql/templates/vip-manager.yml create mode 100644 roles/pgsql/vars/main.yml create mode 100644 roles/redis/README.md create mode 100644 roles/redis/defaults/main.yml create mode 100644 roles/redis/meta/main.yml create mode 100644 roles/redis/tasks/exporter.yml create mode 100644 roles/redis/tasks/instance.yml create mode 100644 roles/redis/tasks/main.yml create mode 100644 roles/redis/tasks/node.yml create mode 100644 roles/redis/templates/redis-sentinel.conf create mode 100644 roles/redis/templates/redis.conf create mode 100644 roles/redis/templates/redis.svc create mode 100644 roles/redis/templates/redis.yaml create mode 100644 roles/redis_remove/README.md create mode 100644 roles/redis_remove/defaults/main.yml create mode 100644 roles/redis_remove/meta/main.yml create mode 100644 roles/redis_remove/tasks/main.yml create mode 100644 roles/repo/README.md create mode 100644 roles/repo/defaults/main.yml create mode 100644 roles/repo/meta/main.yml create mode 100644 roles/repo/tasks/build.yml create mode 100644 roles/repo/tasks/main.yml create mode 100644 roles/repo/tasks/nginx.yml create mode 100644 roles/repo/templates/default.conf.j2 create mode 100644 roles/repo/templates/index.html.j2 create mode 100644 roles/repo/templates/nginx.conf.j2 create mode 100644 roles/vibe/README.md create mode 100644 roles/vibe/defaults/main.yml create mode 100644 roles/vibe/meta/main.yml create mode 100644 roles/vibe/tasks/claude.yml create mode 100644 roles/vibe/tasks/code.yml create mode 100644 roles/vibe/tasks/jupyter.yml create mode 100644 roles/vibe/tasks/main.yml create mode 100644 roles/vibe/tasks/nodejs.yml create mode 100644 roles/vibe/templates/CLAUDE.md create mode 100644 roles/vibe/templates/code.env create mode 100644 roles/vibe/templates/code.svc create mode 100644 roles/vibe/templates/code.yml create mode 100644 roles/vibe/templates/jupyter.env create mode 100644 roles/vibe/templates/jupyter.svc create mode 100644 roles/vibe/templates/jupyter_config.py create mode 100755 slim.yml create mode 120000 templates/crit.yml create mode 120000 templates/distro create mode 120000 templates/nginx create mode 120000 templates/olap.yml create mode 120000 templates/oltp.yml create mode 120000 templates/pg-db.sql create mode 120000 templates/pg-user.sql create mode 120000 templates/tiny.yml create mode 100644 terraform/Makefile create mode 100644 terraform/README.md create mode 100644 terraform/spec/aws-cn.tf create mode 100644 terraform/spec/aws.tf create mode 100644 terraform/spec/azure.tf create mode 100644 terraform/spec/digitalocean.tf create mode 100644 terraform/spec/gcp.tf create mode 100644 terraform/spec/hetzner.tf create mode 100644 terraform/spec/linode.tf create mode 100644 terraform/spec/qcloud.tf create mode 100644 terraform/spec/vultr.tf create mode 100755 terraform/ssh create mode 100644 vagrant/Makefile create mode 100644 vagrant/README.md create mode 100644 vagrant/Vagrantfile.libvirt create mode 100644 vagrant/Vagrantfile.virtualbox create mode 100755 vagrant/config create mode 100755 vagrant/dns create mode 100755 vagrant/nuke create mode 100644 vagrant/spec/all.rb create mode 100644 vagrant/spec/citus.rb create mode 100644 vagrant/spec/deb.rb create mode 100644 vagrant/spec/deci.rb create mode 100644 vagrant/spec/dual.rb create mode 100644 vagrant/spec/example/check.rb create mode 100644 vagrant/spec/example/citus.rb create mode 100644 vagrant/spec/example/el7.rb create mode 100644 vagrant/spec/example/el8.rb create mode 100644 vagrant/spec/example/el9.rb create mode 100644 vagrant/spec/example/os.rb create mode 100644 vagrant/spec/example/ubuntu.rb create mode 100644 vagrant/spec/full.rb create mode 100644 vagrant/spec/meta.rb create mode 100644 vagrant/spec/minio.rb create mode 100644 vagrant/spec/old.rb create mode 100644 vagrant/spec/oss.rb create mode 100644 vagrant/spec/pro.rb create mode 100644 vagrant/spec/rpm.rb create mode 100644 vagrant/spec/simu.rb create mode 100644 vagrant/spec/trio.rb create mode 100755 vagrant/ssh create mode 100755 vibe.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..63714ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) + +######################################################### +# pigsty-ca and other certs +######################################################### +files/*.key +files/*.crt +files/pki/* + +######################################################### +# vagrant & terraform & docker +######################################################### +.vagrant/ +vagrant/Vagranfile +vagrant/.vagrant + +.terraform/ +terraform/.terraform/ +terraform/*.hcl +terraform/*.tfstate* +terraform/.terraform* +docker/data/ + + +######################################################### +# tmp files +######################################################### +# IDE files +.idea/ +.code/ +.vscode/ +.claude/ +llm.txt +.DS_Store +__pycache__/ +bin/__pycache__ +temp/ + +######################################################### +# private file +######################################################### +bin/release-pro +files/license.lic +test.yml +dist/ +files/*.tgz + +app/maybe diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..8902aa9 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,496 @@ +# Pigsty Production Environment - Claude Code Guide + +Pigsty v4.0.0 | PostgreSQL RDS/DBaaS | Apache-2.0 +- EN Docs: https://pigsty.io/docs +- CN Docs: https://pigsty.cc/docs +- Expert Support: https://pigsty.io/price (rh@vonng.com) + +--- + +## Important Files + +**`pigsty.yml`**, source of truth of this deployment, always read it to understand current environment. +Sometimes the real state may drift from the config file, especially after manual changes / failover. trust by verify. + +**`files/pki/ca/ca.key`** - highly sensitive CA private key. Keep it safe, never share it, don't lose it. + +If `pig` cli is available, you can use it to perform operation command with caution. https://pigsty.io/docs/pig + + +--- + +## Standard Operating Procedures + +### Before Any Change + +1. **Identify scope:** Which cluster? Which nodes? +2. **Check current state:** `pig pg list`, system status +3. **Verify backup:** `pig pb info` - is backup recent? +4. **Dry run first:** Add `--check` to playbook + +### For Dangerous Operations + +1. **Confirm with user** - even in YOLO mode +2. **Verify backup exists and is recent** +3. **Ask user to type exact target name** +4. **Execute with explicit `-l` limit flag** + +### Incident Response + +1. Read logs first, understand the situation, you can get it from `/pg/log`, or query victorialogs API +2. Check Patroni/HA status +3. Do NOT take any action without user confirmation +4. Recommend rollback/recovery options before executing + +--- + +## Permission Boundaries + +### ALWAYS ALLOWED (No confirmation needed) + +**Read-only operations:** +- Read `pigsty.yml` configuration file +- Read Ansible inventory and playbook files +- Read system monitoring metrics (CPU, memory, disk, network) +- Read PostgreSQL/Patroni cluster status via `pg list`, `patronictl` +- Read backup status via `pb info`, `pgbackrest info` +- Read service status via `systemctl status` +- Read logs: `/pg/log/`, system journal, nginx access logs +- Query monitoring dashboards and Grafana metrics +- Dry-run playbooks with `--check` flag +- Read extension catalog and documentation + +**Diagnostic commands:** +- `pig pg list ` - Patroni cluster status +- `pig pb info` - Backup information +- System health: `df`, `free`, `uptime`, `ps`, `top`, etc. + + +--- + +### REQUIRES USER CONFIRMATION (Always ask first) + +**Cluster lifecycle operations:** +- `pgsql.yml` - Initialize PostgreSQL cluster +- `pgsql-user.yml` - Create/modify users +- `pgsql-db.yml` - Create/modify databases +- `node.yml` - Provision nodes +- `infra.yml` - Deploy infrastructure +- Any playbook execution without `--check` + +**Configuration changes:** +- Modify `pigsty.yml` +- Change `pg_hba.conf` or HBA rules +- Modify PostgreSQL parameters +- Change Patroni configuration +- Update haproxy/vip configuration + +**Service operations:** +- Restart PostgreSQL/Patroni services +- Reload configuration (`pg_reload_conf()`) +- Switchover primary/replica (`pg switchover`) +- Backup execution (`pg-backup`) + +--- + +### FORBIDDEN - ALWAYS ASK (Even in YOLO mode) + +**Destructive operations - require explicit confirmation + backup verification:** + +Running Ansible playbooks, especially init/remove playbooks, always requires explicit user consent and permission, even in YOLO mode. Improper use may cause data loss! Your policy should be deny-by-default unless the user explicitly authorizes and requests. If you believe the user doesn't fully understand what they're doing, you should pause execution, explain and clarify further, and only proceed after confirming the user truly understands what they're about to do. + +``` +NEVER execute without user saying "yes, I confirm": + +- pgsql-rm.yml # Remove PostgreSQL cluster +- node-rm.yml # Remove node from cluster +- infra-rm.yml # Remove infrastructure +- redis-rm.yml # Remove Redis cluster +- etcd-rm.yml # Remove etcd cluster +- minio-rm.yml # Remove MinIO cluster + +- DROP DATABASE # Delete database +- DROP TABLE # Delete table +- TRUNCATE # Clear table data +- DELETE without WHERE # Mass deletion + +- rm -rf /pg/* # Delete data directories +- rm -rf ~/pigsty # Delete Pigsty installation +- rm pigsty.yml # Delete configuration +- rm -rf files/pki # Delete PKI certificates + +- patronictl remove # Remove cluster member +- pg_ctl stop -m immediate # Force stop +``` + +**Must verify before any destructive operation:** +1. Ask: "Which cluster/database/table?" +2. Ask: "Do you have a recent backup?" +3. Ask: "Type the exact name to confirm" + +--- + +### BUSINESS DATA - RESTRICTED ACCESS + +**Default policy: DO NOT read business data without permission** + +You may NOT execute these without explicit user consent: +- `SELECT * FROM ` +- Read application data tables +- Export or dump business data +- Access data in `/pg/data/` directly + +**Allowed for diagnostics (system tables only):** +- `pg_stat_*` views +- `pg_catalog.*` system catalogs +- `information_schema.*` +- Extension metadata tables + +**If user needs data analysis:** +1. Ask which tables/data are needed +2. Confirm user authorizes access +3. Limit query scope (use LIMIT, specific columns) +4. Never store or transmit sensitive data + +--- + +## Documentation Index + +- English: https://pigsty.io/docs +- Chinese: https://pigsty.cc/docs + +Use pigsty.io for English queries, pigsty.cc for Chinese queries. + +--- + +### About `/docs/about/` +Project information including features, history, licensing, roadmap, and community resources. + +- `feature.md` - Complete feature overview and capabilities of Pigsty +- `history.md` - Project history and version evolution +- `license.md` - Apache 2.0 licensing information and terms +- `roadmap.md` - Future development plans and upcoming features +- `case.md` - Real-world use cases and success stories +- `community.md` - Community resources, discussions, and contribution guidelines +- `compare/` - Comparisons with other database solutions and cloud RDS offerings + +--- + +### Concept `/docs/concept/` +Core concepts explaining Pigsty's architecture, high availability design, backup/recovery mechanisms, and security compliance. Essential reading for understanding how Pigsty works under the hood. + +- `arch/` - System architecture documentation covering PostgreSQL cluster design, node topology, and infrastructure components +- `model/` - Cluster model definitions for each module type (pgsql, infra, etcd, redis, minio) +- `iac/` - Infrastructure as Code principles and implementation + - `inventory.md` - Ansible inventory structure and how to define your infrastructure + - `parameter.md` - Parameter hierarchy and precedence rules + - `configure.md` - Configuration generation process using the configure script +- `ha/` - High availability concepts and implementation details + - `rto.md` - Recovery Time Objective analysis and guarantees + - `rpo.md` - Recovery Point Objective and data loss prevention + - `failure/` - Failure scenario handling (active failures, passive failures, manual intervention, network partitions) +- `pitr/` - Point-in-time recovery concepts and how pgBackRest enables time-travel for your data +- `sec/` - Security architecture including access control levels, CA/PKI infrastructure, and authentication mechanisms +- `monitor.md` - Observability stack design with VictoriaMetrics, Grafana, and alerting + +--- + +### Setup `/docs/setup/` +Quick start guide for single-node deployments. Follow these docs to get Pigsty running on your laptop or a single server within minutes. + +- `install.md` - Standard installation procedure with online package downloads +- `offline.md` - Air-gapped installation using pre-downloaded offline packages +- `slim.md` - Minimal installation without full monitoring stack for resource-constrained environments +- `config.md` - Configuration walkthrough explaining key parameters to customize +- `playbook.md` - How to execute Ansible playbooks and understand their output +- `security.md` - Security considerations for initial deployment +- `webui.md` - Accessing Grafana dashboards, Prometheus, and other web interfaces +- `pgsql.md` - Connecting to PostgreSQL using psql, pgAdmin, and application drivers +- `docker.md` - Alternative Docker-based deployment method + +--- + +### Deploy `/docs/deploy/` +Production deployment guide for multi-node, highly available clusters. Use these docs when deploying Pigsty in serious production environments. + +- `planning.md` - Architecture planning including node sizing, network topology, and resource allocation +- `prepare.md` - Pre-deployment checklist covering OS preparation, SSH access, and privilege requirements +- `install.md` - Step-by-step production installation with HA PostgreSQL clusters +- `sandbox.md` - Setting up sandbox environments for testing and development +- `vagrant.md` - Vagrant templates for local multi-node testing clusters +- `terraform.md` - Terraform templates for cloud infrastructure provisioning (AWS, GCP, Azure) +- `admin.md` - Admin node setup and configuration management best practices +- `security.md` - Production security hardening including SSL/TLS, firewall rules, and access controls + +--- + +### Reference `/docs/ref/` +Quick reference documentation for looking up specific technical details. Bookmark these pages for day-to-day operations. + +- `linux.md` - Supported Linux distributions and version compatibility matrix +- `module.md` - Complete module reference with dependencies and relationships +- `param.md` - Comprehensive parameter reference listing all configuration options with defaults and descriptions +- `playbook.md` - Ansible playbook reference with all available playbooks, tags, and execution examples +- `port.md` - Port allocation table showing all services and their default ports +- `fhs.md` - Filesystem hierarchy standard explaining directory structure and file locations +- `extension.md` - Complete catalog of 440+ PostgreSQL extensions with installation instructions + +--- + +### Apps `/docs/app/` +Application templates for deploying containerized software on top of Pigsty infrastructure. Each app comes with Docker Compose configuration and integration guides. + +- `nocodb.md`, `teable.md` - No-code database platforms for building applications without programming +- `gitea.md` - Self-hosted Git service similar to GitHub/GitLab +- `wiki.md` - Wiki and documentation platforms +- `mattermost.md` - Team collaboration and messaging platform +- `maybe.md` - Personal finance management application +- `metabase.md` - Business intelligence and analytics dashboard +- `kong.md` - API gateway for microservices architecture +- `jupyter.md` - Jupyter Notebook for data science and machine learning +- `bytebase.md` - Database DevOps and schema migration platform +- `pgadmin.md` - PostgreSQL graphical administration tool +- `supabase.md` - Open source Firebase alternative backend platform + +--- + +### Templates `/docs/conf/` +Ready-to-use configuration templates for various deployment scenarios. Copy and customize these for your specific needs. + +**Single-node templates:** `meta.md`, `rich.md`, `fat.md`, `slim.md`, `infra.md` - Various single-node configurations from minimal to feature-rich +**Kernel templates:** `pgsql.md`, `citus.md`, `polar.md`, `oriole.md`, `mysql.md`, `mssql.md` - Templates for different PostgreSQL variants and compatibility layers +**HA templates:** `dual.md`, `trio.md`, `full.md`, `safe.md`, `simu.md` - Multi-node high availability configurations from 2-node to full production +**Application templates:** `odoo.md`, `dify.md`, `teable.md`, `supabase.md`, `registry.md` - Pre-configured templates for specific applications +**Demo templates:** `demo.md`, `el.md`, `debian.md`, `oss.md`, `minio.md` - Demonstration and testing configurations + +--- + +### PGSQL Module `/docs/pgsql/` +The core PostgreSQL module documentation. This is the most important section for database administrators managing PostgreSQL clusters with Pigsty. + +#### Config `/docs/pgsql/config/` +Cluster configuration and definition. Learn how to declare PostgreSQL clusters, users, databases, and access rules in pigsty.yml. + +- `cluster.md` - Cluster topology definition including primary, replicas, and instance placement +- `kernel.md` - PostgreSQL kernel version selection and variant configuration +- `user.md` - User and role definitions with privilege management +- `db.md` - Database definitions including templates, extensions, and schemas +- `hba.md` - Host-based authentication rules controlling client access +- `acl.md` - Fine-grained access control lists for database objects +- `alias.md` - SQL command aliases for convenience + +#### Service `/docs/pgsql/service/` +Service exposure and access patterns. Explains how clients connect to PostgreSQL through HAProxy, VIP, and DNS. + +#### Security `/docs/pgsql/security/` +PostgreSQL security configuration including SSL/TLS, authentication methods, and encryption. + +#### Admin `/docs/pgsql/admin/` +Day-to-day administration operations. The essential guide for cluster lifecycle management. + +- `cluster.md` - Cluster lifecycle management including creation, scaling, and removal procedures +- `user.md` - Runtime user management with CREATE/ALTER/DROP operations +- `db.md` - Database administration including creation, configuration, and maintenance +- `hba.md` - HBA rule management and reload procedures +- `upgrade.md` - PostgreSQL major and minor version upgrade procedures +- `component.md` - Managing cluster components (Patroni, pgBouncer, HAProxy) +- `monitor.md` - Monitoring configuration and troubleshooting +- `patroni.md` - Patroni-specific operations including switchover, failover, and maintenance mode +- `pgbackrest.md` - Backup administration with pgBackRest commands and scheduling +- `pgbouncer.md` - Connection pooler management and configuration +- `ext.md` - Extension management including installation, updates, and removal +- `crontab.md` - Scheduled task configuration for maintenance operations + +#### Backup `/docs/pgsql/backup/` +Backup and recovery operations with pgBackRest. Critical documentation for data protection. + +- `mechanism.md` - How PostgreSQL backup works including WAL archiving and base backups +- `policy.md` - Backup retention policies and scheduling strategies +- `repository.md` - Backup repository configuration for local, S3, and MinIO storage +- `admin.md` - Backup administration commands and monitoring +- `restore.md` - Point-in-time recovery procedures and disaster recovery +- `cluster.md` - Cluster-level backup coordination and consistency + +#### Migration `/docs/pgsql/migration/` +Data migration guides for moving data between PostgreSQL versions and from other databases. + +#### Tutorial `/docs/pgsql/tutorial/` +Hands-on tutorials for common operational tasks. Step-by-step guides with real examples. + +- `drill.md` - Operational drills for practicing failover and recovery +- `drop.md` - Safe procedures for dropping databases, tables, and clusters +- `failure.md` - Failure scenario handling and recovery procedures +- `pitr.md` - Point-in-time recovery tutorial with practical examples +- `hugepage.md` - Linux huge pages configuration for improved performance +- `pg-fork.md` - Using PostgreSQL forks and variants (Citus, PolarDB, etc.) +- `pg-vip.md` - Virtual IP configuration for high availability +- `citus.md` - Citus distributed PostgreSQL setup and operations + +#### Monitor `/docs/pgsql/monitor/` +Monitoring system configuration and dashboard usage. + +- `dashboard.md` - Overview of available Grafana dashboards +- `dashboard/overview/` - System-wide overview dashboards +- `dashboard/cluster/` - Cluster-level monitoring dashboards +- `dashboard/instance/` - Instance-level detailed metrics +- `dashboard/database/` - Database-specific performance dashboards + +#### Param `/docs/pgsql/param/` +Complete PGSQL module parameter reference. All configuration options with descriptions and defaults. + +#### Ext `/docs/pgsql/ext/` +PostgreSQL extension management. Pigsty supports 440+ extensions out of the box. + +- `start.md` - Quick start guide for extension installation +- `intro.md` - Introduction to PostgreSQL extension ecosystem +- `pkg.md` - Extension packaging and distribution +- `download.md` - Downloading extensions from repositories +- `install.md` - Installing extensions on PostgreSQL clusters +- `config.md` - Configuring extensions with shared_preload_libraries and other settings +- `create.md` - Creating extensions in databases with CREATE EXTENSION +- `extension.md` - Detailed extension catalog and compatibility information +- `repo.md` - Extension repository management +- `remove.md` - Safely removing extensions from databases +- `update.md` - Updating extensions to newer versions + +#### Template `/docs/pgsql/template/` +Performance tuning templates for different workload types. + +- `oltp.md` - OLTP template optimized for transactional workloads with low latency +- `olap.md` - OLAP template optimized for analytical queries and large scans +- `crit.md` - Critical/financial template with maximum durability and consistency +- `tiny.md` - Tiny instance template for development and resource-constrained environments +- `tune.md` - Manual tuning guide for custom performance optimization + +#### Kernel `/docs/pgsql/kernel/` +PostgreSQL kernel variants and forks supported by Pigsty. + +- `postgres.md` - Vanilla PostgreSQL from the official PostgreSQL Global Development Group +- `citus.md` - Citus distributed PostgreSQL for horizontal scaling +- `babelfish.md` - WiltonDB/Babelfish for SQL Server protocol compatibility +- `ivorysql.md` - IvorySQL with Oracle PL/SQL compatibility +- `openhalo.md` - OpenHalo distributed database +- `percona.md` - Percona Distribution for PostgreSQL with enterprise features +- `orioledb.md` - OrioleDB with cloud-native storage engine +- `polardb.md` - Alibaba PolarDB for PostgreSQL +- `polardb-o.md` - PolarDB with Oracle compatibility +- `pgml.md` - PostgresML for machine learning workloads +- `supabase.md` - Supabase PostgreSQL configuration +- `greenplum.md` - Greenplum massively parallel processing database +- `cloudberry.md` - Cloudberry Database (Greenplum fork) +- `neon.md` - Neon serverless PostgreSQL + +#### Misc `/docs/pgsql/misc/` +Additional detailed documentation on specific topics. + +- `user.md` - In-depth user and role management guide +- `db.md` - Comprehensive database management documentation +- `acl.md` - Detailed access control list configuration +- `hba.md` - Complete HBA rule syntax and examples +- `svc.md` - Service configuration and load balancing details + +#### Other PGSQL Files +- `playbook.md` - PGSQL module playbook reference +- `metric.md` - PostgreSQL metrics and monitoring indicators +- `faq.md` - Frequently asked questions and troubleshooting + +--- + +### INFRA Module `/docs/infra/` +Infrastructure module providing DNS, NTP, Nginx reverse proxy, package repository, and the complete observability stack (VictoriaMetrics, Grafana, AlertManager). + +- `admin/` - Infrastructure administration (certificates, Grafana, Nginx, package repo management) +- `config.md` - INFRA module configuration options +- `param.md` - Complete parameter reference for infrastructure components +- `playbook.md` - Infrastructure playbooks for deployment and maintenance +- `monitor.md` - Monitoring stack configuration and dashboard access +- `faq.md` - Common infrastructure issues and solutions + +--- + +### NODE Module `/docs/node/` +Node provisioning module for host configuration, VIP management, HAProxy load balancing, and node_exporter metrics collection. + +- `admin.md` - Node administration including adding, removing, and reconfiguring hosts +- `config.md` - Node configuration options and tuning parameters +- `param.md` - Complete node parameter reference +- `playbook.md` - Node provisioning playbooks +- `monitor.md` - Node monitoring and alerting configuration +- `metric.md` - Node metrics collected by node_exporter + +--- + +### ETCD Module `/docs/etcd/` +Distributed consensus store module. ETCD serves as the Distributed Configuration Store (DCS) for Patroni, enabling PostgreSQL high availability with automatic failover. + +- `admin.md` - ETCD cluster administration including member management and maintenance +- `config.md` - ETCD configuration options and cluster topology +- `param.md` - Complete ETCD parameter reference +- `playbook.md` - ETCD deployment and maintenance playbooks +- `monitor.md` - ETCD monitoring and health checks + +--- + +### MINIO Module `/docs/minio/` +S3-compatible object storage module. MinIO serves as the backup repository for pgBackRest, enabling off-site backup storage and cross-cluster backup sharing. + +- `admin.md` - MinIO cluster administration and bucket management +- `config.md` - MinIO configuration for single-node and multi-node deployments +- `param.md` - Complete MinIO parameter reference +- `usage.md` - Using MinIO with pgBackRest and other applications +- `monitor.md` - MinIO monitoring and performance metrics +- `faq.md` - Common MinIO issues and solutions + +--- + +### REDIS Module `/docs/redis/` +Redis caching module supporting standalone, master-slave replication, sentinel, and cluster deployment modes. + +- `admin.md` - Redis administration including cluster management and failover +- `config.md` - Redis configuration for different deployment modes +- `param.md` - Complete Redis parameter reference +- `monitor.md` - Redis monitoring and performance metrics +- `faq.md` - Common Redis issues and troubleshooting + +--- + +### FERRET Module `/docs/ferret/` +FerretDB module providing MongoDB wire protocol compatibility layer on top of PostgreSQL. Allows MongoDB applications to use PostgreSQL as the backend. + +- `admin.md` - FerretDB administration and connection management +- `config.md` - FerretDB configuration and PostgreSQL integration + +--- + +### DOCKER Module `/docs/docker/` +Docker runtime module for deploying containerized applications alongside Pigsty infrastructure. + +- `usage.md` - Docker usage guide and container management +- `playbook.md` - Docker deployment playbooks +- `param.md` - Docker module parameters +- `faq.md` - Common Docker issues and solutions + +--- + +### PIG `/docs/pig/` +PIG package manager for PostgreSQL extensions. Command-line tool for discovering, installing, and managing the 440+ extensions available in the Pigsty ecosystem. + +--- + +### Repo `/docs/repo/` +Software repository documentation covering the package infrastructure that delivers PostgreSQL, extensions, and system packages. + +- `pgsql/` - PostgreSQL package repository (deb.md for Debian/Ubuntu, rpm.md for RHEL/Rocky) +- `infra/` - Infrastructure package repository +- `gpg.md` - GPG key management for package signing and verification + +--- + +### Other Modules (Experimental) +Additional experimental and internal modules not recommended for production use without thorough testing. + +- `pilot/` - Experimental module integrations (Kafka, MySQL, DuckDB, Kubernetes, Consul, etc.) +- `pg_exporter/` - Standalone PostgreSQL metrics exporter for external PostgreSQL instances +- `piglet/` - Lightweight Pigsty variant for minimal deployments +- `juice/`, `vibe/` - Internal development modules + diff --git a/KEYS b/KEYS new file mode 100644 index 0000000..6d3c963 --- /dev/null +++ b/KEYS @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGaV5PwBEACbErI+7yOrsXTT3mR83O6Fw9WyHJqozhyNPF3dA1gAtWpfWqd4 +S9x6vBjVwUbIRn21jYgov0hDiaLABNQhRzifvVr0r1IjBW8lhA8zJGaO42Uz0aBW +YIkajOklsXgYMX+gSmy5WXzM31sDQVMnzptHh9dwW067hMM5pJKDslu2pLMwSb9K +QgIFcYsaR0taBkcDg4dNu1gncriD/GcdXIS0/V4R82DIYeIqj2S0lt0jDTACbUz3 +C6esrTw2XerCeHKHb9c/V+KMhqvLJOOpy/aJWLrTGBoaH7xw6v0qg32OYiBxlUj9 +VEzoQbDfbRkR+jlxiuYP3scUs/ziKrSh+0mshVbeuLRSNfuHLa7C4xTEnATcgD1J +MZeMaJXIcDt+DN+1aHVQjY5YNvr5wA3ykxW51uReZf7/odgqVW3+1rhW5pd8NQKQ +qoVUHOtIrC9KaiGfrczEtJTNUxcNZV9eBgcKHYDXB2hmR2pIf7WvydgXTs/qIsXg +SIzfKjisi795Dd5GrvdLYXVnu9YzylWlkJ5rjod1wnSxkI/CcCJaoPLnXZA9KV7A +cpMWWaUEXP/XBIwIU+vxDd1taBIaPIOv1KIdzvG7QqAQtf5Lphi5HfaGvBud/CVt +mvWhRPJMr1J0ER2xAgU2iZR7dN0vSF6zDqc0W09RAoC0nDS3tupDX2BrOwARAQAB +tCRSdW9oYW5nIEZlbmcgKFBpZ3N0eSkgPHJoQHZvbm5nLmNvbT6JAlEEEwEIADsW +IQSVkqe8emguczM3bgnnk12Nub2LIAUCZpXk/AIbAwULCQgHAgIiAgYVCgkICwIE +FgIDAQIeBwIXgAAKCRDnk12Nub2LIOMuEACBLVc09O4icFwc45R3KMvOMu14Egpn +UkpmBKhErjup0TIunzI0zZH6HG8LGuf6XEdH4ItCJeLg5349UE00BUHNmxk2coo2 +u4Wtu28LPqmxb6sqpuRAaefedU6vqfs7YN6WWp52pVF1KdOHkIOcgAQ9z3ZHdosM +I/Y/UxO2t4pjdCAfJHOmGPrbgLcHSMpoLLxjuf3YIwS5NSfjNDd0Y8sKFUcMGLCF +5P0lv5feLLdZvh2Una34UmHKhZlXC5E3vlY9bf/LgsRzXRFQosD0RsCXbz3Tk+zF ++j/eP3WhUvJshqIDuY6eJYCzMjiA8sM5gety+htVJuD0mewp+qAhjxE0d4bIr4qO +BKQzBt9tT2ackCPdgW42VPS+IZymm1oMET0hgZfKiVpwsKO6qxeWn4RW2jJ0zkUJ +MsrrxOPFdZQAtuFcLwa5PUAHHs6XQT2vzxDpeE9lInQ14lshofU5ZKIeb9sbvb/w +P+xnDqvZ1pcotEIBvDK0S0jHbHHqtioIUdDFvdCBlBlYP1TQRNPlJ7TJDBBvhj8i +fmjQsYSV1u36aHOJVGYNHv+SyJpVd3nHCZn97ADM9qHnDm7xljyHXPzIx4FMmBGJ +UTiLH5yxa1xhWr42Iv3TykaQJVbpydmBuegFR8WbWitAvVqI3HvRG+FalLsjJruc +8YDAf7gHdj/937kCDQRmleT8ARAAmJxscC76NZzqFBiaeq2+aJxOt1HGPqKb4pbz +jLKRX9sFkeXuzhfZaNDljnr2yrnQ75rit9Aah/loEhbSHanNUDCNmvOeSEISr9yA +yfOnqlcVOtcwWQK57n6MvlCSM8Js3jdoSmCFHVtdFFwxejE5ok0dk1VFYDIg6DRk +ZBMuxGO7ZJW7TzCxhK4AL+NNYA2wX6b+IVMn6CA9kwNwCNrrnGHR1sblSxZp7lPo ++GsqzYY0LXGR2eEicgKd4lk38gaO8Q4d1mlpX95vgdhGKxR+CM26y9QU0qrO1hXP +Fw6lX9HfIUkVNrqAa1mzgneYXivnLvcj8gc7bFAdweX4MyBHsmiPm32WqjUJFAmw +kcKYaiyfDJ+1wusa/b+7RCnshWc8B9udYbXfvcpOGgphpUuvomKT8at3ToJfEWmR +BzToYYTsgAAX8diY/X53BHCE/+MhLccglEUYNZyBRkTwDLrS9QgNkhrADaTwxsv1 +8PwnVKve/ZxwOU0QGf4ZOhA2YQOE5hkRDR5uY2OHsOS5vHsd9Y6kNNnO8EBy99d1 +QiBJOW3AP0nr4Cj1/NhdigAujsYRKiCAuPT7dgqART58VU4bZ3PgonMlziLe7+ht +YYxV+wyP6LVqicDd0MLLvG7r/JOiWuABOUxsFFaRecehoPJjeAEQxnWJjedokXKL +HVOFaEkAEQEAAYkCNgQYAQgAIBYhBJWSp7x6aC5zMzduCeeTXY25vYsgBQJmleT8 +AhsMAAoJEOeTXY25vYsgG8sP/3UdsWuiwTsf/x4BTW82K+Uk9YwZDnUNH+4dUMED +bKT1C6CbuSZ7Mnbi2rVsmGzOMs9MehIx6Ko8/iCR2OCeWi8Q+wM+iffAfWuT1GK6 +7f/VIfoYBUWEa+kvDcPgEbd5Tu7ZdUO/jROVBSlXRSjzK9LpIj7GozBTJ8Vqy5x7 +oqbWPPEYtGDVHime8o6f5/wfhNgL3mFnoq6srK7KhwACwfTXlNqAlGiXGa30Yj+b +Cj6IvmxoII49E67/ovMEmzDCb3RXiaL6OATy25P+HQJvWvAam7Qq5Xn+bZg65Mup +vXq3zoX0a7EKXc5vsJVNtTlXO1ATdYszKP5uNzkHrNAN52VRYaowq1vPy/MVMbSI +rL/hTFKr7ZNhmC7jmS3OuJyCYQsfEerubtBUuc/W6JDc2oTI3xOG1S2Zj8f4PxLl +H7vMG4E+p6eOrUGw6VQXjFsH9GtwhkPh/ZGMKENb2+JztJ02674Cok4s5c/lZFKz +mmRUcNjX2bm2K0GfGG5/hAog/CHCeUZvwIh4hZLkdeJ1QsIYpN8xbvY7QP6yh4VB +XrL18+2sontZ45MsGResrRibB35x7IrCrxZsVtRJZthHqshiORPatgy+AiWcAtEv +UWEnnC1xBSasNebw4fSE8AJg9JMCRw+3GAetlotOeW9q7PN6yrXD9rGuV/QquQNd +/c7w +=4rRi +-----END PGP PUBLIC KEY BLOCK----- diff --git a/LICENSE b/LICENSE index 261eeb9..33145a4 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -186,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright (C) 2018-2026 Ruohang Feng, @Vonng (rh@vonng.com) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -198,4 +199,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. + limitations under the License. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5778038 --- /dev/null +++ b/Makefile @@ -0,0 +1,696 @@ +#==============================================================# +# File : Makefile +# Desc : pigsty shortcuts +# Ctime : 2019-04-13 +# Mtime : 2026-01-27 +# Path : Makefile +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +#==============================================================# +# pigsty version string +VERSION?=v4.0.0 + +# detect architecture +ARCH?=x86_64 +UNAME_ARCH := $(shell uname -m) +ifeq ($(UNAME_ARCH),arm64) + ARCH := aarch64 +else ifeq ($(UNAME_ARCH),aarch64) + ARCH := aarch64 +else + ARCH := $(UNAME_ARCH) +endif + +# variables +SRC_PKG=pigsty-$(VERSION).tgz +APP_PKG=pigsty-app-$(VERSION).tgz +DBA_PKG=pigsty-dba-$(VERSION).tgz +DOCKER_PKG=pigsty-docker-$(VERSION).tgz +EL8_PKG=pigsty-pkg-$(VERSION).el8.${ARCH}.tgz +EL9_PKG=pigsty-pkg-$(VERSION).el9.${ARCH}.tgz +EL10_PKG=pigsty-pkg-$(VERSION).el10.${ARCH}.tgz +D12_PKG=pigsty-pkg-$(VERSION).d12.${ARCH}.tgz +D13_PKG=pigsty-pkg-$(VERSION).d13.${ARCH}.tgz +U22_PKG=pigsty-pkg-$(VERSION).u22.${ARCH}.tgz +U24_PKG=pigsty-pkg-$(VERSION).u24.${ARCH}.tgz +META?=10.10.10.10 +OS?=el9 +PKG?="" +#PKG?=pro + +# append / to PKG if not already present +ifeq ($(filter %/,$(PKG)),) +ifneq ($(PKG),) +PKG:=$(PKG)/ +endif +endif + + +############################################################### +# 1. Quick Start # +############################################################### +# run with nopass SUDO user (or root) on Linux node +default: tip +tip: + echo $(ARCH) + @echo "# Run on Linux node with nopass sudo & ssh access" + @echo 'curl -fsSL https://repo.pigsty.io/get | bash' + @echo "./bootstrap # prepare local repo & ansible" + @echo "./configure # pre-check and templating config" + @echo "./deploy.yml # install pigsty on current node" + +# print pkg download links +link: + @echo 'curl -fsSL https://repo.pigsty.io/get | bash' + +# serve a local docs with docsify or python http +doc: + docs/serve + +#-------------------------------------------------------------# +# (1). BOOTSTRAP pigsty pkg & util preparedness +boot: bootstrap +bootstrap: + ./bootstrap + +# (2). CONFIGURE pigsty in interactive mode +conf: configure +configure: + ./configure + +# (3). DEPLOY pigsty on current node +deploy: + ./deploy.yml +############################################################### + + + +############################################################### +# OUTLINE # +############################################################### +# (1). Quick-Start : shortcuts for launching pigsty (above) +# (2). Download : shortcuts for downloading resources +# (3). Configure : shortcuts for configure pigsty +# (4). Install : shortcuts for running playbooks +# (5). Sandbox : shortcuts for manage sandbox vm nodes +# (6). Testing : shortcuts for testing features +# (7). Develop : shortcuts for dev purpose +# (8). Release : shortcuts for release and publish +# (9). Misc : shortcuts for miscellaneous tasks +############################################################### + + + +############################################################### +# 2. Download # +############################################################### +# There are two things that need to be downloaded: +# pigsty.tgz : source code +# pkg.tgz : offline rpm packages (optional) +# +# get latest stable version to ~/pigsty +src: + curl -SL https://github.com/pgsty/pigsty/releases/download/${VERSION}/${SRC_PKG} -o ~/pigsty.tgz +############################################################### + + + +############################################################### +# 3. Configure # +############################################################### +# there are several things that need to be checked before deploy +# use ./configure or `make config` to run interactive wizard +# it will install ansible (from offline rpm repo if available) + +# common interactive configuration procedure +c: configure + +# validate config inventory +v: validate +validate: + bin/validate +############################################################### + + + +############################################################### +# 4. Install # +############################################################### +# pigsty is deployed via ansible-playbook + +# install pigsty on meta nodes +infra: + ./infra.yml + +# rebuild repo +repo: repo-build node-repo + +# write upstream repo to /etc/yum.repos.d +repo-upstream: + ./infra.yml --tags=repo_upstream + +repo-check: + ./deploy.yml -t node_repo,node_pkg,infra_pkg,pg_pkg + +# re-build local repo +repo-build: + ./infra.yml -t repo_build + +# add extra packages to local repo +repo-add: + ./infra.yml -t repo_build -t repo_upstream,repo_cache,repo_pkg -e '{"repo_packages":[],"repo_url_packages":[]}' + +repo-clean: + ansible all -b -a 'rm -rf /www/pigsty/repo_complete' + +node-repo: + ./node.yml -t node_repo + +node-pkg: + ./node.yml -t node_pkg + +node-upstream: + ./node.yml -t node_repo -e node_repo_modules=node,pgsql,infra,local + +redeploy: repo-clean + ./deploy.yml + +# init grafana +grafana: + ./infra.yml --tags=grafana + ./pgsql.yml --tags=register_ds + +# init vlogs & vector +vv: vlogs vector +vlogs: + ./infra.yml --tags=vlogs -e vlogs_clean=true +vector: + ./node.yml --tags=vector -e vector_clean=true -e vector_read_from=beginning + +# nginx & certbot +nginx: + ./infra.yml -t nginx +cert: + ./infra.yml -t nginx_certbot,nginx_reload -e certbot_sign=true + +# init docker +docker: + ./docker.yml +app: + ./app.yml + +# claude +claude: + ./vibe.yml -t nodejs,claude + +# install & uninstall pgsql (dangerous!!) +pgsql-add: + ./infra.yml -t repo_build + ./pgsql.yml +pgsql-rm: + ./pgsql-rm.yml -e pg_uninstall=true +pgsql-ext: repo-add + ./infra.yml -t repo_build + ./pgsql.yml -t pg_ext + +############################################################### + + + + +############################################################### +# 5. Vagrant # +############################################################### +# shortcuts to pull up vm nodes with vagrant on your own MacOS +# DO NOT RUN THESE SHORTCUTS ON YOUR META NODE!!! +# These shortcuts are running on your HOST machine which run +# pigsty sandbox via virtualbox managed by vagrant. +#=============================================================# +# to setup vagrant sandbox env on your MacOS host: +# +# Prepare +# (1). make deps (once) Install MacOS deps with homebrew +# (2). make dns (once) Write static DNS +# (3). make start (once) Pull-up vm nodes and setup ssh access +# (4). make demo Boot meta node same as Quick-Start +#=============================================================# + +#------------------------------# +# 1. deps (macos) +#------------------------------# +# install macos sandbox software dependencies +deps: + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + brew install vagrant virtualbox ansible + +#------------------------------# +# 2. dns +#------------------------------# +# write static dns records (sudo password required) (only run on first time) +dns: + sudo vagrant/dns + +#------------------------------# +# 3. start +#------------------------------# +# start will pull up node and write ssh-config +# it may take a while to download the box for the first time +start: up ssh # 1-node version +ssh: # add current vagrant ssh config to your ~/.ssh/pigsty_config + vagrant/ssh +tssh: # add current terraform ssh config to your ~/.ssh/pigsty_config + terraform/ssh + +#------------------------------# +# vagrant vm management +#------------------------------# +# default node (meta) +up: + cd vagrant && vagrant up +dw: + cd vagrant && vagrant halt +del: + cd vagrant && vagrant destroy -f +nuke: + cd vagrant && ./nuke +new: del up +clean: del +#------------------------------# +# extra nodes: node-{1,2,3} +up-test: + cd vagrant && vagrant up node-1 node-2 node-3 +dw-test: + cd vagrant && vagrant halt node-1 node-2 node-3 +del-test: + cd vagrant && vagrant destroy -f node-1 node-2 node-3 +new-test: del-test up-test +#------------------------------# +# status +st: status +status: + cd vagrant && vagrant status +suspend: + cd vagrant && vagrant suspend +resume: + cd vagrant && vagrant resume + +############################################################### + + + +############################################################### +# 6. Testing # +############################################################### +# Convenient shortcuts for add traffic to sandbox pgsql clusters +# ri test-ri : init pgbench on meta or pg-test cluster +# rw test-rw : read-write pgbench traffic on meta or pg-test +# ro test-ro : read-only pgbench traffic on meta or pg-test +# rc test-rc : clean-up pgbench tables on meta or pg-test +# test-rw2 & test-ro2 : heavy load version of test-rw, test-ro +# test-rb{1,2,3} : reboot node 1,2,3 +#=============================================================# +# meta cmdb bench +ri: + pgbench -is10 postgres://dbuser_meta:DBUser.Meta@$(META):5433/meta +rc: + psql -AXtw postgres://dbuser_meta:DBUser.Meta@$(META):5433/meta -c 'DROP TABLE IF EXISTS pgbench_accounts, pgbench_branches, pgbench_history, pgbench_tellers;' +rw: + while true; do pgbench -nv -P1 -c4 --rate=64 -T10 postgres://dbuser_meta:DBUser.Meta@$(META):5433/meta; done +ro: + while true; do pgbench -nv -P1 -c8 --rate=256 -S -T10 postgres://dbuser_meta:DBUser.Meta@$(META):5434/meta; done +rh: + ssh $(META) 'sudo -iu postgres /pg/bin/pg-heartbeat' +# pg-test cluster benchmark +test-ri: + pgbench -is10 postgres://test:test@pg-test:5436/test +test-rc: + psql -AXtw postgres://test:test@pg-test:5433/test -c 'DROP TABLE IF EXISTS pgbench_accounts, pgbench_branches, pgbench_history, pgbench_tellers;' +# pgbench small read-write / read-only traffic (rw=64TPS, ro=512QPS) +test-rw: + while true; do pgbench -nv -P1 -c4 --rate=32 -T10 postgres://test:test@pg-test:5433/test; done +test-ro: + while true; do pgbench -nv -P1 -c8 -S --rate=256 -T10 postgres://test:test@pg-test:5434/test; done +# pgbench read-write / read-only traffic (maximum speed) +test-rw2: + while true; do pgbench -nv -P1 -c16 -T10 postgres://test:test@pg-test:5433/test; done +test-ro2: + while true; do pgbench -nv -P1 -c64 -T10 -S postgres://test:test@pg-test:5434/test; done +test-rh: + ssh node-1 'sudo -iu postgres /pg/bin/pg-heartbeat' +#------------------------------# +# show patroni status for pg-test cluster +test-st: + ssh -t node-1 "sudo -iu postgres patronictl -c /etc/patroni/patroni.yml list -W" +# reboot node 1,2,3 +test-rb1: + ssh -t node-1 "sudo reboot" +test-rb2: + ssh -t node-2 "sudo reboot" +test-rb3: + ssh -t node-3 "sudo reboot" +############################################################### + + + + +############################################################### +# 7. Develop # +############################################################### +# other shortcuts for development +#=============================================================# + +#------------------------------# +# grafana dashboard management +#------------------------------# +di: dashboard-init # init grafana dashboards +dashboard-init: + cd files/grafana/ && ./grafana.py init + +dd: dashboard-dump # dump grafana dashboards +dashboard-dump: + cd files/grafana/ && ./grafana.py dump + +dc: dashboard-clean # cleanup grafana dashboards +dashboard-clean: + cd files/grafana/ && ./grafana.py clean + +du: dashboard-clean dashboard-init # update grafana dashboards + +#------------------------------# +# copy source & packages +#------------------------------# +# copy latest source code +copy: copy-src copy-pkg use-src use-pkg +cc: release copy-src copy-pkg use-src use-pkg + +# copy pigsty source code +copy-src: + scp "dist/${VERSION}/${SRC_PKG}" $(META):~/pigsty.tgz + +# copy offline packages (set OS=el8/el9/el10/d12/d13/u22/u24) +copy-pkg: + scp dist/${VERSION}/$(PKG)pigsty-pkg-${VERSION}.${OS}.${ARCH}.tgz $(META):/tmp/pkg.tgz +copy-el8: + scp dist/${VERSION}/$(PKG)${EL8_PKG} $(META):/tmp/pkg.tgz +copy-el9: + scp dist/${VERSION}/$(PKG)${EL9_PKG} $(META):/tmp/pkg.tgz +copy-el10: + scp dist/${VERSION}/$(PKG)${EL10_PKG} $(META):/tmp/pkg.tgz +copy-d12: + scp dist/${VERSION}/$(PKG)${D12_PKG} $(META):/tmp/pkg.tgz +copy-d13: + scp dist/${VERSION}/$(PKG)${D13_PKG} $(META):/tmp/pkg.tgz +copy-u22: + scp dist/${VERSION}/$(PKG)${U22_PKG} $(META):/tmp/pkg.tgz +copy-u24: + scp dist/${VERSION}/$(PKG)${U24_PKG} $(META):/tmp/pkg.tgz +copy-app: + scp dist/${VERSION}/${APP_PKG} $(META):~/app.tgz + ssh -t $(META) 'rm -rf ~/app; tar -xf app.tgz; rm -rf app.tgz' +copy-all: copy-src copy-pkg + +# extract packages +use-src: + ssh -t $(META) 'rm -rf ~/pigsty; tar -xf pigsty.tgz; rm -rf pigsty.tgz' +use-pkg: + ssh $(META) "sudo mkdir -p /www; sudo tar -xf /tmp/pkg.tgz -C /www" +use-all: use-src use-pkg + +# load config into cmdb +cmdb: + bin/inventory_load + bin/inventory_cmdb + +#------------------------------# +# build env shortcuts +#------------------------------# +# copy src to build environment +cso: copy-src-oss +copy-src-oss: + scp "dist/${VERSION}/${SRC_PKG}" el9:~/pigsty.tgz + scp "dist/${VERSION}/${SRC_PKG}" d12:~/pigsty.tgz + scp "dist/${VERSION}/${SRC_PKG}" u24:~/pigsty.tgz + ssh -t el9 'rm -rf ~/pigsty; tar -xf pigsty.tgz; rm -rf pigsty.tgz' + ssh -t d12 'rm -rf ~/pigsty; tar -xf pigsty.tgz; rm -rf pigsty.tgz' + ssh -t u24 'rm -rf ~/pigsty; tar -xf pigsty.tgz; rm -rf pigsty.tgz' +copy-src-pro: copy-src-oss + scp "dist/${VERSION}/${SRC_PKG}" el8:~/pigsty.tgz + scp "dist/${VERSION}/${SRC_PKG}" u22:~/pigsty.tgz + ssh -t el8 'rm -rf ~/pigsty; tar -xf pigsty.tgz; rm -rf pigsty.tgz' + ssh -t u22 'rm -rf ~/pigsty; tar -xf pigsty.tgz; rm -rf pigsty.tgz' +csr: copy-src-rpm +copy-src-rpm: + scp "dist/${VERSION}/${SRC_PKG}" el8:~/pigsty.tgz + scp "dist/${VERSION}/${SRC_PKG}" el9:~/pigsty.tgz + ssh -t el8 'rm -rf ~/pigsty; tar -xf pigsty.tgz; rm -rf pigsty.tgz' + ssh -t el9 'rm -rf ~/pigsty; tar -xf pigsty.tgz; rm -rf pigsty.tgz' + ssh -t el8 'cd ~/pigsty && ./configure -i 10.10.10.8' + ssh -t el9 'cd ~/pigsty && ./configure -i 10.10.10.9' +csd: copy-src-deb +copy-src-deb: + scp "dist/${VERSION}/${SRC_PKG}" d12:~/pigsty.tgz + scp "dist/${VERSION}/${SRC_PKG}" u22:~/pigsty.tgz + scp "dist/${VERSION}/${SRC_PKG}" u24:~/pigsty.tgz + ssh -t d12 'rm -rf ~/pigsty; tar -xf pigsty.tgz; rm -rf pigsty.tgz' + ssh -t u22 'rm -rf ~/pigsty; tar -xf pigsty.tgz; rm -rf pigsty.tgz' + ssh -t u24 'rm -rf ~/pigsty; tar -xf pigsty.tgz; rm -rf pigsty.tgz' + ssh -t d12 'cd ~/pigsty && ./configure -i 10.10.10.12' + ssh -t u22 'cd ~/pigsty && ./configure -i 10.10.10.22' + ssh -t u24 'cd ~/pigsty && ./configure -i 10.10.10.24' +dfx: deb-fix +deb-fix: + scp /etc/resolv.conf d12:/tmp/resolv.conf; + ssh -t d12 'sudo mv /tmp/resolv.conf /etc/resolv.conf' + scp /etc/resolv.conf u24:/tmp/resolv.conf; + ssh -t u24 'sudo mv /tmp/resolv.conf /etc/resolv.conf' + +#------------------------------# +# push / pull +#------------------------------# +push: + rsync -avz ./ sv:~/pigsty/ --delete --exclude-from 'vagrant/Vagrantfile' +pull: + rsync -avz sv:~/pigsty/ ./ --exclude-from 'vagrant/Vagrantfile' --exclude-from 'vagrant/.vagrant' +ss: + rsync -avz --exclude=temp --exclude=dist --exclude=vagrant --exclude=terraform --delete ./ ai:/data/pigsty/ +gsync: + rsync -avz --delete .git/ sv:/data/pgsty/pigsty/.git/ +gpull: + rsync -avz --delete sv:/data/pgsty/pigsty/.git/ .git/ +grestore: + git restore pigsty.yml + git restore vagrant/Vagrantfile +gpush: + git push origin main +############################################################### + + + +############################################################### +# 8. Release # +############################################################### +# make pigsty release (source code tarball) +r: release +release: + bin/release ${VERSION} + +u: upload-src +upload-src: + cp -f dist/$(VERSION)/$(SRC_PKG) ~/pgsty/repo/src/ + cd ~/pgsty/repo && make up-src + +rd: release-dba +release-dba: + @mkdir -p dist/$(VERSION) + tar -czf dist/$(VERSION)/$(DBA_PKG) .claude CLAUDE.md + @echo "Created: dist/$(VERSION)/$(DBA_PKG)" + +gd: get-dba +get-dba: + curl -fsSL "https://repo.pigsty.cc/dba/$(DBA_PKG)" | tar -xzf - + @echo "DBA package extracted to current directory" + +ud: upload-dba +upload-dba: + cp -f dist/$(VERSION)/$(DBA_PKG) ~/pgsty/repo/dba/ + cd ~/pgsty/repo && make up-dba + @echo "Uploaded: $(DBA_PKG)" + + +############################################################### +# 9. Environment # +############################################################### + +#------------------------------# +# Terraform # +#------------------------------# +tu: # terraform up + cd terraform && make u + cd terraform && make ssh +td: # terraform destroy + cd terraform && make d +ts: # terraform ssh + cd terraform && make ssh +to: # terraform output + cd terraform && make out + +#------------------------------# +# Change Configuration # +#------------------------------# +cmeta: + ./configure -s -c meta +cdual: + ./configure -s -c ha/dual +ctrio: + ./configure -s -c ha/trio +cfull: + ./configure -s -c ha/full +csimu: + ./configure -s -c ha/simu +coss: + cp conf/build/oss.yml pigsty.yml +cpro: + cp conf/build/pro.yml pigsty.yml + +#------------------------------# +# Building Environment # +#------------------------------# +oss: coss del vo new ssh copy-src-oss dfx +pro: cpro del vp new ssh dfx +all: del va new ssh dfx +vo: # oss building environment + vagrant/config oss +vp: # pro building environment + vagrant/config pro +vr: # rpm building environment + vagrant/config rpm +vd: # deb building environment + vagrant/config deb +va: # all building environment + vagrant/config all +boot-pkg: + bin/boot $(VERSION) + +#------------------------------# +# meta, single node, the devbox +#------------------------------# +# simple 1-node devbox for quick setup, demonstration, and development + +meta: meta9 +meta8: cmeta del vmeta8 up ssh #copy-el8 use-pkg +meta9: cmeta del vmeta9 up ssh #copy-el9 use-pkg +meta12: cmeta del vmeta12 up ssh #copy-d12 use-pkg +meta22: cmeta del vmeta22 up ssh #copy-u22 use-pkg +meta24: cmeta del vmeta24 up ssh #use-pkg + +vm: vmeta +vmeta: + vagrant/config meta +vmeta8: + vagrant/config meta el8 +vmeta9: + vagrant/config meta el9 +vmeta12: + vagrant/config meta debian12 +vmeta22: + vagrant/config meta ubuntu22 +vmeta24: + vagrant/config meta ubuntu24 + +#------------------------------# +# full, four nodes, the sandbox +#------------------------------# +# full-featured 4-node sandbox for HA-testing & tutorial & practices + +full: full9 +full8: cfull del vfull8 up ssh #copy-el8 use-pkg +full9: cfull del vfull9 up ssh #copy-el9 use-pkg +full12: cfull del vfull12 up ssh #copy-d12 use-pkg +full22: cfull del vfull22 up ssh #copy-u22 use-pkg +full24: cfull del vfull24 up ssh #copy-u24 use-pkg + +vf: vfull +vfull: + vagrant/config full +vfull8: + vagrant/config full el8 +vfull9: + vagrant/config full el9 +vfull12: + vagrant/config full debian12 +vfull22: + vagrant/config full ubuntu22 +vfull24: + vagrant/config full ubuntu24 + +#------------------------------# +# simu, 20 nodes, the simubox +#------------------------------# +# complex 20-node simubox for production simulation & complete testing +simu-conf: + ./configure -s -c ha/simu +vsimu: + vagrant/config simu +vsimu8: + vagrant/config simu el8 +vsimu9: + vagrant/config simu el9 +vsimu12: + vagrant/config simu debian12 +vsimu22: + vagrant/config simu ubuntu22 +vsimu24: + vagrant/config simu ubuntu24 + +vs: simu +simu: simu9 +simu8: csimu del vsimu8 new ssh + scp dist/${VERSION}/$(PKG)pigsty-pkg-${VERSION}.el8.${ARCH}.tgz 10.10.10.10:/tmp/pkg.tgz ; ssh 10.10.10.10 'sudo mkdir -p /www; sudo tar -xf /tmp/pkg.tgz -C /www' +simu9: csimu del vsimu9 new ssh + scp dist/${VERSION}/$(PKG)pigsty-pkg-${VERSION}.el9.${ARCH}.tgz 10.10.10.10:/tmp/pkg.tgz ; ssh 10.10.10.10 'sudo mkdir -p /www; sudo tar -xf /tmp/pkg.tgz -C /www' +simu12: csimu del vsimu12 new ssh + scp dist/${VERSION}/$(PKG)pigsty-pkg-${VERSION}.d12.${ARCH}.tgz 10.10.10.10:/tmp/pkg.tgz ; ssh 10.10.10.10 'sudo mkdir -p /www; sudo tar -xf /tmp/pkg.tgz -C /www' +simu22: csimu del vsimu22 new ssh + scp dist/${VERSION}/$(PKG)pigsty-pkg-${VERSION}.u22.${ARCH}.tgz 10.10.10.10:/tmp/pkg.tgz ; ssh 10.10.10.10 'sudo mkdir -p /www; sudo tar -xf /tmp/pkg.tgz -C /www' +simu24: csimu del vsimu24 new ssh + scp dist/${VERSION}/$(PKG)pigsty-pkg-${VERSION}.u24.${ARCH}.tgz 10.10.10.10:/tmp/pkg.tgz ; ssh 10.10.10.10 'sudo mkdir -p /www; sudo tar -xf /tmp/pkg.tgz -C /www' +rs: + rsync -avz --exclude=vagrant ./ ai:~/pigsty/ +############################################################### + + + + +############################################################### +# Inventory # +############################################################### +.PHONY: default tip link doc boot bootstrap conf configure deploy \ + src \ + c v validate \ + infra repo repo-upstream repo-check repo-build repo-add repo-clean \ + node-repo node-upstream node-pkg redeploy \ + grafana vv vlogs vector nginx cert claude cc docker app \ + pgsql-add pgsql-rm pgsql-ext \ + deps dns start ssh tssh \ + up dw del nuke new clean \ + up-test dw-test del-test new-test \ + st status suspend resume \ + ri rc rw ro rh test-ri test-rc test-rw test-ro test-rw2 test-ro2 test-rh test-st test-rb1 test-rb2 test-rb3 \ + di dd dc du dashboard-init dashboard-dump dashboard-clean \ + copy cc copy-src copy-pkg copy-el8 copy-el9 copy-el10 copy-d12 copy-d13 copy-u22 copy-u24 copy-app copy-all \ + use-src use-pkg use-all cmdb \ + cso copy-src-oss copy-src-pro csr copy-src-rpm csd copy-src-deb dfx deb-fix \ + push pull ss gsync gpull grestore gpush \ + r release u upload-src \ + tu td ts to \ + cmeta cdual ctrio cfull csimu coss cpro \ + oss pro all vo vp vr vd va boot-pkg \ + meta meta8 meta9 meta12 meta22 meta24 \ + vm vmeta vmeta8 vmeta9 vmeta12 vmeta22 vmeta24 \ + full full8 full9 full12 full22 full24 \ + vf vfull vfull8 vfull9 vfull12 vfull22 vfull24 \ + simu-conf vsimu vsimu8 vsimu9 vsimu12 vsimu22 vsimu24 \ + vs simu simu8 simu9 simu12 simu22 simu24 rs \ + rd release-dba gd get-dba ud upload-dba + +############################################################### diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..737b045 --- /dev/null +++ b/NOTICE @@ -0,0 +1,8 @@ +PIGSTY - Battery-Included PostgreSQL Distribution + +Copyright 2018-2025 Vonng (Feng Ruohang) + +Project Homepage: https://pigsty.io / https://pigsty.cc +Source Repository: https://github.com/pgsty/pigsty + +Pigsty® and PGSTY® are trademarks of Ruohang Feng. diff --git a/README.md b/README.md index 591c24a..e92e8fa 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,663 @@ -这是针对 VictoriaMetrics 全家桶 (Metrics, Logs, Traces) 配合 OpenTelemetry 的 GitHub Description 和 README。这个架构的核心优势是：极致的高性能和低资源占用（相比 Prometheus/Loki/Elasticsearch）。1. GitHub Description (Repository Description)🏆 专业解决方案版 (The Professional Solution)English:"High-performance, unified observability solution combining OpenTelemetry for collection with the VictoriaMetrics Ecosystem (Metrics, Logs, Traces) and Grafana."中文:"基于 VictoriaMetrics 全家桶 (Metrics, Logs, Traces) 和 OpenTelemetry 的高性能统一可观测性解决方案，集成 Grafana 可视化。"🏷 推荐 Topics (Tags)observability victoriametrics victorialogs opentelemetry grafana high-performance monitoring vmetrics vlogs2. README.md此 README 强调了 VictoriaMetrics 体系的简单性和效率。Markdown# High-Performance Observability Stack (VictoriaMetrics + OTel) +# PIGSTY - Self-Hosting PostgreSQL like a Pro -[![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -[![Docker](https://img.shields.io/badge/docker-compose-ready-green.svg)](docker-compose.yml) -[![VictoriaMetrics](https://img.shields.io/badge/Powered_by-VictoriaMetrics-red.svg)](https://victoriametrics.com/) +[![Webite: pigsty.io](https://img.shields.io/badge/website-pigsty.io-slategray?style=flat&logo=cilium&logoColor=white)](https://pigsty.io) +[![网站: pigsty.cc](https://img.shields.io/badge/网站-pigsty.cc-slategray?style=flat&logo=cilium&logoColor=white)](https://pigsty.cc) +[![Version: v4.0.0](https://img.shields.io/badge/version-v4.0.0-slategray?style=flat&logo=cilium&logoColor=white)](https://github.com/pgsty/pigsty/releases/tag/v4.0.0) +[![License: Apache-2.0](https://img.shields.io/github/license/pgsty/pigsty?logo=opensourceinitiative&logoColor=green&color=slategray)](https://pigsty.io/docs/about/license/) +[![GitHub Stars](https://img.shields.io/github/stars/pgsty/pigsty?style=flat&logo=github&logoColor=black&color=slategray)](https://star-history.com/#pgsty/pigsty&Date) +[![Extensions: 444](https://img.shields.io/badge/extensions-444-%233E668F?style=flat&logo=postgresql&logoColor=white&labelColor=3E668F)](https://pgext.cloud/list) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/pgsty/pigsty) -> **Next-generation observability solution** engineered for efficiency. Leveraging **OpenTelemetry** for unified data collection and the **VictoriaMetrics ecosystem** for high-speed storage of Metrics, Logs, and Traces. +Battery-Included, Local-First **PostgreSQL** Distribution as a Free & Better **RDS** Alternative! -## ⚡ Why This Stack? +> "**P**ostgreSQL **I**n **G**reat **STY**le": **P**ostgres, **I**nfras, **G**raphics, **S**ervice, **T**oolbox, it's all **Y**ours. -Moving away from traditional stacks (Prometheus/Loki/ELK), this solution focuses on: -* **🚀 High Performance:** Handling millions of data points with minimal latency. -* **📉 Low RAM Usage:** Significantly lower resource footprint compared to Prometheus or Java-based stacks. -* **🧩 Unified Architecture:** Single ecosystem for all observability signals. +[Website](https://pigsty.io) | [ORG](https://github.com/pgsty) | [PGEXT.CLOUD](https://pgext.cloud) | [AI INFRA](https://pigsty.io/docs/piglet) | [Public Demo](https://demo.pigsty.io) | [Blog](https://pigsty.io/blog/) | [Discuss](https://github.com/orgs/pgsty/discussions) | [Support](https://pigsty.io/price) | [中文站](https://pigsty.cc/) | [博客](https://vonng.com) -## 🏗 Architecture +[![banner](https://pigsty.io/img/pigsty/banner.png)](https://pigsty.io) -Data flows from your application via OTel Agent to the storage backends, visualized centrally in Grafana. +Pigsty scales from [**laptop**](https://pigsty.io/docs/setup) to [**datacenter**](https://pigsty.io/docs/deploy). Enterprise-grade. Developer-friendly. [**Get Started**](https://pigsty.io/docs/setup/install) with the latest [**v4.0.0**](https://github.com/pgsty/pigsty/releases/tag/v4.0.0-c1): -```mermaid -flowchart LR - App[Your Application] -->|OTLP| OTel[OTel Collector] - - subgraph Storage [VictoriaMetrics Ecosystem] - OTel -->|Metrics & Traces| VM[VictoriaMetrics] - OTel -->|Logs| VL[VictoriaLogs] - end - - Grafana -->|PromQL| VM - Grafana -->|LogQL-like| VL -🛠 ComponentsComponentRoleDescriptionOpenTelemetryCollectorUniversal agent for Metrics, Logs, and Traces.VictoriaMetricsMetrics & TracesLong-term storage for time-series and trace data. Drop-in replacement for Prometheus.VictoriaLogsLogsHigh-performance, resource-efficient log database.GrafanaVisualizationUnified dashboard for all data sources.🚀 Quick StartPrerequisitesDocker & Docker ComposeDeploymentClone the repo:Bashgit clone [https://github.com/your-username/vm-otel-stack.git](https://github.com/your-username/vm-otel-stack.git) -cd vm-otel-stack -Start services:Bashdocker-compose up -d -Access Dashboards:Grafana: http://localhost:3000 (Default: admin/admin)VictoriaMetrics UI: http://localhost:8428VictoriaLogs UI: http://localhost:9428⚙️ Configuration & PortsServiceInternal PortExternal PortFunctionVictoriaMetrics84288428Metrics Ingest & Query (PromQL)VictoriaLogs94289428Logs Ingest & QueryOTel Receiver43174317OTLP gRPCOTel Receiver43184318OTLP HTTPGrafana30003000UI📊 Data Source Setup (Grafana)When configuring Grafana, use the following settings:Metrics (Prometheus Type):URL: http://victoriametrics:8428Logs (VictoriaLogs Plugin):URL: http://victorialogs:9428🤝 ContributingPRs regarding performance tuning and dashboard examples are highly welcome!📄 LicenseMIT © [Your Name] -### 💡 关键调整说明 +```bash +curl -fsSL https://repo.pigsty.io/get | bash -s v4.0.0 +``` -1. **Highlight Performance (强调性能):** VictoriaMetrics 最大的卖点就是比 Prometheus 省内存、速度快，所以我在 Introduction 部分专门加了 "Why This Stack" 章节。 -2. **Mermaid 结构:** 流程图中清晰地画出了 OTel 分流，一部分去 VM (Metrics/Traces)，一部分去 VL (Logs)。 -3. **Data Source 提示:** 在 README 里加了一小段 Grafana 配置提示，因为 VictoriaLogs 在 Grafana 里的配置方式和 Loki 不太一样（通常需要专门的插件或配置方式），这是一个贴心的细节。 -需要我为您生成对应的 `docker-compose.yml` 吗？VictoriaLogs 目前比较新，配置起来比 Loki 简单，但端口和镜像需要注意版本。 +## Features + +- **Extensible**: **444** [**PG Extensions**](https://pgext.cloud/list) & **9** [**PG Kernel**](https://pigsty.io/docs/pgsql/kernel) available, with [**MsSQL**](https://pigsty.io/docs/pgsql/kernel/babelfish), [**Oracle**](https://pigsty.io/docs/pgsql/kernel/ivorysql), [**MySQL**](https://pigsty.io/docs/pgsql/kernel/openhalo), [**Mongo**](https://pigsty.io/docs/ferret) compatibility. +- **Reliable**: Self-healing [**HA**](https://pigsty.io/docs/concept/ha) clusters with pre-configured [**PITR**](https://pigsty.io/docs/pgsql/backup) and built-in [**ACL**](https://pigsty.io/docs/pgsql/config/acl), [**CA & SSL**](https://pigsty.io/docs/infra/admin/cert) secure best practice. +- **Observable**: SOTA monitoring for [**PG**](https://demo.pigsty.io/ui/d/pgrds-instance/pgrds-instance) / [**Infra**](https://pigsty.io/docs/infra) / [**Node**](https://pigsty.io/docs/node) based on **Victoria** & **Grafana** stack: [**Demo**](https://demo.pigsty.io) & [**Gallery**](https://github.com/pgsty/pigsty/wiki/Gallery). +- **Scalable**: Auto-routed & pooled customizable database [**Services**](https://pigsty.io/docs/pgsql/service) [**Access**](https://pigsty.io/docs/pgsql/service#access-service) with **haproxy**, **pgbouncer**, and **VIP**. +- **Maintainable**: [**One-Cmd Deploy**](https://pigsty.io/docs/setup/install), [**Admin SOP**](https://pigsty.io/docs/pgsql/admin), **Auto-Tune**, **Local Repo**, [**IaC**](https://pigsty.io/docs/pgsql/config) and [**Vagrant**](https://pigsty.io/docs/deploy/vagrant) / [**Terraform**](https://pigsty.io/docs/deploy/terraform) support. +- **Composable**: Bonus [**Modules**](https://pigsty.io/docs/ref/module) such as [**Redis**](https://pigsty.io/docs/redis), [**MinIO**](https://pigsty.io/docs/minio), [**Etcd**](https://pigsty.io/docs/etcd), [**Docker**](https://pigsty.io/docs/app), [**DuckDB**](https://pigsty.io/docs/pilot/duckdb), [**FerretDB**](https://pigsty.io/docs/ferret), [**Supabase**](https://pigsty.io/docs/app/supabase), [**& More**](https://pigsty.io/docs/ref/module)! +- **Controllable**: Self-sufficient [**Infra Closure**](https://pigsty.io/docs/infra), Cloud Neutral FOSS. Run on [**bare Linux**](https://pigsty.io/docs/ref/linux) without [**Docker & K8S**](https://vonng.com/en/db/db-in-k8s/)! +- **Affordable**: Self-Hosting like a [**Pro**](https://pigsty.io/price/), without the cost of [**RDS**](https://vonng.com/en/cloud/rds/). Empowering Cloud-Exit to [**save lot more!**](https://pigsty.io/docs/about/compare/) + +You can even use exotic [**PG kernel forks**](https://pigsty.io/docs/pgsql/kernel) as an in-place replacement and wrap it as a full RDS service: + +| Kernel | Key Feature | Description | +|------------------------------------------------------------|:--------------------------------|------------------------------------------------| +| [PostgreSQL](https://pigsty.io/docs/pgsql/kernel/postgres) | **Extension Overwhelming** | Vanilla PostgreSQL with 444 extensions | +| [Citus](https://pigsty.io/docs/pgsql/kernel/citus) | **Horizontal Scaling** | Distributive PostgreSQL via native extension | +| [WiltonDB](https://pigsty.io/docs/pgsql/kernel/babelfish) | **SQL Server Migration** | Microsoft SQL Server wire-compatibility | +| [IvorySQL](https://pigsty.io/docs/pgsql/kernel/ivorysql) | **Oracle Migration** | Oracle Grammar and PL/SQL compatible | +| [OpenHalo](https://pigsty.io/docs/pgsql/kernel/openhalo) | **MySQL Migration** | MySQL wire-protocol compatibility | +| [Percona](https://pigsty.io/docs/pgsql/kernel/percona) | **Transparent Data Encryption** | Percona Distribution with pg_tde | +| [FerretDB](https://pigsty.io/docs/ferret) | **MongoDB Migration** | MongoDB wire-protocol compatibility | +| [OrioleDB](https://pigsty.io/docs/pgsql/kernel/orioledbdb) | **OLTP Optimization** | No bloat, No XID Wraparound, S3 Storage | +| [PolarDB](https://pigsty.io/docs/pgsql/kernel/polardb) | **Aurora flavor RAC** | RAC, China domestic compliance | +| [Supabase](https://pigsty.io/docs/app/supabase) | **Backend as Service** | BaaS based on PostgreSQL, Firebase alternative | + +And gather the synergistic superpowers of all [**444+ PostgreSQL Extensions**](https://pgext.cloud/list) all together: + +[![ecosystem](https://github.com/user-attachments/assets/c952441e-5ff7-4acb-aace-dd3021d28622)](https://pgext.cloud) + + +## Get Started + +[![Postgres: 18.1](https://img.shields.io/badge/PostgreSQL-18.1-%233E668F?style=flat&logo=postgresql&labelColor=3E668F&logoColor=white)](https://pigsty.io/docs/pgsql) +[![Linux](https://img.shields.io/badge/Linux-AMD64-%23FCC624?style=flat&logo=linux&labelColor=FCC624&logoColor=black)](https://pigsty.io/docs/node) +[![Linux](https://img.shields.io/badge/Linux-ARM64-%23FCC624?style=flat&logo=linux&labelColor=FCC624&logoColor=black)](https://pigsty.io/docs/node) +[![EL Support: 8/9/10](https://img.shields.io/badge/EL-8/9/10-red?style=flat&logo=redhat&logoColor=red)](https://pigsty.io/docs/ref/linux#el) +[![Debian Support: 12/13](https://img.shields.io/badge/Debian-12/13-%23A81D33?style=flat&logo=debian&logoColor=%23A81D33)](https://pigsty.io/docs/ref/linux#debian) +[![Ubuntu Support: 22/24](https://img.shields.io/badge/Ubuntu-22/24-%23E95420?style=flat&logo=ubuntu&logoColor=%23E95420)](https://pigsty.io/docs/ref/linux#ubuntu) +[![Docker Image](https://img.shields.io/badge/Docker-v4.0.0-%232496ED?style=flat&logo=docker&logoColor=white)](https://pigsty.io/docs/setup/docker) + +[**Prepare**](https://pigsty.io/docs/deploy/prepare) a fresh `x86_64` / `aarch64` node runs any [**compatible**](https://pigsty.io/docs/ref/linux) **Linux** OS Distros, then [**Install**](https://pigsty.io/docs/setup/install#install) **Pigsty** with: + +```bash +curl -fsSL https://repo.pigsty.io/get | bash; cd ~/pigsty; +``` + +Then [**configure**](https://pigsty.io/docs/concept/iac/configure) and run the [**`deploy.yml`**](https://pigsty.io/docs/setup/playbook) playbook with an [**admin user**](https://pigsty.io/docs/deploy/admin) (**nopass** `ssh` & `sudo`): + +```bash +./configure -g # generate config and random passwords +./deploy.yml # deploy everything on current node +``` + +Finally, you will get a pigsty [**singleton node ready**](https://pigsty.io/docs/setup/install), with [**WebUI**](https://pigsty.io/docs/setup/webui) on port `80/443` and [**Postgres**](https://pigsty.io/docs/setup/pgsql) on port `5432`. + +For dev/testing purposes, you can also run Pigsty inside [**Docker**](https://pigsty.io/docs/setup/docker) containers: `cd docker; make launch` + +-------- + +> [**Single-Node Setup**](https://pigsty.io/docs/setup/install) | [**Production Deploy**](https://pigsty.io/docs/deploy) | [**Offline Install**](https://pigsty.io/docs/setup/offline) | [**Minimal Install**](https://pigsty.io/docs/setup/slim) | [**Docker Install**](https://pigsty.io/docs/setup/docker) | [**Run Supabase**](https://pigsty.io/docs/app/supabase) + +
Install with the pig cli
+ +Then you can launch pigsty with `pig sty` sub command: + +```bash +curl -fsSL https://repo.pigsty.io/pig | bash # install pig +pig sty init # install latest pigsty src to ~/pigsty +pig sty conf # auto-generate pigsty.yml config file +pig sty deploy # run the deploy.yml playbook +``` + +
+ + +
Install with get script
+ +```bash +[root@pg-meta ~]# curl -fsSL https://repo.pigsty.io/get | bash -s v4.0.0 +[v4.0.0] =========================================== +$ curl -fsSL https://repo.pigsty.io/get | bash +[Docs] https://pigsty.io/docs +[Demo] https://demo.pigsty.io +[Repo] https://github.com/pgsty/pigsty +[Download] =========================================== +[ OK ] version = v4.0.0 (from arg) +curl -fSL https://repo.pigsty.io/src/pigsty-v4.0.0.tgz -o /tmp/pigsty-v4.0.0.tgz +######################################################################## 100.0% +[ OK ] md5sums = 53cb5980f999f661fbb832d7ee2fc93a /tmp/pigsty-v4.0.0.tgz +[Install] =========================================== +[WARN] os user = root , it's recommended to use a non-root sudo-able admin +[ OK ] install = /root/pigsty, from /tmp/pigsty-v4.0.0.tgz + +[Bootstrap] =========================================== +[WARN] ansible = not found, bootstrap +bootstrap pigsty v4.0.0 begin +[ OK ] region = china +[ OK ] kernel = Linux +[ OK ] machine = x86_64 +[ OK ] package = rpm,dnf +[ OK ] vendor = rocky (Rocky Linux) +[ OK ] version = 10 (10.0) +[ OK ] sudo = root ok +[WARN] ssh = root@127.0.0.1 fixed +[WARN] old repos = moved to /etc/yum.repos.d/backup +[ OK ] repo file = add el10.x86_64 china upstream +[WARN] rpm cache = updating, may take a while +Pigsty PGSQL 10 - x86_64 364 kB/s | 251 kB 00:00 +EL 10 BaseOS 10 - x86_64 32 MB/s | 6.4 MB 00:00 +EL 10 AppStream 10 - x86_64 11 MB/s | 2.1 MB 00:00 +EL 10 CRB 10 - x86_64 1.8 MB/s | 492 kB 00:00 +EL 10 EPEL 10.0 - x86_64 27 MB/s | 4.8 MB 00:00 +Metadata cache created. +[ OK ] repo cache = created +[ OK ] install el10 utils +Last metadata expiration check: 0:00:02 ago on Wed 07 Jan 2026 05:58:22 PM CST. +..... + +Installed: + ansible-2.16.14-1.el10.noarch ansible-collection-ansible-posix-2.0.0-1.el10_0.noarch ansible-collection-community-crypto-2.15.0-1PIGSTY.el10.noarch ansible-collection-community-general-10.2.0-1.el10_0.noarch + ansible-core-1:2.16.14-1.el10.noarch git-core-2.47.3-1.el10.x86_64 python3-cffi-1.16.0-7.el10.x86_64 python3-cryptography-43.0.0-4.el10.x86_64 + python3-jmespath-1.0.1-8.el10.noarch python3-ply-3.11-25.el10.noarch python3-pycparser-2.20-16.el10.noarch python3-resolvelib-1.0.1-6.el10.noarch + +Complete! +[ OK ] ansible = ansible [core 2.16.14] +[ OK ] boostrap pigsty complete +proceed with ./configure +``` + +> HINT: To install a specific version, pass the version string as the first parameter: +> +> ```bash +> curl -fsSL https://repo.pigsty.io/get | bash -s v4.0.0 +> ``` + +
+ + +
Clone src with git
+ +You can also download the pigsty source with `git`, remember to check out a specific version tag, the `main` branch is for development. + +```bash +git clone https://github.com/pgsty/pigsty; cd pigsty; git checkout v4.0.0 +``` + +
+ + + + +## Architecture + +Pigsty uses a [**modular**](https://pigsty.io/docs/concept/arch) design. you can [**use one or all**](https://pigsty.io/docs/deploy/planning), Best of breed products. Integrated as a platform. + +[![board](https://pigsty.io/img/pigsty/motherboard.gif)](https://pigsty.io/docs/concept/arch) + +[![PGSQL](https://img.shields.io/badge/PGSQL-%233E668F?style=flat&logo=postgresql&labelColor=3E668F&logoColor=white)](https://pigsty.io/docs/pgsql) Self-healing PostgreSQL HA cluster powered by Patroni, Pgbouncer, PgBackrest & HAProxy + +[![INFRA](https://img.shields.io/badge/INFRA-%23009639?style=flat&logo=nginx&labelColor=009639&logoColor=white)](https://pigsty.io/docs/infra) Nginx, Local Repo, DNSMasq, and the entire Victoria & Grafana observability stack. + +[![NODE](https://img.shields.io/badge/NODE-%23FCC624?style=flat&logo=linux&labelColor=FCC624&logoColor=black)](https://pigsty.io/docs/node) Init node name, repo, pkg, NTP, ssh, admin, tune, expose services, collect logs & metrics. + +[![ETCD](https://img.shields.io/badge/ETCD-%23419EDA?style=flat&logo=etcd&labelColor=419EDA&logoColor=white)](https://pigsty.io/docs/etcd) Etcd cluster is used as a reliable distributive configuration store by PostgreSQL HA Agents. + +You can compose them freely in a declarative manner. `INFRA` & `NODE` will suffice for host monitoring. +`ETCD` and `PGSQL` are used for HA PG clusters; Installing them on multiple nodes automatically forms HA clusters. + +The default [`deploy.yml`](https://github.com/pgsty/pigsty/blob/main/deploy.yml) playbook will deploy `INFRA`, `NODE`, `ETCD` & `PGSQL` on the current node. +Which gives you an out-of-the-box PostgreSQL singleton instance (`admin_ip:5432`) with everything ready. + +[![pigsty-arch](https://pigsty.io/img/pigsty/arch.png)](https://pigsty.io/docs/concept/arch) + +The node can be used as an admin controller to deploy & monitor more nodes & clusters. For example, you can install these **6** **OPTIONAL** [extra modules](https://pigsty.io/docs/ref/module#extra-modules) for advanced use cases: + +[![MinIO](https://img.shields.io/badge/MINIO-%23C72E49?style=flat&logo=minio&logoColor=white)](https://pigsty.io/docs/minio) S3-compatible object storage service; used as an optional central backup server for `PGSQL`. + +[![Redis](https://img.shields.io/badge/REDIS-%23FF4438?style=flat&logo=redis&logoColor=white)](https://pigsty.io/docs/infra) Deploy Redis servers in standalone master-replica, sentinel, and native cluster mode. + +[![Ferret](https://img.shields.io/badge/FERRET-%23042133?style=flat&logo=ferretdb&logoColor=white)](https://pigsty.io/docs/ferret) Native support for FerretDB — adding MongoDB wire protocol compatibility to Postgres! + +[![Docker](https://img.shields.io/badge/DOCKER-%232496ED?style=flat&logo=docker&logoColor=white)](https://pigsty.io/docs/docker) Launch optional docker daemons to run other stateless parts besides Pigsty RDS. + +[![Juice](https://img.shields.io/badge/JUICE-%2300C853?style=flat&logo=infinityfree&logoColor=white)](https://pigsty.io/docs/juice) JuiceFS can mount S3/MinIO, and even PostgreSQL as a filesystem shared by multi users. + +[![Vibe](https://img.shields.io/badge/VIBE-%23FF6B35?style=flat&logo=claude&logoColor=white)](https://pigsty.io/docs/vibe) Vibe coding environment with VS Code Server, JupyterLab, Node.js, and Claude Code. + +Of course, you can deploy different kinds of HA **PostgreSQL** clusters on multiple nodes, as much as you want. + + +---------------- + +## PostgreSQL RDS + +To deploy an [**additional**](https://pigsty.io/docs/deploy) 3-node HA Postgres cluster `pg-test`. Add the cluster [**definition**](https://github.com/pgsty/pigsty/blob/main/conf/ha/full.yml#L66) to the [**config inventory**](https://pigsty.io/docs/concept/iac/inventory): + +```yaml +pg-test: + hosts: + 10.10.10.11: { pg_seq: 1, pg_role: primary } + 10.10.10.12: { pg_seq: 2, pg_role: replica } + 10.10.10.13: { pg_seq: 3, pg_role: offline } + vars: { pg_cluster: pg-test } +``` + +The default config file is [`pigsty.yml`](https://github.com/pgsty/pigsty/blob/main/pigsty.yml) under pigsty home, add the snippet above to the `all.children.pg-test`, +Then, create the cluster with built-in playbooks in one command: + +```bash +bin/pgsql-add pg-test # init pg-test cluster +``` + +
Example: Complex PostgreSQL Customization
+ +This config file provides a detailed example of a complex PostgreSQL cluster `pg-meta` with multiple +[databases](https://pigsty.io/docs/pgsql/config/db), [users](https://pigsty.io/docs/pgsql/config/user), and [service](https://pigsty.io/docs/pgsql/service) definition: + +```yaml +pg-meta: + hosts: { 10.10.10.10: { pg_seq: 1, pg_role: primary , pg_offline_query: true } } + vars: + pg_cluster: pg-meta + pg_databases: # define business databases on this cluster, array of database definition + - name: meta # REQUIRED, `name` is the only mandatory field of a database definition + baseline: cmdb.sql # optional, database sql baseline path, (relative path among ansible search path, e.g files/) + pgbouncer: true # optional, add this database to pgbouncer database list? true by default + schemas: [pigsty] # optional, additional schemas to be created, array of schema names + extensions: # optional, additional extensions to be installed: array of `{name[,schema]}` + - { name: postgis , schema: public } + - { name: timescaledb } + comment: pigsty meta database # optional, comment string for this database + owner: postgres # optional, database owner, postgres by default + template: template1 # optional, which template to use, template1 by default + encoding: UTF8 # optional, database encoding, UTF8 by default. (MUST same as template database) + locale: C # optional, database locale, C by default. (MUST same as template database) + lc_collate: C # optional, database collate, C by default. (MUST same as template database) + lc_ctype: C # optional, database ctype, C by default. (MUST same as template database) + tablespace: pg_default # optional, default tablespace, 'pg_default' by default. + allowconn: true # optional, allow connection, true by default. false will disable connect at all + revokeconn: false # optional, revoke public connection privilege. false by default. (leave connect with grant option to owner) + register_datasource: true # optional, register this database to grafana datasources? true by default + connlimit: -1 # optional, database connection limit, default -1 disable limit + pool_auth_user: dbuser_meta # optional, all connection to this pgbouncer database will be authenticated by this user + pool_mode: transaction # optional, pgbouncer pool mode at database level, default transaction + pool_size: 64 # optional, pgbouncer pool size at database level, default 64 + pool_size_reserve: 32 # optional, pgbouncer pool size reserve at database level, default 32 + pool_size_min: 0 # optional, pgbouncer pool size min at database level, default 0 + pool_max_db_conn: 100 # optional, max database connections at database level, default 100 + # shared parameters for CREATE DATABASE (shared with PostgreSQL) + is_template: false # optional, mark this database as template? default false + strategy: wal_log # optional (shared), PostgreSQL 15+, wal_log or file_copy + locale_provider: libc # optional (shared), PostgreSQL 15+, icu or libc + icu_locale: '' # optional (shared), PostgreSQL 15+, ICU locale + icu_rules: '' # optional (shared), PostgreSQL 16+, ICU rules + builtin_locale: '' # optional (shared), PostgreSQL 17+, builtin locale + parameters: {} # optional, database-level parameters with `ALTER DATABASE SET` + - { name: grafana ,owner: dbuser_grafana ,revokeconn: true ,comment: grafana primary database } + - { name: bytebase ,owner: dbuser_bytebase ,revokeconn: true ,comment: bytebase primary database } + - { name: kong ,owner: dbuser_kong ,revokeconn: true ,comment: kong the api gateway database } + - { name: gitea ,owner: dbuser_gitea ,revokeconn: true ,comment: gitea meta database } + - { name: wiki ,owner: dbuser_wiki ,revokeconn: true ,comment: wiki meta database } + pg_users: # define business users/roles on this cluster, array of user definition + - name: dbuser_meta # REQUIRED, `name` is the only mandatory field of a user definition + password: DBUser.Meta # optional, password, can be a scram-sha-256 hash string or plain text + login: true # optional, can log in, true by default (new biz ROLE should be false) + superuser: false # optional, is superuser? false by default + createdb: false # optional, can create database? false by default + createrole: false # optional, can create role? false by default + inherit: true # optional, can this role use inherited privileges? true by default + replication: false # optional, can this role do replication? false by default + bypassrls: false # optional, can this role bypass row level security? false by default + pgbouncer: true # optional, add this user to pgbouncer user-list? false by default (production user should be true explicitly) + connlimit: -1 # optional, user connection limit, default -1 disable limit + expire_in: 3650 # optional, now + n days when this role is expired (OVERWRITE expire_at) + expire_at: '2030-12-31' # optional, YYYY-MM-DD 'timestamp' when this role is expired (OVERWRITTEN by expire_in) + comment: pigsty admin user # optional, comment string for this user/role + roles: [dbrole_admin] # optional, belonged roles. default roles are: dbrole_{admin,readonly,readwrite,offline} + parameters: {} # optional, role level parameters with `ALTER ROLE SET` + pool_mode: transaction # optional, pgbouncer pool mode at user level, transaction by default + pool_connlimit: -1 # optional, max database connections at user level, default -1 disable limit + # shared parameters for CREATE ROLE (shared with PostgreSQL) + # roles can also be defined in extended format for fine-grained control (PostgreSQL 16+) + # roles: + # - dbrole_readwrite # simple format: just grant the role + # - name: dbrole_admin # extended format with options + # state: grant # grant (default) | revoke/absent + # admin: false # grant with admin option? (default false) + # inherit: true # grant with inherit option? (PG16+, default null) + # set: true # grant with set option? (PG16+, default null) + - {name: dbuser_view ,password: DBUser.Viewer ,pgbouncer: true ,roles: [dbrole_readonly], comment: read-only viewer for meta database} + - {name: dbuser_grafana ,password: DBUser.Grafana ,pgbouncer: true ,roles: [dbrole_admin] ,comment: admin user for grafana database } + - {name: dbuser_bytebase ,password: DBUser.Bytebase ,pgbouncer: true ,roles: [dbrole_admin] ,comment: admin user for bytebase database } + - {name: dbuser_kong ,password: DBUser.Kong ,pgbouncer: true ,roles: [dbrole_admin] ,comment: admin user for kong api gateway } + - {name: dbuser_gitea ,password: DBUser.Gitea ,pgbouncer: true ,roles: [dbrole_admin] ,comment: admin user for gitea service } + - {name: dbuser_wiki ,password: DBUser.Wiki ,pgbouncer: true ,roles: [dbrole_admin] ,comment: admin user for wiki.js service } + pg_services: # extra services in addition to pg_default_services, array of service definition + # standby service will route {ip|name}:5435 to sync replica's pgbouncer (5435->6432 standby) + - name: standby # required, service name, the actual svc name will be prefixed with `pg_cluster`, e.g: pg-meta-standby + port: 5435 # required, service exposed port (work as kubernetes service node port mode) + ip: "*" # optional, service bind ip address, `*` for all ip by default + selector: "[]" # required, service member selector, use JMESPath to filter inventory + dest: default # optional, destination port, default|postgres|pgbouncer|, 'default' by default + check: /sync # optional, health check url path, / by default + backup: "[? pg_role == `primary`]" # backup server selector + maxconn: 3000 # optional, max allowed front-end connection + balance: roundrobin # optional, haproxy load balance algorithm (roundrobin by default, other: leastconn) + #options: 'inter 3s fastinter 1s downinter 5s rise 3 fall 3 on-marked-down shutdown-sessions slowstart 30s maxconn 3000 maxqueue 128 weight 100' + pg_hba_rules: + - {user: dbuser_view , db: all ,addr: infra ,auth: pwd ,title: 'allow grafana dashboard access cmdb from infra nodes'} + pg_vip_enabled: true + pg_vip_address: 10.10.10.2/24 + pg_vip_interface: eth1 + pg_crontab: # make a full backup 1 am everyday + - '00 01 * * * /pg/bin/pg-backup full' + +``` + +[![home](https://pigsty.io/img/pigsty/home.jpg)](https://pigsty.io/img/pigsty/home.jpg) + +
+ +It will create a cluster with everything properly configured: [**High Availability**](https://pigsty.io/docs/concept/ha) powered by patroni & etcd; [**Point-In-Time-Recovery**](https://pigsty.io/docs/concept/pitr) powered by pgBackRest & optional MinIO / S3; +auto-routed, pooled [**Services & Access**](https://pigsty.io/docs/pgsql/service#default-service) pooled by pgBouncer and exposed by haproxy; and out-of-the-box [**Monitoring**](https://pigsty.io/docs/pgsql/monitor/dashboard) & alerting powered by the **`INFRA`** module. + +[![HA PostgreSQL Arch](https://pigsty.io/img/pigsty/ha.png)](https://pigsty.io/docs/concept/ha) + +The cluster keeps serving as long as **ANY** instance survives, with excellent [fault-tolerance](https://pigsty.io/docs/concept/ha/failure) performance: + +> [**RPO = 0**](https://pigsty.io/docs/concept/ha/rpo) on sync mode, **RPO < 1MB** on async mode; **RTO < 2s** on switchover, [**RTO < 30s**](https://pigsty.io/docs/concept/ha/rto) on failover. + + + + +---------------- + +## Customization + +Pigsty is highly customizable, You can describe the entire database and infra deployment with **300+** [**parameters**](https://pigsty.io/docs/concept/iac/inventory) in a single config file and materialize them with one command. +There are many built-in configuration [**templates**](https://pigsty.io/docs/concept/iac/template). + +
Example: Sandbox (4-node) with two PG cluster
+ +The [`conf/full.yml`](https://github.com/pgsty/pigsty/blob/main/conf/full.yml) utilize four nodes to deploy two PostgreSQL clusters `pg-meta` and `pg-test`: + +```yaml +pg-meta: + hosts: { 10.10.10.10: { pg_seq: 1, pg_role: primary } } + vars: + pg_cluster: pg-meta + pg_users: + - {name: dbuser_meta ,password: DBUser.Meta ,pgbouncer: true ,roles: [dbrole_admin] ,comment: pigsty admin user } + - {name: dbuser_view ,password: DBUser.Viewer ,pgbouncer: true ,roles: [dbrole_readonly] ,comment: read-only viewer for meta database } + pg_databases: + - {name: meta ,baseline: cmdb.sql ,comment: pigsty meta database ,schemas: [pigsty]} + pg_hba_rules: + - {user: dbuser_view , db: all ,addr: infra ,auth: pwd ,title: 'allow grafana dashboard access cmdb from infra nodes'} + pg_vip_enabled: true + pg_vip_address: 10.10.10.2/24 + pg_vip_interface: eth1 + +# pgsql 3 node ha cluster: pg-test +pg-test: + hosts: + 10.10.10.11: { pg_seq: 1, pg_role: primary } # primary instance, leader of cluster + 10.10.10.12: { pg_seq: 2, pg_role: replica } # replica instance, follower of leader + 10.10.10.13: { pg_seq: 3, pg_role: replica, pg_offline_query: true } # replica with offline access + vars: + pg_cluster: pg-test # define pgsql cluster name + pg_users: [{ name: test , password: test , pgbouncer: true , roles: [ dbrole_admin ] }] + pg_databases: [{ name: test }] + pg_vip_enabled: true + pg_vip_address: 10.10.10.3/24 + pg_vip_interface: eth1 +``` + +You can even deploy PostgreSQL with different major versions and kernel forks in the same deployment: + +[![kernels](https://pigsty.io/img/pigsty/kernels.jpg)](https://pigsty.io/docs/pgsql/kernel) + +
+ +
Example: Security Setup & Delayed Replica
+ +The following [`conf/safe.yml`](https://github.com/pgsty/pigsty/blob/main/conf/ha/safe.yml) provision a 4-node [security](https://pigsty.io/docs/deploy/security//) enhanced postgres cluster `pg-meta` with a delayed replica `pg-meta-delay`: + +```yaml +pg-meta: # 3 instance postgres cluster `pg-meta` + hosts: + 10.10.10.10: { pg_seq: 1, pg_role: primary } + 10.10.10.11: { pg_seq: 2, pg_role: replica } + 10.10.10.12: { pg_seq: 3, pg_role: replica , pg_offline_query: true } + vars: + pg_cluster: pg-meta + pg_conf: crit.yml + pg_users: + - { name: dbuser_meta , password: DBUser.Meta , pgbouncer: true , roles: [ dbrole_admin ] , comment: pigsty admin user } + - { name: dbuser_view , password: DBUser.Viewer , pgbouncer: true , roles: [ dbrole_readonly ] , comment: read-only viewer for meta database } + pg_databases: + - {name: meta ,baseline: cmdb.sql ,comment: pigsty meta database ,schemas: [pigsty] ,extensions: [{name: postgis, schema: public}, {name: timescaledb}]} + pg_default_service_dest: postgres + pg_services: + - { name: standby ,src_ip: "*" ,port: 5435 , dest: default ,selector: "[]" , backup: "[? pg_role == `primary`]" } + pg_vip_enabled: true + pg_vip_address: 10.10.10.2/24 + pg_vip_interface: eth1 + pg_listen: '${ip},${vip},${lo}' + patroni_ssl_enabled: true + pgbouncer_sslmode: require + pgbackrest_method: minio + pg_libs: 'timescaledb, $libdir/passwordcheck, pg_stat_statements, auto_explain' # add passwordcheck extension to enforce strong password + pg_default_roles: # default roles and users in postgres cluster + - { name: dbrole_readonly ,login: false ,comment: role for global read-only access } + - { name: dbrole_offline ,login: false ,comment: role for restricted read-only access } + - { name: dbrole_readwrite ,login: false ,roles: [dbrole_readonly] ,comment: role for global read-write access } + - { name: dbrole_admin ,login: false ,roles: [pg_monitor, dbrole_readwrite] ,comment: role for object creation } + - { name: postgres ,superuser: true ,expire_in: 7300 ,comment: system superuser } + - { name: replicator ,replication: true ,expire_in: 7300 ,roles: [pg_monitor, dbrole_readonly] ,comment: system replicator } + - { name: dbuser_dba ,superuser: true ,expire_in: 7300 ,roles: [dbrole_admin] ,pgbouncer: true ,pool_mode: session, pool_connlimit: 16 , comment: pgsql admin user } + - { name: dbuser_monitor ,roles: [pg_monitor] ,expire_in: 7300 ,pgbouncer: true ,parameters: {log_min_duration_statement: 1000 } ,pool_mode: session ,pool_connlimit: 8 ,comment: pgsql monitor user } + pg_default_hba_rules: # postgres host-based auth rules by default + - {user: '${dbsu}' ,db: all ,addr: local ,auth: ident ,title: 'dbsu access via local os user ident' } + - {user: '${dbsu}' ,db: replication ,addr: local ,auth: ident ,title: 'dbsu replication from local os ident' } + - {user: '${repl}' ,db: replication ,addr: localhost ,auth: ssl ,title: 'replicator replication from localhost'} + - {user: '${repl}' ,db: replication ,addr: intra ,auth: ssl ,title: 'replicator replication from intranet' } + - {user: '${repl}' ,db: postgres ,addr: intra ,auth: ssl ,title: 'replicator postgres db from intranet' } + - {user: '${monitor}' ,db: all ,addr: localhost ,auth: pwd ,title: 'monitor from localhost with password' } + - {user: '${monitor}' ,db: all ,addr: infra ,auth: ssl ,title: 'monitor from infra host with password'} + - {user: '${admin}' ,db: all ,addr: infra ,auth: ssl ,title: 'admin @ infra nodes with pwd & ssl' } + - {user: '${admin}' ,db: all ,addr: world ,auth: cert ,title: 'admin @ everywhere with ssl & cert' } + - {user: '+dbrole_readonly',db: all ,addr: localhost ,auth: ssl ,title: 'pgbouncer read/write via local socket'} + - {user: '+dbrole_readonly',db: all ,addr: intra ,auth: ssl ,title: 'read/write biz user via password' } + - {user: '+dbrole_offline' ,db: all ,addr: intra ,auth: ssl ,title: 'allow etl offline tasks from intranet'} + pgb_default_hba_rules: # pgbouncer host-based authentication rules + - {user: '${dbsu}' ,db: pgbouncer ,addr: local ,auth: peer ,title: 'dbsu local admin access with os ident'} + - {user: 'all' ,db: all ,addr: localhost ,auth: pwd ,title: 'allow all user local access with pwd' } + - {user: '${monitor}' ,db: pgbouncer ,addr: intra ,auth: ssl ,title: 'monitor access via intranet with pwd' } + - {user: '${monitor}' ,db: all ,addr: world ,auth: deny ,title: 'reject all other monitor access addr' } + - {user: '${admin}' ,db: all ,addr: intra ,auth: ssl ,title: 'admin access via intranet with pwd' } + - {user: '${admin}' ,db: all ,addr: world ,auth: deny ,title: 'reject all other admin access addr' } + - {user: 'all' ,db: all ,addr: intra ,auth: ssl ,title: 'allow all user intra access with pwd' } + +# OPTIONAL delayed cluster for pg-meta +pg-meta-delay: # delayed instance for pg-meta (1 hour ago) + hosts: { 10.10.10.13: { pg_seq: 1, pg_role: primary, pg_upstream: 10.10.10.10, pg_delay: 1h } } + vars: { pg_cluster: pg-meta-delay } +``` + +
+ + +
Example: Horizontal Sharding with Citus
+ +You can perform horizontal sharding on vanilla postgres with [**`CITUS`**](https://pigsty.io/docs/pgsql/kernel/citus/). + +```yaml +# pg-citus: 10 node citus cluster (5 x primary-replica pair) +pg-citus: # citus group + hosts: + 10.10.10.50: { pg_group: 0, pg_cluster: pg-citus0 ,pg_vip_address: 10.10.10.60/24 ,pg_seq: 0, pg_role: primary } + 10.10.10.51: { pg_group: 0, pg_cluster: pg-citus0 ,pg_vip_address: 10.10.10.60/24 ,pg_seq: 1, pg_role: replica } + 10.10.10.52: { pg_group: 1, pg_cluster: pg-citus1 ,pg_vip_address: 10.10.10.61/24 ,pg_seq: 0, pg_role: primary } + 10.10.10.53: { pg_group: 1, pg_cluster: pg-citus1 ,pg_vip_address: 10.10.10.61/24 ,pg_seq: 1, pg_role: replica } + 10.10.10.54: { pg_group: 2, pg_cluster: pg-citus2 ,pg_vip_address: 10.10.10.62/24 ,pg_seq: 0, pg_role: primary } + 10.10.10.55: { pg_group: 2, pg_cluster: pg-citus2 ,pg_vip_address: 10.10.10.62/24 ,pg_seq: 1, pg_role: replica } + 10.10.10.56: { pg_group: 3, pg_cluster: pg-citus3 ,pg_vip_address: 10.10.10.63/24 ,pg_seq: 0, pg_role: primary } + 10.10.10.57: { pg_group: 3, pg_cluster: pg-citus3 ,pg_vip_address: 10.10.10.63/24 ,pg_seq: 1, pg_role: replica } + 10.10.10.58: { pg_group: 4, pg_cluster: pg-citus4 ,pg_vip_address: 10.10.10.64/24 ,pg_seq: 0, pg_role: primary } + 10.10.10.59: { pg_group: 4, pg_cluster: pg-citus4 ,pg_vip_address: 10.10.10.64/24 ,pg_seq: 1, pg_role: replica } + vars: + pg_mode: citus # pgsql cluster mode: citus + pg_shard: pg-citus # citus shard name: pg-citus + pg_primary_db: test # primary database used by citus + pg_dbsu_password: DBUser.Postgres # all dbsu password access for citus cluster + pg_vip_enabled: true + pg_vip_interface: eth1 + pg_extensions: [ 'citus postgis timescaledb pgvector' ] + pg_libs: 'citus, timescaledb, pg_stat_statements, auto_explain' # citus will be added by patroni automatically + pg_users: [ { name: test ,password: test ,pgbouncer: true ,roles: [ dbrole_admin ] } ] + pg_databases: [ { name: test ,owner: test ,extensions: [ { name: citus }, { name: postgis } ] } ] + pg_hba_rules: + - { user: 'all' ,db: all ,addr: 10.10.10.0/24 ,auth: trust ,title: 'trust citus cluster members' } + - { user: 'all' ,db: all ,addr: 127.0.0.1/32 ,auth: ssl ,title: 'all user ssl access from localhost' } + - { user: 'all' ,db: all ,addr: intra ,auth: ssl ,title: 'all user ssl access from intranet' } +``` + +[![citus](https://pigsty.io/img/pigsty/citus.jpg)](https://pigsty.io/docs/pgsql/kernel/citus) + +
+ + +You can deploy different kinds of PostgreSQL instance such as [`primary`](https://pigsty.io/docs/pgsql/config/cluster#primary), [`replica`](https://pigsty.io/docs/pgsql/config/cluster#replica), [`offline`](https://pigsty.io/docs/pgsql/config/cluster#offline), [`delayed`](https://pigsty.io/docs/pgsql/config/cluster#delayed), [`sync standby`](https://pigsty.io/docs/pgsql/config/cluster#sync-standby), etc., +and customize with scene-optimize [**config templates**](https://pigsty.io/docs/concept/iac/template) and all **444** [**extensions**](https://pgext.cloud/list) out-of-the-box. + +You can define [**Users**](https://pigsty.io/docs/pgsql/config/user), [**Databases**](https://pigsty.io/docs/pgsql/config/db), [**Service**](https://pigsty.io/docs/pgsql/service), [**HBAs**](https://pigsty.io/docs/pgsql/config/hba) and other entities with code and provision them in one pass. + +-------- + +You can also self-host postgres-centric software like [**`SUPABASE`**](https://pigsty.io/docs/app/supabase), [**`Odoo`**](https://pigsty.io/docs/app/odoo) & [**`Dify`**](https://pigsty.io/docs/app/dify), Electric, GitLab, ... with Pigsty: + +
Example: Self-hosting Supabase
+ +You can launch a [self-hosting supabase](https://pigsty.io/docs/app/supabase) with MinIO and PostgreSQL with just several commands: + +```bash +./configure -c supabase # use supabase config +./deploy.yml # install pigsty +./docker.yml # install docker compose +./app.yml # launch supabase stateless part with docker +``` + +The [`conf/supabase.yml`](https://github.com/pgsty/pigsty/blob/main/conf/supabase.yml) just describe everything you need: + +```yaml +pg-meta: + hosts: { 10.10.10.10: { pg_seq: 1, pg_role: primary } } + vars: + pg_cluster: pg-meta + pg_users: + # supabase roles: anon, authenticated, dashboard_user + - { name: anon ,login: false } + - { name: authenticated ,login: false } + - { name: dashboard_user ,login: false ,replication: true ,createdb: true ,createrole: true } + - { name: service_role ,login: false ,bypassrls: true } + # supabase users: please use the same password + - { name: supabase_admin ,password: 'DBUser.Supa' ,pgbouncer: true ,inherit: true ,roles: [ dbrole_admin ] ,superuser: true ,replication: true ,createdb: true ,createrole: true ,bypassrls: true } + - { name: authenticator ,password: 'DBUser.Supa' ,pgbouncer: true ,inherit: false ,roles: [ dbrole_admin, authenticated ,anon ,service_role ] } + - { name: supabase_auth_admin ,password: 'DBUser.Supa' ,pgbouncer: true ,inherit: false ,roles: [ dbrole_admin ] ,createrole: true } + - { name: supabase_storage_admin ,password: 'DBUser.Supa' ,pgbouncer: true ,inherit: false ,roles: [ dbrole_admin, authenticated ,anon ,service_role ] ,createrole: true } + - { name: supabase_functions_admin ,password: 'DBUser.Supa' ,pgbouncer: true ,inherit: false ,roles: [ dbrole_admin ] ,createrole: true } + - { name: supabase_replication_admin ,password: 'DBUser.Supa' ,replication: true ,roles: [ dbrole_admin ]} + - { name: supabase_read_only_user ,password: 'DBUser.Supa' ,bypassrls: true ,roles: [ dbrole_readonly, pg_read_all_data ] } + pg_databases: + - name: postgres + baseline: supabase.sql + owner: supabase_admin + comment: supabase postgres database + schemas: [ extensions ,auth ,realtime ,storage ,graphql_public ,supabase_functions ,_analytics ,_realtime ] + extensions: + - { name: pgcrypto ,schema: extensions } # cryptographic functions + - { name: pg_net ,schema: extensions } # async HTTP + - { name: pgjwt ,schema: extensions } # json web token API for postgres + - { name: uuid-ossp ,schema: extensions } # generate universally unique identifiers (UUIDs) + - { name: pgsodium } # pgsodium is a modern cryptography library for Postgres. + - { name: supabase_vault } # Supabase Vault Extension + - { name: pg_graphql } # pg_graphql: GraphQL support + - { name: pg_jsonschema } # pg_jsonschema: Validate json schema + - { name: wrappers } # wrappers: FDW collections + - { name: http } # http: allows web page retrieval inside the database. + - { name: pg_cron } # pg_cron: Job scheduler for PostgreSQL + - { name: timescaledb } # timescaledb: Enables scalable inserts and complex queries for time-series data + - { name: pg_tle } # pg_tle: Trusted Language Extensions for PostgreSQL + - { name: vector } # pgvector: the vector similarity search + - { name: pgmq } # pgmq: A lightweight message queue like AWS SQS and RSMQ + # supabase required extensions + pg_libs: 'timescaledb, plpgsql, plpgsql_check, pg_cron, pg_net, pg_stat_statements, auto_explain, pg_tle, plan_filter' + pg_parameters: + cron.database_name: postgres + pgsodium.enable_event_trigger: off + pg_hba_rules: # supabase hba rules, require access from docker network + - { user: all ,db: postgres ,addr: intra ,auth: pwd ,title: 'allow supabase access from intranet' } + - { user: all ,db: postgres ,addr: 172.17.0.0/16 ,auth: pwd ,title: 'allow access from local docker network' } + pg_crontab: [ '00 01 * * * /pg/bin/pg-backup full' ] # make a full backup every 1am +``` + +![](https://pigsty.io/img/docs/supa-home.png) + +
+ +There are other pro, beta, or pilot modules, and there will be more coming in the future: + +[![BABELFISH](https://img.shields.io/badge/WILTONDB-%2388A3CA?style=flat&logo=postgresql&labelColor=88A3CA&logoColor=black)](https://pigsty.io/docs/pgsql/kernel/babelfish) +[![POLARDB PG](https://img.shields.io/badge/POLARDB_PG-%23DF6F2E?style=flat&logo=postgresql&labelColor=DF6F2E&logoColor=black)](https://pigsty.io/docs/pgsql/kernel/polardb) +[![POLARDB ORACLE](https://img.shields.io/badge/POLARDB_ORACLE-%23DF6F2E?style=flat&logo=postgresql&labelColor=DF6F2E&logoColor=black)](https://pigsty.io/docs/pgsql/kernel/polardb-o) +[![IVORYSQL](https://img.shields.io/badge/IVORYSQL-%23E8AC52?style=flat&logo=postgresql&labelColor=E8AC52&logoColor=black)](https://pigsty.io/docs/pgsql/kernel/ivorysql) +[![GREENPLUM](https://img.shields.io/badge/GREENPLUM-%23578B09?style=flat&logo=postgresql&labelColor=578B09&logoColor=black)](https://pigsty.io/docs/pgsql/kernel/greenplum) +[![CLOUDBERRY](https://img.shields.io/badge/CLOUDBERRY-orange?style=flat&logo=postgresql&labelColor=orange&logoColor=black)](https://pigsty.io/docs/pgsql/kernel/cloudberry) +[![HALO](https://img.shields.io/badge/HALO-%2366D9C6?style=flat&logo=postgresql&labelColor=66D9C6&logoColor=black)](https://pigsty.io/docs/pgsql/kernel/openhalo) +[![SUPABASE](https://img.shields.io/badge/SUPABASE-%233FCF8E?style=flat&logo=supabase&labelColor=3FCF8E&logoColor=white)](https://pigsty.io/docs/pgsql/kernel/supabase) +[![KAFKA](https://img.shields.io/badge/KAFKA-%23231F20?style=flat&logo=apachekafka&labelColor=231F20&logoColor=white)](https://pigsty.io/docs/pilot/kafka) +[![MYSQL](https://img.shields.io/badge/MYSQL-%234479A1?style=flat&logo=mysql&labelColor=4479A1&logoColor=white)](https://pigsty.io/docs/pilot/kafka) +[![DUCKDB](https://img.shields.io/badge/DUCKDB-%23FFF000?style=flat&logo=duckdb&labelColor=FFF000&logoColor=white)](https://pigsty.io/docs/pilot/duckdb) +[![TIGERBEETLE](https://img.shields.io/badge/TIGERBEETLE-%231919191?style=flat&logo=openbugbounty&labelColor=1919191&logoColor=white)](https://pigsty.io/docs/pilot/tigerbeetle) +[![VICTORIA](https://img.shields.io/badge/VICTORIA-%23621773?style=flat&logo=victoriametrics&labelColor=621773&logoColor=white)](https://pigsty.io/docs/pilot/victoria) +[![KUBERNETES](https://img.shields.io/badge/KUBERNETES-%23326CE5?style=flat&logo=kubernetes&labelColor=326CE5&logoColor=white)](https://pigsty.io/docs/pilot/kube) +[![CONSUL](https://img.shields.io/badge/CONSUL-%23F24C53?style=flat&logo=consul&labelColor=F24C53&logoColor=white)](https://pigsty.io/docs/pilot/consul) +[![JUPYTER](https://img.shields.io/badge/JUPYTER-%23F37626?style=flat&logo=jupyter&labelColor=F37626&logoColor=white)](https://pigsty.io/docs/vibe/) +[![COCKROACH](https://img.shields.io/badge/COCKROACH-%236933FF?style=flat&logo=cockroachlabs&labelColor=6933FF&logoColor=white)](https://pigsty.io/docs/pilot) + + +---------------- + +## Compatibility + +We recommend using RockyLinux 10.0, Debian 13.2, and Ubuntu 24.04.2 for production use. + +Pigsty runs on bare linux directly, and focuses on active maintained mainstream LTS [**Linux Distros**](https://pigsty.io/docs/ref/linux): + +| Code | Distro | `x86_64` | Status | `aarch64` | Status | +|:--------:|-----------------------------------|:---------------------------------------------------:|:-------|:-----------------------------------------------------:|:-------| +| **EL10** | RHEL 10 / Rocky10 / Alma10 / ... | [`el10.x86_64`](roles/node_id/vars/el10.x86_64.yml) | ✅📦 | [`el10.aarch64`](roles/node_id/vars/el10.aarch64.yml) | ✅📦 | +| **EL9** | RHEL 9 / Rocky9 / Alma9 / ... | [`el9.x86_64`](roles/node_id/vars/el9.x86_64.yml) | ✅📦 | [`el9.aarch64`](roles/node_id/vars/el9.aarch64.yml) | ✅📦 | +| **EL8** | RHEL 8 / Rocky8 / Alma8 / Anolis8 | [`el8.x86_64`](roles/node_id/vars/el8.x86_64.yml) | ✅📦 | [`el8.aarch64`](roles/node_id/vars/el8.aarch64.yml) | ✅📦 | +| **U24** | Ubuntu 24.04 (noble) | [`u24.x86_64`](roles/node_id/vars/u24.x86_64.yml) | ✅📦 | [`u24.aarch64`](roles/node_id/vars/u24.aarch64.yml) | ✅📦 | +| **U22** | Ubuntu 22.04 (jammy) | [`u22.x86_64`](roles/node_id/vars/u22.x86_64.yml) | ✅📦 | [`u22.aarch64`](roles/node_id/vars/u22.aarch64.yml) | ✅📦 | +| **D13** | Debian 13 (trixie) | [`d13.x86_64`](roles/node_id/vars/d13.x86_64.yml) | ✅📦 | [`d13.aarch64`](roles/node_id/vars/d13.aarch64.yml) | ✅📦 | +| **D12** | Debian 12 (bookworm) | [`d12.x86_64`](roles/node_id/vars/d12.x86_64.yml) | ✅📦 | [`d12.aarch64`](roles/node_id/vars/d12.aarch64.yml) | ✅📦 | + + + +## Sponsors + +Many thanks to our contributors and sponsors for making Pigsty possible. +Special thanks to MiraclePlus for fund, to Cloudflare for hosting the Pigsty repo, and to Vercel for hosting the Pigsty website. + +[![Vercel OSS Program](https://vercel.com/oss/program-badge.svg)](https://vercel.com/oss) + + +## License + +Pigsty is licensed under [**Apache-2.0**](LICENSE) (since v4.0), Check [**Docs**](https://pigsty.io/docs/about/license) for details. + + +## About + +[![Webite: pigsty.io](https://img.shields.io/badge/Website-pigsty.io-slategray?style=flat)](https://pigsty.io/docs) +[![Github: Discussions](https://img.shields.io/badge/GitHub-Discussions-slategray?style=flat&logo=github&logoColor=black)](https://github.com/pgsty/pigsty/discussions) +[![Telegram: gV9zfZraNPM3YjFh](https://img.shields.io/badge/Telegram-gV9zfZraNPM3YjFh-cornflowerblue?style=flat&logo=telegram&logoColor=cornflowerblue)](https://t.me/joinchat/gV9zfZraNPM3YjFh) +[![Discord: j5pG8qfKxU](https://img.shields.io/badge/Discord-j5pG8qfKxU-mediumpurple?style=flat&logo=discord&logoColor=mediumpurple)](https://discord.gg/j5pG8qfKxU) +[![Wechat: pigsty-cc](https://img.shields.io/badge/WeChat-pigsty--cc-green?style=flat&logo=wechat&logoColor=green)](https://pigsty.io/img/pigsty/pigsty-cc.jpg) +[![QQ: 619377403](https://img.shields.io/badge/QQ-619377403-blue?style=flat&logo=qq)](https://qm.qq.com/q/vm8LIeUqGc) + +[![Author: RuohangFeng](https://img.shields.io/badge/Author-Ruohang_Feng-steelblue?style=flat)](https://vonng.com/) +[![About: @Vonng](https://img.shields.io/badge/%40Vonng-steelblue?style=flat)](https://vonng.com/en/) +[![Mail: rh@vonng.com](https://img.shields.io/badge/rh%40vonng.com-steelblue?style=flat)](mailto:rh@vonng.com) +[![Copyright: 2018-2026 rh@Vonng.com](https://img.shields.io/badge/Copyright-2018--2026_(rh%40vonng.com)-red?logo=c&color=steelblue)](https://github.com/Vonng) +[![License: Apache-2.0](https://img.shields.io/badge/License-Apache--2.0-steelblue?style=flat&logo=opensourceinitiative&logoColor=green)](https://pigsty.io/docs/about/license/) +[![Service: PGSTY PRO](https://img.shields.io/badge/Service-PGSTY-steelblue?style=flat)](https://pigsty.cc/docs/about/service) diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..999d330 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,13 @@ +[defaults] +forks = 10 +nocows = 1 +timeout = 15 +pipelining = True +inventory = pigsty.yml +host_key_checking = False +command_warnings = False +deprecation_warnings = False +force_valid_group_names = ignore +use_persistent_connections = True +allow_world_readable_tmpfiles = False +ansible_managed = 'ansible managed: %Y-%m-%d %H:%M:%S' \ No newline at end of file diff --git a/app.yml b/app.yml new file mode 100755 index 0000000..bb8ed87 --- /dev/null +++ b/app.yml @@ -0,0 +1,136 @@ +#!/usr/bin/env ansible-playbook +--- +#==============================================================# +# File : app.yml +# Desc : copy and launch docker compose app +# Ctime : 2025-01-11 +# Mtime : 2025-06-17 +# Path : app.yml +# Docs : https://pigsty.io/docs/app +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +#==============================================================# + + +#--------------------------------------------------------------# +# Usage +#--------------------------------------------------------------# +# 1. specify the `app` param +# by default, it's a docker compose app in the app/ folder +# configure it in the pigsty.yml or pass it as `-e app=` +# +# 2. OPTIONAL, specify app details with apps: +# +# This playbook relies on the DOCKER module to work +# This playbook will: +# 1. copy docker compose resource from app/ to /opt/ +# 2. create required dirs with app.file (optional) +# 3. overwrite .env config with app.conf (optional) +# 4. launch app with docker compose up +# +#--------------------------------------------------------------# +# Example +#--------------------------------------------------------------# +# ./app.yml -e app=pgweb # simple app can be launched directly +# ./app.yml -e app=pgadmin # without any further configuration +# +# ./app.yml -e app=bytebase # for sophisticated software +# ./app.yml -e app=odoo # configure pigsty.yml `apps` +# ./app.yml -e app=dify # to have fine-grained control +#--------------------------------------------------------------# + +- name: NODE ID + become: true + hosts: all + gather_facts: no + roles: + - { role: node_id ,tags: id } + #- { role: docker ,tags: docker } + +# Run docker compose application (require the docker module) +- name: APP INSTALL + hosts: all + gather_facts: no + become: true + tasks: + + #----------------------------------------------------------# + # Validate app and app definition [preflight] + #----------------------------------------------------------# + - name: preflight + tags: [ app_check, preflight, always ] + connection: local + block: + - name: validate app parameter + assert: + that: + - app is defined + - app != '' + fail_msg: "the 'app' arg is not given (-e app=)" + + - name: validate docker exists + assert: + that: + - docker_enabled is defined and docker_enabled + fail_msg: "docker_enabled is required to install app" + + - name: fetch app definition + set_fact: + app_def: "{{ apps[app] | default({}) }}" + + - name: set app properties + set_fact: + app_name: "{{ app }}" + app_src: "{{ app_def.src | default(playbook_dir + '/app/' + app) }}" # override src name with app.src + app_dest: "{{ app_def.dest | default('/opt/' + app) }}" # override dest name with app.dest + app_conf: "{{ app_def.conf | default({}) }}" # application configuration + app_file: "{{ app_def.file | default([]) }}" # application files & directories + app_args: "{{ app_def.args | default('') }}" # application makefile shortcuts + + - name: print app details + debug: + msg: "app: {{ app_name }}, src: {{ app_src }}, dest: {{ app_dest }}, conf: {{ app_conf }}, file: {{ app_file }}" + + #----------------------------------------------------------# + # Install app resources to /opt [app_install] + #----------------------------------------------------------# + # copy docker app folder to /opt/ or /opt/ + # this will fail if the app/ folder does not exist + - name: install app resource to /opt + tags: app_install + copy: src="{{ app_src }}/" dest="/{{ app_dest }}/" + + #----------------------------------------------------------# + # Prepare files & directories [app_file] + #----------------------------------------------------------# + - name: setup app files & directories + tags: app_file + when: app_file | length > 0 + file: "{{ item }}" + with_items: "{{ app_file }}" + + #----------------------------------------------------------# + # Configure app with .env [app_config] + #----------------------------------------------------------# + # append entries to app .env config (override existing entries) + - name: configure app by updating .env + tags: app_config + when: app_conf | length > 0 + lineinfile: + path: "{{ app_dest }}/.env" + regexp: '^{{ item.key | upper }}=' + line: '{{ item.key | upper }}={{ item.value if item.value is not boolean else item.value | string | lower }}' + create: yes + loop: "{{ app_conf | dict2items }}" + + #----------------------------------------------------------# + # Launch app with make command [app_launch] + #----------------------------------------------------------# + # it may take a while to pull images, or very fast if you already have local image loaded, or proxy, mirror. + - name: launch app + tags: app_launch + shell: + cmd: make {{ app_args }} + chdir: "{{ app_dest }}" + +... \ No newline at end of file diff --git a/app/README.md b/app/README.md new file mode 100644 index 0000000..c2822a4 --- /dev/null +++ b/app/README.md @@ -0,0 +1,153 @@ +# App + +Here are some docker compose templates for popular applications that works well with PostgreSQL + +* [Supabase](supabase/) : The open source Firebase alternative, built on Postgres +* [FerretDB](ferretdb/) : Open Source MongoDB alternative, built on Postgres +* [PolarDB](polardb/) : Open Source RAC alternative, built on Postgres +* [PgAdmin4](pgadmin/) : Postgres Admin Tools +* [PgWeb](pgweb/) : Postgres Web Console +* [ByteBase](bytebase/) : Postgres DDL Migration +* [PostgREST](postgrest/) : Auto-Generated PG Backend REST API +* [Kong](kong/) : Kong API Gateway +* [Gitea](gitea/) : Self-Hosting Git Services +* [Wiki](wiki/) : Local Wiki Service +* [NocoDB](nocodb/) : Open source airtable alternative +* [Odoo](odoo/) : Open Source ERP +* [Dify](dify/) : Open Source AI Workflow Platform for GenAI Apps. +* [Metabase](metabase/) : Fast analytics with the friendly UX and integrated tooling +* [Jupyter](jupyter/) : Run jupyter notebook, the data analysis IDE with docker +* etc.... + + +**Docker** + +You can install docker with the [`docker.yml`](https://github.com/pgsty/pigsty/blob/master/docker.yml) playbook. +If you wish to use a mirror, add them to `docker_registry_mirrors`. To use another registry: + +```bash +docker login quay.io +```` + + +**Portals** + +There are several predefined portals + +```yaml +postgrest : { domain: api.pigsty ,endpoint: "127.0.0.1:8884" } +pgadmin : { domain: adm.pigsty ,endpoint: "127.0.0.1:8885" } +pgweb : { domain: cli.pigsty ,endpoint: "127.0.0.1:8886" } +bytebase : { domain: ddl.pigsty ,endpoint: "127.0.0.1:8887" } +jupyter : { domain: lab.pigsty ,endpoint: "127.0.0.1:8888" ,websocket: true } +gitea : { domain: git.pigsty ,endpoint: "127.0.0.1:8889" } +minio : { domain: sss.pigsty ,endpoint: "127.0.0.1:9000" } +wiki : { domain: wiki.pigsty ,endpoint: "127.0.0.1:9002" } +noco : { domain: noco.pigsty ,endpoint: "127.0.0.1:9003" } +odoo : { domain: odoo.pigsty ,endpoint: "127.0.0.1:8069" ,websocket: true } +supa : { domain: supa.pigsty ,endpoint: "127.0.0.1:8000" ,websocket: true } +dify : { domain: dify.pigsty ,endpoint: "127.0.0.1:8001" ,websocket: true } +``` + +**Pull Image** + +```bash +docker pull dpage/pgadmin4 +docker pull sosedoff/pgweb +docker pull vonng/pg_exporter +docker pull postgrest/postgrest +docker pull bytebase/bytebase +docker pull jupyter/minimal-notebook +#docker pull jupyter/scipy-notebook:latest +docker pull alpine +docker pull registry +docker pull andrewjones/schemaspy-postgres +docker pull requarks/wiki +docker pull gitea/gitea +docker pull kong +docker pull odoo +docker pull quay.io/ferretdb/ferretdb +``` + + +**Make Cache** + +```bash +mkdir -p /tmp/docker +docker save dpage/pgadmin4 | gzip -9 -c > /tmp/docker/pgadmin4.tgz +docker save sosedoff/pgweb | gzip -9 -c > /tmp/docker/pgweb.tgz +docker save vonng/pg_exporter | gzip -9 -c > /tmp/docker/pg_exporter.tgz +docker save postgrest/postgrest | gzip -9 -c > /tmp/docker/postgrest.tgz +docker save bytebase/bytebase | gzip -9 -c > /tmp/docker/bytebase.tgz +docker save jupyter/minimal-notebook | gzip -9 -c > /tmp/docker/jupyter.tgz +docker save alpine | gzip -9 -c > /tmp/docker/alpine.tgz +docker save registry | gzip -9 -c > /tmp/docker/registry.tgz +docker save andrewjones/schemaspy-postgres | gzip -9 -c > /tmp/docker/schemaspy.tgz +docker save requarks/wiki | gzip -9 -c > /tmp/docker/wiki.tgz +docker save gitea/gitea | gzip -9 -c > /tmp/docker/gitea.tgz +docker save kong | gzip -9 -c > /tmp/docker/kong.tgz +docker save odoo | gzip -9 -c > /tmp/docker/odoo.tgz +docker save quay.io/ferretdb/ferretdb | gzip -9 -c > /tmp/docker/ferretdb.tgz +docker save nocodb/nocodb | gzip -9 -c > /tmp/docker/nocodb.tgz +``` + + +**Load Cache** + +```bash +cat /tmp/docker/pg_exporter.tgz | gzip -d -c - | docker load; +cat /tmp/docker/postgrest.tgz | gzip -d -c - | docker load; +cat /tmp/docker/pgweb.tgz | gzip -d -c - | docker load; +cat /tmp/docker/pgadmin4.tgz | gzip -d -c - | docker load; +cat /tmp/docker/bytebase.tgz | gzip -d -c - | docker load; +cat /tmp/docker/jupyter.tgz | gzip -d -c - | docker load; +cat /tmp/docker/wiki.tgz | gzip -d -c - | docker load; +cat /tmp/docker/gitea.tgz | gzip -d -c - | docker load; +cat /tmp/docker/kong.tgz | gzip -d -c - | docker load; +cat /tmp/docker/odoo.tgz | gzip -d -c - | docker load; +cat /tmp/docker/alpine.tgz | gzip -d -c - | docker load; +cat /tmp/docker/registry.tgz | gzip -d -c - | docker load; +cat /tmp/docker/schemaspy.tgz | gzip -d -c - | docker load; +cat /tmp/docker/nocodb.tgz | gzip -d -c - | docker load; +``` + + +## Software + +There are lots of software using PostgreSQL / Redis. + +* KeyCloak : SSO Solution +* Gitlab : Code Management +* Jira : Project Management +* Confluence: Document Management +* Harbour : Image Management +* JupyterLab : Data Analysis +* Grafana : Data Visualization +* Mastodon : Social Network +* Discourse : Community + +```bash +docker pull quay.io/keycloak/keycloak:18.0.0 # 18.0.0 # 562MB +docker pull tootsuite/mastodon # latest # 1.76GB +docker pull cptactionhank/atlassian-confluence:7.7.3 # 7.7.3 # 835MB +docker pull cptactionhank/atlassian-jira-software:8.1.0 # 8.1.0 # 531MB +docker pull jupyter/scipy-notebook # latest # 3.01GB +docker pull gitlab/gitlab-ee # latest # 2.69GB +docker pull grafana/grafana-oss # latest # 286MB +``` + +**Make Cache for Software** + +```bash +docker save quay.io/keycloak/keycloak tootsuite/mastodon cptactionhank/atlassian-confluence cptactionhank/atlassian-jira-software jupyter/scipy-notebook gitlab/gitlab-ee grafana/grafana-oss | gzip -c - > docker.tgz +cat /tmp/docker.tgz | gzip -d -c - | docker load +``` + + +## Visualization App + +Check [pigsty-app](https://github.com/pgsty/pigsty-app) for details. + +Pigsty has one embed visualization app: [pglog](https://demo.pigsty.io/ui/d/pglog-overview) which is used for pg csv log analysis. + +There's another visualization app works on pigsty: ISD : noaa weather data visualization: [github.com/Vonng/isd](https://github.com/Vonng/isd), [Demo](https://demo.pigsty.io/ui/d/isd-overview) diff --git a/app/bytebase/.env b/app/bytebase/.env new file mode 100644 index 0000000..66c71e8 --- /dev/null +++ b/app/bytebase/.env @@ -0,0 +1,3 @@ +BB_PORT=8887 +BB_DOMAIN=http://ddl.pigsty +BB_PGURL="postgresql://dbuser_bytebase:DBUser.Bytebase@10.10.10.10:5432/bytebase?sslmode=prefer" \ No newline at end of file diff --git a/app/bytebase/Makefile b/app/bytebase/Makefile new file mode 100644 index 0000000..16e4321 --- /dev/null +++ b/app/bytebase/Makefile @@ -0,0 +1,53 @@ +default: up view + +up: view + docker compose up -d + +# run with local data dir & external pg +run: pg + mkdir -p /data/bytebase/data; + docker run --init --name bytebase \ + --restart always --detach \ + --publish 8887:8887 \ + --volume /data/bytebase/data:/var/opt/bytebase \ + -e PG_URL=postgresql://dbuser_bytebase:DBUser.Bytebase@10.10.10.10:5432/bytebase \ + bytebase/bytebase:latest \ + --data /var/opt/bytebase \ + --external-url http://ddl.pigsty \ + --port 8887 + +view: + @echo "visit http://ddl.pigsty or http://10.10.10.10:8887" + +log: + docker logs -f bytebase + +info: + docker inspect bytebase | jq + +# prepare external pg +pg: + @echo "pg_users: { name: dbuser_bytebase , password: DBUser.Bytebase, roles: [ dbrole_admin ] }" + @echo "pg_databases: { name: bytebase, owner: dbuser_bytebase, comment: bytebase primary database }" + @echo "bin/pgsql-user pg-meta dbuser_bytebase ; bin/pgsql-db pg-meta bytebase" + +stop: + docker stop bytebase; + +clean: + docker stop bytebase; docker rm -f bytebase + +pull: + docker pull bytebase/bytebase:latest + +rmi: + docker rmi bytebase/bytebase:latest + +save: + mkdir -p /tmp/docker + docker save bytebase/bytebase:latest | gzip -9 -c > /tmp/docker/bytebase.tgz + +load: + cat /tmp/docker/bytebase.tgz | gzip -d -c - | docker load + +.PHONY: default up run view log info stop clean conf dump pull rmi save load \ No newline at end of file diff --git a/app/bytebase/README.md b/app/bytebase/README.md new file mode 100644 index 0000000..7c336d3 --- /dev/null +++ b/app/bytebase/README.md @@ -0,0 +1,66 @@ +# Bytebase + +Bytebase: https://www.bytebase.com/ + +open-source schema migrator for PostgreSQL (and other databases) + +Check public demo: http://ddl.pigsty.io, username: `admin@pigsty.cc`, password: `pigsty` + +If you want to access bytebase through SSL, you have to trust `files/pki/ca/ca.crt` on your browser (or use the dirty hack `thisisunsafe` in chrome) + + +## Get Started + +Check [`.env`](.env) file for configurable environment variables: + +```bash +BB_PORT=8887 +BB_DOMAIN=http://ddl.pigsty +BB_PGURL="postgresql://dbuser_bytebase:DBUser.Bytebase@10.10.10.10:5432/bytebase?sslmode=prefer" +``` + +Then launch bytebase with: + +```bash +make up # docker compose up +``` + +Visit [http://ddl.pigsty](http://ddl.pigsty) or http://10.10.10.10:8887 + +## Makefile + +```bash +make up # pull up bytebase with docker compose in minimal mode +make run # launch bytebase with docker , local data dir and external PostgreSQL +make view # print bytebase access point +make log # tail -f bytebase logs +make info # introspect bytebase with jq +make stop # stop bytebase container +make clean # remove bytebase container +make pull # pull latest bytebase image +make rmi # remove bytebase image +make save # save bytebase image to /tmp/docker/bytebase.tgz +make load # load bytebase image from /tmp/docker/bytebase.tgz +``` + +## Use External PostgreSQL + +Bytebase use its internal PostgreSQL database by default, You can use external PostgreSQL for higher durability. + +```yaml +pg_users: [ { name: dbuser_bytebase ,password: DBUser.Bytebase ,pgbouncer: true ,roles: [ dbrole_admin ] ,comment: admin user for bytebase database } ] +pg_databases: [ { name: bytebase ,owner: dbuser_bytebase ,revokeconn: true ,comment: bytebase primary database } ] +``` + +And create business user & database with: + +```bash +bin/pgsql-user pg-meta dbuser_bytebase +bin/pgsql-db pg-meta bytebase +``` + +Check connectivity: + +```bash +psql postgres://dbuser_bytebase:DBUser.Bytebase@10.10.10.10:5432/bytebase +``` \ No newline at end of file diff --git a/app/bytebase/docker-compose.yml b/app/bytebase/docker-compose.yml new file mode 100644 index 0000000..230b2b1 --- /dev/null +++ b/app/bytebase/docker-compose.yml @@ -0,0 +1,13 @@ +# check .env files before running + +services: + bytebase: + container_name: bytebase + image: bytebase/bytebase:latest + restart: unless-stopped + environment: + PG_URL: ${BB_PGURL} + ports: + - "${BB_PORT}:${BB_PORT}" + command: | + --external-url ${BB_DOMAIN} --port ${BB_PORT} \ No newline at end of file diff --git a/app/dify/Makefile b/app/dify/Makefile new file mode 100644 index 0000000..2ce9cd6 --- /dev/null +++ b/app/dify/Makefile @@ -0,0 +1,39 @@ +default: up view + +DOCKER_IMAGE_DIR=/tmp/docker/dify + +up: view + docker compose up -d + +view: + cat .env | grep -A 1 DIFY_PORT + +log: + docker compose logs -f + +info: + docker compose ps + +down: stop +stop: + docker compose down + +clean: + docker compose rm + sudo rm -rf volumes/app; + @echo "dont' forget to nuke the dify database in postgres" + +pull: + docker compose pull + +save: + mkdir -p $(DOCKER_IMAGE_DIR) + docker save langgenius/dify-api:1.0.0 | gzip -c -9 > $(DOCKER_IMAGE_DIR)/dify-api.tgz + docker save langgenius/dify-web:1.0.0 | gzip -c -9 > $(DOCKER_IMAGE_DIR)/dify-web.tgz + +tarball: save + tar -cvf /tmp/dify.tgz -C /tmp/docker dify + +load: + cat $(DOCKER_IMAGE_DIR)/dify-api.tgz | gzip -d -c - | docker load + cat $(DOCKER_IMAGE_DIR)/dify-web.tgz | gzip -d -c - | docker load diff --git a/app/dify/docker-compose.yml b/app/dify/docker-compose.yml new file mode 100644 index 0000000..3a865d8 --- /dev/null +++ b/app/dify/docker-compose.yml @@ -0,0 +1,851 @@ +# ================================================================== +# WARNING: This file is auto-generated by generate_docker_compose +# Do not modify this file directly. Instead, update the .env.example +# or docker-compose-template.yaml and regenerate this file. +# ================================================================== +# File: https://github.com/pgsty/pigsty/blob/main/app/dify/docker-compose.yml +# Src: https://github.com/langgenius/dify/blob/main/docker/docker-compose.yaml +# Doc: https://pigsty.io/docs/app/dify + +x-shared-env: &shared-api-worker-env + CONSOLE_API_URL: ${CONSOLE_API_URL:-} + CONSOLE_WEB_URL: ${CONSOLE_WEB_URL:-} + SERVICE_API_URL: ${SERVICE_API_URL:-} + APP_API_URL: ${APP_API_URL:-} + APP_WEB_URL: ${APP_WEB_URL:-} + FILES_URL: ${FILES_URL:-} + INTERNAL_FILES_URL: ${INTERNAL_FILES_URL:-} + LANG: ${LANG:-en_US.UTF-8} + LC_ALL: ${LC_ALL:-en_US.UTF-8} + PYTHONIOENCODING: ${PYTHONIOENCODING:-utf-8} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + LOG_FILE: ${LOG_FILE:-/app/logs/server.log} + LOG_FILE_MAX_SIZE: ${LOG_FILE_MAX_SIZE:-20} + LOG_FILE_BACKUP_COUNT: ${LOG_FILE_BACKUP_COUNT:-5} + LOG_DATEFORMAT: ${LOG_DATEFORMAT:-%Y-%m-%d %H:%M:%S} + LOG_TZ: ${LOG_TZ:-UTC} + DEBUG: ${DEBUG:-false} + FLASK_DEBUG: ${FLASK_DEBUG:-false} + ENABLE_REQUEST_LOGGING: ${ENABLE_REQUEST_LOGGING:-False} + SECRET_KEY: ${SECRET_KEY:-sk-9f73s3ljTXVcMT3Blb3ljTqtsKiGHXVcMT3BlbkFJLK7U} + INIT_PASSWORD: ${INIT_PASSWORD:-} + DEPLOY_ENV: ${DEPLOY_ENV:-PRODUCTION} + CHECK_UPDATE_URL: ${CHECK_UPDATE_URL:-https://updates.dify.ai} + OPENAI_API_BASE: ${OPENAI_API_BASE:-https://api.openai.com/v1} + MIGRATION_ENABLED: ${MIGRATION_ENABLED:-true} + FILES_ACCESS_TIMEOUT: ${FILES_ACCESS_TIMEOUT:-300} + ACCESS_TOKEN_EXPIRE_MINUTES: ${ACCESS_TOKEN_EXPIRE_MINUTES:-60} + REFRESH_TOKEN_EXPIRE_DAYS: ${REFRESH_TOKEN_EXPIRE_DAYS:-30} + APP_MAX_ACTIVE_REQUESTS: ${APP_MAX_ACTIVE_REQUESTS:-0} + APP_MAX_EXECUTION_TIME: ${APP_MAX_EXECUTION_TIME:-1200} + DIFY_BIND_ADDRESS: ${DIFY_BIND_ADDRESS:-0.0.0.0} + DIFY_PORT: ${DIFY_PORT:-5001} + SERVER_WORKER_AMOUNT: ${SERVER_WORKER_AMOUNT:-1} + SERVER_WORKER_CLASS: ${SERVER_WORKER_CLASS:-gevent} + SERVER_WORKER_CONNECTIONS: ${SERVER_WORKER_CONNECTIONS:-10} + CELERY_WORKER_CLASS: ${CELERY_WORKER_CLASS:-} + GUNICORN_TIMEOUT: ${GUNICORN_TIMEOUT:-360} + CELERY_WORKER_AMOUNT: ${CELERY_WORKER_AMOUNT:-} + CELERY_AUTO_SCALE: ${CELERY_AUTO_SCALE:-false} + CELERY_MAX_WORKERS: ${CELERY_MAX_WORKERS:-} + CELERY_MIN_WORKERS: ${CELERY_MIN_WORKERS:-} + API_TOOL_DEFAULT_CONNECT_TIMEOUT: ${API_TOOL_DEFAULT_CONNECT_TIMEOUT:-10} + API_TOOL_DEFAULT_READ_TIMEOUT: ${API_TOOL_DEFAULT_READ_TIMEOUT:-60} + ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} + ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} + ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} + DB_USERNAME: ${DB_USERNAME:-postgres} + DB_PASSWORD: ${DB_PASSWORD:-difyai123456} + DB_HOST: ${DB_HOST:-db} + DB_PORT: ${DB_PORT:-5432} + DB_DATABASE: ${DB_DATABASE:-dify} + SQLALCHEMY_POOL_SIZE: ${SQLALCHEMY_POOL_SIZE:-30} + SQLALCHEMY_MAX_OVERFLOW: ${SQLALCHEMY_MAX_OVERFLOW:-10} + SQLALCHEMY_POOL_RECYCLE: ${SQLALCHEMY_POOL_RECYCLE:-3600} + SQLALCHEMY_ECHO: ${SQLALCHEMY_ECHO:-false} + SQLALCHEMY_POOL_PRE_PING: ${SQLALCHEMY_POOL_PRE_PING:-false} + SQLALCHEMY_POOL_USE_LIFO: ${SQLALCHEMY_POOL_USE_LIFO:-false} + POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-100} + POSTGRES_SHARED_BUFFERS: ${POSTGRES_SHARED_BUFFERS:-128MB} + POSTGRES_WORK_MEM: ${POSTGRES_WORK_MEM:-4MB} + POSTGRES_MAINTENANCE_WORK_MEM: ${POSTGRES_MAINTENANCE_WORK_MEM:-64MB} + POSTGRES_EFFECTIVE_CACHE_SIZE: ${POSTGRES_EFFECTIVE_CACHE_SIZE:-4096MB} + REDIS_HOST: ${REDIS_HOST:-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + REDIS_USERNAME: ${REDIS_USERNAME:-} + REDIS_PASSWORD: ${REDIS_PASSWORD:-difyai123456} + REDIS_USE_SSL: ${REDIS_USE_SSL:-false} + REDIS_SSL_CERT_REQS: ${REDIS_SSL_CERT_REQS:-CERT_NONE} + REDIS_SSL_CA_CERTS: ${REDIS_SSL_CA_CERTS:-} + REDIS_SSL_CERTFILE: ${REDIS_SSL_CERTFILE:-} + REDIS_SSL_KEYFILE: ${REDIS_SSL_KEYFILE:-} + REDIS_DB: ${REDIS_DB:-0} + REDIS_USE_SENTINEL: ${REDIS_USE_SENTINEL:-false} + REDIS_SENTINELS: ${REDIS_SENTINELS:-} + REDIS_SENTINEL_SERVICE_NAME: ${REDIS_SENTINEL_SERVICE_NAME:-} + REDIS_SENTINEL_USERNAME: ${REDIS_SENTINEL_USERNAME:-} + REDIS_SENTINEL_PASSWORD: ${REDIS_SENTINEL_PASSWORD:-} + REDIS_SENTINEL_SOCKET_TIMEOUT: ${REDIS_SENTINEL_SOCKET_TIMEOUT:-0.1} + REDIS_USE_CLUSTERS: ${REDIS_USE_CLUSTERS:-false} + REDIS_CLUSTERS: ${REDIS_CLUSTERS:-} + REDIS_CLUSTERS_PASSWORD: ${REDIS_CLUSTERS_PASSWORD:-} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://:difyai123456@redis:6379/1} + CELERY_BACKEND: ${CELERY_BACKEND:-redis} + BROKER_USE_SSL: ${BROKER_USE_SSL:-false} + CELERY_USE_SENTINEL: ${CELERY_USE_SENTINEL:-false} + CELERY_SENTINEL_MASTER_NAME: ${CELERY_SENTINEL_MASTER_NAME:-} + CELERY_SENTINEL_PASSWORD: ${CELERY_SENTINEL_PASSWORD:-} + CELERY_SENTINEL_SOCKET_TIMEOUT: ${CELERY_SENTINEL_SOCKET_TIMEOUT:-0.1} + WEB_API_CORS_ALLOW_ORIGINS: ${WEB_API_CORS_ALLOW_ORIGINS:-*} + CONSOLE_CORS_ALLOW_ORIGINS: ${CONSOLE_CORS_ALLOW_ORIGINS:-*} + STORAGE_TYPE: ${STORAGE_TYPE:-opendal} + OPENDAL_SCHEME: ${OPENDAL_SCHEME:-fs} + OPENDAL_FS_ROOT: ${OPENDAL_FS_ROOT:-storage} + CLICKZETTA_VOLUME_TYPE: ${CLICKZETTA_VOLUME_TYPE:-user} + CLICKZETTA_VOLUME_NAME: ${CLICKZETTA_VOLUME_NAME:-} + CLICKZETTA_VOLUME_TABLE_PREFIX: ${CLICKZETTA_VOLUME_TABLE_PREFIX:-dataset_} + CLICKZETTA_VOLUME_DIFY_PREFIX: ${CLICKZETTA_VOLUME_DIFY_PREFIX:-dify_km} + S3_ENDPOINT: ${S3_ENDPOINT:-} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET_NAME: ${S3_BUCKET_NAME:-difyai} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-} + S3_SECRET_KEY: ${S3_SECRET_KEY:-} + S3_USE_AWS_MANAGED_IAM: ${S3_USE_AWS_MANAGED_IAM:-false} + AZURE_BLOB_ACCOUNT_NAME: ${AZURE_BLOB_ACCOUNT_NAME:-difyai} + AZURE_BLOB_ACCOUNT_KEY: ${AZURE_BLOB_ACCOUNT_KEY:-difyai} + AZURE_BLOB_CONTAINER_NAME: ${AZURE_BLOB_CONTAINER_NAME:-difyai-container} + AZURE_BLOB_ACCOUNT_URL: ${AZURE_BLOB_ACCOUNT_URL:-https://.blob.core.windows.net} + GOOGLE_STORAGE_BUCKET_NAME: ${GOOGLE_STORAGE_BUCKET_NAME:-your-bucket-name} + GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: ${GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64:-} + ALIYUN_OSS_BUCKET_NAME: ${ALIYUN_OSS_BUCKET_NAME:-your-bucket-name} + ALIYUN_OSS_ACCESS_KEY: ${ALIYUN_OSS_ACCESS_KEY:-your-access-key} + ALIYUN_OSS_SECRET_KEY: ${ALIYUN_OSS_SECRET_KEY:-your-secret-key} + ALIYUN_OSS_ENDPOINT: ${ALIYUN_OSS_ENDPOINT:-https://oss-ap-southeast-1-internal.aliyuncs.com} + ALIYUN_OSS_REGION: ${ALIYUN_OSS_REGION:-ap-southeast-1} + ALIYUN_OSS_AUTH_VERSION: ${ALIYUN_OSS_AUTH_VERSION:-v4} + ALIYUN_OSS_PATH: ${ALIYUN_OSS_PATH:-your-path} + TENCENT_COS_BUCKET_NAME: ${TENCENT_COS_BUCKET_NAME:-your-bucket-name} + TENCENT_COS_SECRET_KEY: ${TENCENT_COS_SECRET_KEY:-your-secret-key} + TENCENT_COS_SECRET_ID: ${TENCENT_COS_SECRET_ID:-your-secret-id} + TENCENT_COS_REGION: ${TENCENT_COS_REGION:-your-region} + TENCENT_COS_SCHEME: ${TENCENT_COS_SCHEME:-your-scheme} + OCI_ENDPOINT: ${OCI_ENDPOINT:-https://your-object-storage-namespace.compat.objectstorage.us-ashburn-1.oraclecloud.com} + OCI_BUCKET_NAME: ${OCI_BUCKET_NAME:-your-bucket-name} + OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-your-access-key} + OCI_SECRET_KEY: ${OCI_SECRET_KEY:-your-secret-key} + OCI_REGION: ${OCI_REGION:-us-ashburn-1} + HUAWEI_OBS_BUCKET_NAME: ${HUAWEI_OBS_BUCKET_NAME:-your-bucket-name} + HUAWEI_OBS_SECRET_KEY: ${HUAWEI_OBS_SECRET_KEY:-your-secret-key} + HUAWEI_OBS_ACCESS_KEY: ${HUAWEI_OBS_ACCESS_KEY:-your-access-key} + HUAWEI_OBS_SERVER: ${HUAWEI_OBS_SERVER:-your-server-url} + VOLCENGINE_TOS_BUCKET_NAME: ${VOLCENGINE_TOS_BUCKET_NAME:-your-bucket-name} + VOLCENGINE_TOS_SECRET_KEY: ${VOLCENGINE_TOS_SECRET_KEY:-your-secret-key} + VOLCENGINE_TOS_ACCESS_KEY: ${VOLCENGINE_TOS_ACCESS_KEY:-your-access-key} + VOLCENGINE_TOS_ENDPOINT: ${VOLCENGINE_TOS_ENDPOINT:-your-server-url} + VOLCENGINE_TOS_REGION: ${VOLCENGINE_TOS_REGION:-your-region} + BAIDU_OBS_BUCKET_NAME: ${BAIDU_OBS_BUCKET_NAME:-your-bucket-name} + BAIDU_OBS_SECRET_KEY: ${BAIDU_OBS_SECRET_KEY:-your-secret-key} + BAIDU_OBS_ACCESS_KEY: ${BAIDU_OBS_ACCESS_KEY:-your-access-key} + BAIDU_OBS_ENDPOINT: ${BAIDU_OBS_ENDPOINT:-your-server-url} + SUPABASE_BUCKET_NAME: ${SUPABASE_BUCKET_NAME:-your-bucket-name} + SUPABASE_API_KEY: ${SUPABASE_API_KEY:-your-access-key} + SUPABASE_URL: ${SUPABASE_URL:-your-server-url} + VECTOR_STORE: ${VECTOR_STORE:-weaviate} + VECTOR_INDEX_NAME_PREFIX: ${VECTOR_INDEX_NAME_PREFIX:-Vector_index} + WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080} + WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} + QDRANT_URL: ${QDRANT_URL:-http://qdrant:6333} + QDRANT_API_KEY: ${QDRANT_API_KEY:-difyai123456} + QDRANT_CLIENT_TIMEOUT: ${QDRANT_CLIENT_TIMEOUT:-20} + QDRANT_GRPC_ENABLED: ${QDRANT_GRPC_ENABLED:-false} + QDRANT_GRPC_PORT: ${QDRANT_GRPC_PORT:-6334} + QDRANT_REPLICATION_FACTOR: ${QDRANT_REPLICATION_FACTOR:-1} + MILVUS_URI: ${MILVUS_URI:-http://host.docker.internal:19530} + MILVUS_DATABASE: ${MILVUS_DATABASE:-} + MILVUS_TOKEN: ${MILVUS_TOKEN:-} + MILVUS_USER: ${MILVUS_USER:-} + MILVUS_PASSWORD: ${MILVUS_PASSWORD:-} + MILVUS_ENABLE_HYBRID_SEARCH: ${MILVUS_ENABLE_HYBRID_SEARCH:-False} + MILVUS_ANALYZER_PARAMS: ${MILVUS_ANALYZER_PARAMS:-} + MYSCALE_HOST: ${MYSCALE_HOST:-myscale} + MYSCALE_PORT: ${MYSCALE_PORT:-8123} + MYSCALE_USER: ${MYSCALE_USER:-default} + MYSCALE_PASSWORD: ${MYSCALE_PASSWORD:-} + MYSCALE_DATABASE: ${MYSCALE_DATABASE:-dify} + MYSCALE_FTS_PARAMS: ${MYSCALE_FTS_PARAMS:-} + COUCHBASE_CONNECTION_STRING: ${COUCHBASE_CONNECTION_STRING:-couchbase://couchbase-server} + COUCHBASE_USER: ${COUCHBASE_USER:-Administrator} + COUCHBASE_PASSWORD: ${COUCHBASE_PASSWORD:-password} + COUCHBASE_BUCKET_NAME: ${COUCHBASE_BUCKET_NAME:-Embeddings} + COUCHBASE_SCOPE_NAME: ${COUCHBASE_SCOPE_NAME:-_default} + PGVECTOR_HOST: ${PGVECTOR_HOST:-pgvector} + PGVECTOR_PORT: ${PGVECTOR_PORT:-5432} + PGVECTOR_USER: ${PGVECTOR_USER:-postgres} + PGVECTOR_PASSWORD: ${PGVECTOR_PASSWORD:-difyai123456} + PGVECTOR_DATABASE: ${PGVECTOR_DATABASE:-dify} + PGVECTOR_MIN_CONNECTION: ${PGVECTOR_MIN_CONNECTION:-1} + PGVECTOR_MAX_CONNECTION: ${PGVECTOR_MAX_CONNECTION:-5} + PGVECTOR_PG_BIGM: ${PGVECTOR_PG_BIGM:-false} + PGVECTOR_PG_BIGM_VERSION: ${PGVECTOR_PG_BIGM_VERSION:-1.2-20240606} + VASTBASE_HOST: ${VASTBASE_HOST:-vastbase} + VASTBASE_PORT: ${VASTBASE_PORT:-5432} + VASTBASE_USER: ${VASTBASE_USER:-dify} + VASTBASE_PASSWORD: ${VASTBASE_PASSWORD:-Difyai123456} + VASTBASE_DATABASE: ${VASTBASE_DATABASE:-dify} + VASTBASE_MIN_CONNECTION: ${VASTBASE_MIN_CONNECTION:-1} + VASTBASE_MAX_CONNECTION: ${VASTBASE_MAX_CONNECTION:-5} + PGVECTO_RS_HOST: ${PGVECTO_RS_HOST:-pgvecto-rs} + PGVECTO_RS_PORT: ${PGVECTO_RS_PORT:-5432} + PGVECTO_RS_USER: ${PGVECTO_RS_USER:-postgres} + PGVECTO_RS_PASSWORD: ${PGVECTO_RS_PASSWORD:-difyai123456} + PGVECTO_RS_DATABASE: ${PGVECTO_RS_DATABASE:-dify} + ANALYTICDB_KEY_ID: ${ANALYTICDB_KEY_ID:-your-ak} + ANALYTICDB_KEY_SECRET: ${ANALYTICDB_KEY_SECRET:-your-sk} + ANALYTICDB_REGION_ID: ${ANALYTICDB_REGION_ID:-cn-hangzhou} + ANALYTICDB_INSTANCE_ID: ${ANALYTICDB_INSTANCE_ID:-gp-ab123456} + ANALYTICDB_ACCOUNT: ${ANALYTICDB_ACCOUNT:-testaccount} + ANALYTICDB_PASSWORD: ${ANALYTICDB_PASSWORD:-testpassword} + ANALYTICDB_NAMESPACE: ${ANALYTICDB_NAMESPACE:-dify} + ANALYTICDB_NAMESPACE_PASSWORD: ${ANALYTICDB_NAMESPACE_PASSWORD:-difypassword} + ANALYTICDB_HOST: ${ANALYTICDB_HOST:-gp-test.aliyuncs.com} + ANALYTICDB_PORT: ${ANALYTICDB_PORT:-5432} + ANALYTICDB_MIN_CONNECTION: ${ANALYTICDB_MIN_CONNECTION:-1} + ANALYTICDB_MAX_CONNECTION: ${ANALYTICDB_MAX_CONNECTION:-5} + TIDB_VECTOR_HOST: ${TIDB_VECTOR_HOST:-tidb} + TIDB_VECTOR_PORT: ${TIDB_VECTOR_PORT:-4000} + TIDB_VECTOR_USER: ${TIDB_VECTOR_USER:-} + TIDB_VECTOR_PASSWORD: ${TIDB_VECTOR_PASSWORD:-} + TIDB_VECTOR_DATABASE: ${TIDB_VECTOR_DATABASE:-dify} + MATRIXONE_HOST: ${MATRIXONE_HOST:-matrixone} + MATRIXONE_PORT: ${MATRIXONE_PORT:-6001} + MATRIXONE_USER: ${MATRIXONE_USER:-dump} + MATRIXONE_PASSWORD: ${MATRIXONE_PASSWORD:-111} + MATRIXONE_DATABASE: ${MATRIXONE_DATABASE:-dify} + TIDB_ON_QDRANT_URL: ${TIDB_ON_QDRANT_URL:-http://127.0.0.1} + TIDB_ON_QDRANT_API_KEY: ${TIDB_ON_QDRANT_API_KEY:-dify} + TIDB_ON_QDRANT_CLIENT_TIMEOUT: ${TIDB_ON_QDRANT_CLIENT_TIMEOUT:-20} + TIDB_ON_QDRANT_GRPC_ENABLED: ${TIDB_ON_QDRANT_GRPC_ENABLED:-false} + TIDB_ON_QDRANT_GRPC_PORT: ${TIDB_ON_QDRANT_GRPC_PORT:-6334} + TIDB_PUBLIC_KEY: ${TIDB_PUBLIC_KEY:-dify} + TIDB_PRIVATE_KEY: ${TIDB_PRIVATE_KEY:-dify} + TIDB_API_URL: ${TIDB_API_URL:-http://127.0.0.1} + TIDB_IAM_API_URL: ${TIDB_IAM_API_URL:-http://127.0.0.1} + TIDB_REGION: ${TIDB_REGION:-regions/aws-us-east-1} + TIDB_PROJECT_ID: ${TIDB_PROJECT_ID:-dify} + TIDB_SPEND_LIMIT: ${TIDB_SPEND_LIMIT:-100} + CHROMA_HOST: ${CHROMA_HOST:-127.0.0.1} + CHROMA_PORT: ${CHROMA_PORT:-8000} + CHROMA_TENANT: ${CHROMA_TENANT:-default_tenant} + CHROMA_DATABASE: ${CHROMA_DATABASE:-default_database} + CHROMA_AUTH_PROVIDER: ${CHROMA_AUTH_PROVIDER:-chromadb.auth.token_authn.TokenAuthClientProvider} + CHROMA_AUTH_CREDENTIALS: ${CHROMA_AUTH_CREDENTIALS:-} + ORACLE_USER: ${ORACLE_USER:-dify} + ORACLE_PASSWORD: ${ORACLE_PASSWORD:-dify} + ORACLE_DSN: ${ORACLE_DSN:-oracle:1521/FREEPDB1} + ORACLE_CONFIG_DIR: ${ORACLE_CONFIG_DIR:-/app/api/storage/wallet} + ORACLE_WALLET_LOCATION: ${ORACLE_WALLET_LOCATION:-/app/api/storage/wallet} + ORACLE_WALLET_PASSWORD: ${ORACLE_WALLET_PASSWORD:-dify} + ORACLE_IS_AUTONOMOUS: ${ORACLE_IS_AUTONOMOUS:-false} + RELYT_HOST: ${RELYT_HOST:-db} + RELYT_PORT: ${RELYT_PORT:-5432} + RELYT_USER: ${RELYT_USER:-postgres} + RELYT_PASSWORD: ${RELYT_PASSWORD:-difyai123456} + RELYT_DATABASE: ${RELYT_DATABASE:-postgres} + OPENSEARCH_HOST: ${OPENSEARCH_HOST:-opensearch} + OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200} + OPENSEARCH_SECURE: ${OPENSEARCH_SECURE:-true} + OPENSEARCH_VERIFY_CERTS: ${OPENSEARCH_VERIFY_CERTS:-true} + OPENSEARCH_AUTH_METHOD: ${OPENSEARCH_AUTH_METHOD:-basic} + OPENSEARCH_USER: ${OPENSEARCH_USER:-admin} + OPENSEARCH_PASSWORD: ${OPENSEARCH_PASSWORD:-admin} + OPENSEARCH_AWS_REGION: ${OPENSEARCH_AWS_REGION:-ap-southeast-1} + OPENSEARCH_AWS_SERVICE: ${OPENSEARCH_AWS_SERVICE:-aoss} + TENCENT_VECTOR_DB_URL: ${TENCENT_VECTOR_DB_URL:-http://127.0.0.1} + TENCENT_VECTOR_DB_API_KEY: ${TENCENT_VECTOR_DB_API_KEY:-dify} + TENCENT_VECTOR_DB_TIMEOUT: ${TENCENT_VECTOR_DB_TIMEOUT:-30} + TENCENT_VECTOR_DB_USERNAME: ${TENCENT_VECTOR_DB_USERNAME:-dify} + TENCENT_VECTOR_DB_DATABASE: ${TENCENT_VECTOR_DB_DATABASE:-dify} + TENCENT_VECTOR_DB_SHARD: ${TENCENT_VECTOR_DB_SHARD:-1} + TENCENT_VECTOR_DB_REPLICAS: ${TENCENT_VECTOR_DB_REPLICAS:-2} + TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH: ${TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH:-false} + ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-0.0.0.0} + ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} + ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_USERNAME:-elastic} + ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-elastic} + KIBANA_PORT: ${KIBANA_PORT:-5601} + ELASTICSEARCH_USE_CLOUD: ${ELASTICSEARCH_USE_CLOUD:-false} + ELASTICSEARCH_CLOUD_URL: ${ELASTICSEARCH_CLOUD_URL:-YOUR-ELASTICSEARCH_CLOUD_URL} + ELASTICSEARCH_API_KEY: ${ELASTICSEARCH_API_KEY:-YOUR-ELASTICSEARCH_API_KEY} + ELASTICSEARCH_VERIFY_CERTS: ${ELASTICSEARCH_VERIFY_CERTS:-False} + ELASTICSEARCH_CA_CERTS: ${ELASTICSEARCH_CA_CERTS:-} + ELASTICSEARCH_REQUEST_TIMEOUT: ${ELASTICSEARCH_REQUEST_TIMEOUT:-100000} + ELASTICSEARCH_RETRY_ON_TIMEOUT: ${ELASTICSEARCH_RETRY_ON_TIMEOUT:-True} + ELASTICSEARCH_MAX_RETRIES: ${ELASTICSEARCH_MAX_RETRIES:-10} + BAIDU_VECTOR_DB_ENDPOINT: ${BAIDU_VECTOR_DB_ENDPOINT:-http://127.0.0.1:5287} + BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS: ${BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS:-30000} + BAIDU_VECTOR_DB_ACCOUNT: ${BAIDU_VECTOR_DB_ACCOUNT:-root} + BAIDU_VECTOR_DB_API_KEY: ${BAIDU_VECTOR_DB_API_KEY:-dify} + BAIDU_VECTOR_DB_DATABASE: ${BAIDU_VECTOR_DB_DATABASE:-dify} + BAIDU_VECTOR_DB_SHARD: ${BAIDU_VECTOR_DB_SHARD:-1} + BAIDU_VECTOR_DB_REPLICAS: ${BAIDU_VECTOR_DB_REPLICAS:-3} + VIKINGDB_ACCESS_KEY: ${VIKINGDB_ACCESS_KEY:-your-ak} + VIKINGDB_SECRET_KEY: ${VIKINGDB_SECRET_KEY:-your-sk} + VIKINGDB_REGION: ${VIKINGDB_REGION:-cn-shanghai} + VIKINGDB_HOST: ${VIKINGDB_HOST:-api-vikingdb.xxx.volces.com} + VIKINGDB_SCHEMA: ${VIKINGDB_SCHEMA:-http} + VIKINGDB_CONNECTION_TIMEOUT: ${VIKINGDB_CONNECTION_TIMEOUT:-30} + VIKINGDB_SOCKET_TIMEOUT: ${VIKINGDB_SOCKET_TIMEOUT:-30} + LINDORM_URL: ${LINDORM_URL:-http://lindorm:30070} + LINDORM_USERNAME: ${LINDORM_USERNAME:-lindorm} + LINDORM_PASSWORD: ${LINDORM_PASSWORD:-lindorm} + LINDORM_QUERY_TIMEOUT: ${LINDORM_QUERY_TIMEOUT:-1} + OCEANBASE_VECTOR_HOST: ${OCEANBASE_VECTOR_HOST:-oceanbase} + OCEANBASE_VECTOR_PORT: ${OCEANBASE_VECTOR_PORT:-2881} + OCEANBASE_VECTOR_USER: ${OCEANBASE_VECTOR_USER:-root@test} + OCEANBASE_VECTOR_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} + OCEANBASE_VECTOR_DATABASE: ${OCEANBASE_VECTOR_DATABASE:-test} + OCEANBASE_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} + OCEANBASE_MEMORY_LIMIT: ${OCEANBASE_MEMORY_LIMIT:-6G} + OCEANBASE_ENABLE_HYBRID_SEARCH: ${OCEANBASE_ENABLE_HYBRID_SEARCH:-false} + OPENGAUSS_HOST: ${OPENGAUSS_HOST:-opengauss} + OPENGAUSS_PORT: ${OPENGAUSS_PORT:-6600} + OPENGAUSS_USER: ${OPENGAUSS_USER:-postgres} + OPENGAUSS_PASSWORD: ${OPENGAUSS_PASSWORD:-Dify@123} + OPENGAUSS_DATABASE: ${OPENGAUSS_DATABASE:-dify} + OPENGAUSS_MIN_CONNECTION: ${OPENGAUSS_MIN_CONNECTION:-1} + OPENGAUSS_MAX_CONNECTION: ${OPENGAUSS_MAX_CONNECTION:-5} + OPENGAUSS_ENABLE_PQ: ${OPENGAUSS_ENABLE_PQ:-false} + HUAWEI_CLOUD_HOSTS: ${HUAWEI_CLOUD_HOSTS:-https://127.0.0.1:9200} + HUAWEI_CLOUD_USER: ${HUAWEI_CLOUD_USER:-admin} + HUAWEI_CLOUD_PASSWORD: ${HUAWEI_CLOUD_PASSWORD:-admin} + UPSTASH_VECTOR_URL: ${UPSTASH_VECTOR_URL:-https://xxx-vector.upstash.io} + UPSTASH_VECTOR_TOKEN: ${UPSTASH_VECTOR_TOKEN:-dify} + TABLESTORE_ENDPOINT: ${TABLESTORE_ENDPOINT:-https://instance-name.cn-hangzhou.ots.aliyuncs.com} + TABLESTORE_INSTANCE_NAME: ${TABLESTORE_INSTANCE_NAME:-instance-name} + TABLESTORE_ACCESS_KEY_ID: ${TABLESTORE_ACCESS_KEY_ID:-xxx} + TABLESTORE_ACCESS_KEY_SECRET: ${TABLESTORE_ACCESS_KEY_SECRET:-xxx} + TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE: ${TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE:-false} + CLICKZETTA_USERNAME: ${CLICKZETTA_USERNAME:-} + CLICKZETTA_PASSWORD: ${CLICKZETTA_PASSWORD:-} + CLICKZETTA_INSTANCE: ${CLICKZETTA_INSTANCE:-} + CLICKZETTA_SERVICE: ${CLICKZETTA_SERVICE:-api.clickzetta.com} + CLICKZETTA_WORKSPACE: ${CLICKZETTA_WORKSPACE:-quick_start} + CLICKZETTA_VCLUSTER: ${CLICKZETTA_VCLUSTER:-default_ap} + CLICKZETTA_SCHEMA: ${CLICKZETTA_SCHEMA:-dify} + CLICKZETTA_BATCH_SIZE: ${CLICKZETTA_BATCH_SIZE:-100} + CLICKZETTA_ENABLE_INVERTED_INDEX: ${CLICKZETTA_ENABLE_INVERTED_INDEX:-true} + CLICKZETTA_ANALYZER_TYPE: ${CLICKZETTA_ANALYZER_TYPE:-chinese} + CLICKZETTA_ANALYZER_MODE: ${CLICKZETTA_ANALYZER_MODE:-smart} + CLICKZETTA_VECTOR_DISTANCE_FUNCTION: ${CLICKZETTA_VECTOR_DISTANCE_FUNCTION:-cosine_distance} + UPLOAD_FILE_SIZE_LIMIT: ${UPLOAD_FILE_SIZE_LIMIT:-15} + UPLOAD_FILE_BATCH_LIMIT: ${UPLOAD_FILE_BATCH_LIMIT:-5} + ETL_TYPE: ${ETL_TYPE:-dify} + UNSTRUCTURED_API_URL: ${UNSTRUCTURED_API_URL:-} + UNSTRUCTURED_API_KEY: ${UNSTRUCTURED_API_KEY:-} + SCARF_NO_ANALYTICS: ${SCARF_NO_ANALYTICS:-true} + PROMPT_GENERATION_MAX_TOKENS: ${PROMPT_GENERATION_MAX_TOKENS:-512} + CODE_GENERATION_MAX_TOKENS: ${CODE_GENERATION_MAX_TOKENS:-1024} + PLUGIN_BASED_TOKEN_COUNTING_ENABLED: ${PLUGIN_BASED_TOKEN_COUNTING_ENABLED:-false} + MULTIMODAL_SEND_FORMAT: ${MULTIMODAL_SEND_FORMAT:-base64} + UPLOAD_IMAGE_FILE_SIZE_LIMIT: ${UPLOAD_IMAGE_FILE_SIZE_LIMIT:-10} + UPLOAD_VIDEO_FILE_SIZE_LIMIT: ${UPLOAD_VIDEO_FILE_SIZE_LIMIT:-100} + UPLOAD_AUDIO_FILE_SIZE_LIMIT: ${UPLOAD_AUDIO_FILE_SIZE_LIMIT:-50} + SENTRY_DSN: ${SENTRY_DSN:-} + API_SENTRY_DSN: ${API_SENTRY_DSN:-} + API_SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} + API_SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} + WEB_SENTRY_DSN: ${WEB_SENTRY_DSN:-} + PLUGIN_SENTRY_ENABLED: ${PLUGIN_SENTRY_ENABLED:-false} + PLUGIN_SENTRY_DSN: ${PLUGIN_SENTRY_DSN:-} + NOTION_INTEGRATION_TYPE: ${NOTION_INTEGRATION_TYPE:-public} + NOTION_CLIENT_SECRET: ${NOTION_CLIENT_SECRET:-} + NOTION_CLIENT_ID: ${NOTION_CLIENT_ID:-} + NOTION_INTERNAL_SECRET: ${NOTION_INTERNAL_SECRET:-} + MAIL_TYPE: ${MAIL_TYPE:-resend} + MAIL_DEFAULT_SEND_FROM: ${MAIL_DEFAULT_SEND_FROM:-} + RESEND_API_URL: ${RESEND_API_URL:-https://api.resend.com} + RESEND_API_KEY: ${RESEND_API_KEY:-your-resend-api-key} + SMTP_SERVER: ${SMTP_SERVER:-} + SMTP_PORT: ${SMTP_PORT:-465} + SMTP_USERNAME: ${SMTP_USERNAME:-} + SMTP_PASSWORD: ${SMTP_PASSWORD:-} + SMTP_USE_TLS: ${SMTP_USE_TLS:-true} + SMTP_OPPORTUNISTIC_TLS: ${SMTP_OPPORTUNISTIC_TLS:-false} + SENDGRID_API_KEY: ${SENDGRID_API_KEY:-} + INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000} + INVITE_EXPIRY_HOURS: ${INVITE_EXPIRY_HOURS:-72} + RESET_PASSWORD_TOKEN_EXPIRY_MINUTES: ${RESET_PASSWORD_TOKEN_EXPIRY_MINUTES:-5} + CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES: ${CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES:-5} + OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES: ${OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES:-5} + CODE_EXECUTION_ENDPOINT: ${CODE_EXECUTION_ENDPOINT:-http://sandbox:8194} + CODE_EXECUTION_API_KEY: ${CODE_EXECUTION_API_KEY:-dify-sandbox} + CODE_MAX_NUMBER: ${CODE_MAX_NUMBER:-9223372036854775807} + CODE_MIN_NUMBER: ${CODE_MIN_NUMBER:--9223372036854775808} + CODE_MAX_DEPTH: ${CODE_MAX_DEPTH:-5} + CODE_MAX_PRECISION: ${CODE_MAX_PRECISION:-20} + CODE_MAX_STRING_LENGTH: ${CODE_MAX_STRING_LENGTH:-80000} + CODE_MAX_STRING_ARRAY_LENGTH: ${CODE_MAX_STRING_ARRAY_LENGTH:-30} + CODE_MAX_OBJECT_ARRAY_LENGTH: ${CODE_MAX_OBJECT_ARRAY_LENGTH:-30} + CODE_MAX_NUMBER_ARRAY_LENGTH: ${CODE_MAX_NUMBER_ARRAY_LENGTH:-1000} + CODE_EXECUTION_CONNECT_TIMEOUT: ${CODE_EXECUTION_CONNECT_TIMEOUT:-10} + CODE_EXECUTION_READ_TIMEOUT: ${CODE_EXECUTION_READ_TIMEOUT:-60} + CODE_EXECUTION_WRITE_TIMEOUT: ${CODE_EXECUTION_WRITE_TIMEOUT:-10} + TEMPLATE_TRANSFORM_MAX_LENGTH: ${TEMPLATE_TRANSFORM_MAX_LENGTH:-80000} + WORKFLOW_MAX_EXECUTION_STEPS: ${WORKFLOW_MAX_EXECUTION_STEPS:-500} + WORKFLOW_MAX_EXECUTION_TIME: ${WORKFLOW_MAX_EXECUTION_TIME:-1200} + WORKFLOW_CALL_MAX_DEPTH: ${WORKFLOW_CALL_MAX_DEPTH:-5} + MAX_VARIABLE_SIZE: ${MAX_VARIABLE_SIZE:-204800} + WORKFLOW_PARALLEL_DEPTH_LIMIT: ${WORKFLOW_PARALLEL_DEPTH_LIMIT:-3} + WORKFLOW_FILE_UPLOAD_LIMIT: ${WORKFLOW_FILE_UPLOAD_LIMIT:-10} + WORKFLOW_NODE_EXECUTION_STORAGE: ${WORKFLOW_NODE_EXECUTION_STORAGE:-rdbms} + CORE_WORKFLOW_EXECUTION_REPOSITORY: ${CORE_WORKFLOW_EXECUTION_REPOSITORY:-core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository} + CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY:-core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository} + API_WORKFLOW_RUN_REPOSITORY: ${API_WORKFLOW_RUN_REPOSITORY:-repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository} + API_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${API_WORKFLOW_NODE_EXECUTION_REPOSITORY:-repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository} + WORKFLOW_LOG_CLEANUP_ENABLED: ${WORKFLOW_LOG_CLEANUP_ENABLED:-false} + WORKFLOW_LOG_RETENTION_DAYS: ${WORKFLOW_LOG_RETENTION_DAYS:-30} + WORKFLOW_LOG_CLEANUP_BATCH_SIZE: ${WORKFLOW_LOG_CLEANUP_BATCH_SIZE:-100} + HTTP_REQUEST_NODE_MAX_BINARY_SIZE: ${HTTP_REQUEST_NODE_MAX_BINARY_SIZE:-10485760} + HTTP_REQUEST_NODE_MAX_TEXT_SIZE: ${HTTP_REQUEST_NODE_MAX_TEXT_SIZE:-1048576} + HTTP_REQUEST_NODE_SSL_VERIFY: ${HTTP_REQUEST_NODE_SSL_VERIFY:-True} + RESPECT_XFORWARD_HEADERS_ENABLED: ${RESPECT_XFORWARD_HEADERS_ENABLED:-false} + SSRF_PROXY_HTTP_URL: ${SSRF_PROXY_HTTP_URL:-http://ssrf_proxy:3128} + SSRF_PROXY_HTTPS_URL: ${SSRF_PROXY_HTTPS_URL:-http://ssrf_proxy:3128} + LOOP_NODE_MAX_COUNT: ${LOOP_NODE_MAX_COUNT:-100} + MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} + MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} + MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99} + TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} + ALLOW_UNSAFE_DATA_SCHEME: ${ALLOW_UNSAFE_DATA_SCHEME:-false} + MAX_TREE_DEPTH: ${MAX_TREE_DEPTH:-50} + POSTGRES_USER: ${POSTGRES_USER:-${DB_USERNAME}} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-${DB_PASSWORD}} + POSTGRES_DB: ${POSTGRES_DB:-${DB_DATABASE}} + PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata} + SANDBOX_API_KEY: ${SANDBOX_API_KEY:-dify-sandbox} + SANDBOX_GIN_MODE: ${SANDBOX_GIN_MODE:-release} + SANDBOX_WORKER_TIMEOUT: ${SANDBOX_WORKER_TIMEOUT:-15} + SANDBOX_ENABLE_NETWORK: ${SANDBOX_ENABLE_NETWORK:-true} + SANDBOX_HTTP_PROXY: ${SANDBOX_HTTP_PROXY:-http://ssrf_proxy:3128} + SANDBOX_HTTPS_PROXY: ${SANDBOX_HTTPS_PROXY:-http://ssrf_proxy:3128} + SANDBOX_PORT: ${SANDBOX_PORT:-8194} + WEAVIATE_PERSISTENCE_DATA_PATH: ${WEAVIATE_PERSISTENCE_DATA_PATH:-/var/lib/weaviate} + WEAVIATE_QUERY_DEFAULTS_LIMIT: ${WEAVIATE_QUERY_DEFAULTS_LIMIT:-25} + WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: ${WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED:-true} + WEAVIATE_DEFAULT_VECTORIZER_MODULE: ${WEAVIATE_DEFAULT_VECTORIZER_MODULE:-none} + WEAVIATE_CLUSTER_HOSTNAME: ${WEAVIATE_CLUSTER_HOSTNAME:-node1} + WEAVIATE_AUTHENTICATION_APIKEY_ENABLED: ${WEAVIATE_AUTHENTICATION_APIKEY_ENABLED:-true} + WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS: ${WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} + WEAVIATE_AUTHENTICATION_APIKEY_USERS: ${WEAVIATE_AUTHENTICATION_APIKEY_USERS:-hello@dify.ai} + WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED: ${WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED:-true} + WEAVIATE_AUTHORIZATION_ADMINLIST_USERS: ${WEAVIATE_AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai} + CHROMA_SERVER_AUTHN_CREDENTIALS: ${CHROMA_SERVER_AUTHN_CREDENTIALS:-difyai123456} + CHROMA_SERVER_AUTHN_PROVIDER: ${CHROMA_SERVER_AUTHN_PROVIDER:-chromadb.auth.token_authn.TokenAuthenticationServerProvider} + CHROMA_IS_PERSISTENT: ${CHROMA_IS_PERSISTENT:-TRUE} + ORACLE_PWD: ${ORACLE_PWD:-Dify123456} + ORACLE_CHARACTERSET: ${ORACLE_CHARACTERSET:-AL32UTF8} + ETCD_AUTO_COMPACTION_MODE: ${ETCD_AUTO_COMPACTION_MODE:-revision} + ETCD_AUTO_COMPACTION_RETENTION: ${ETCD_AUTO_COMPACTION_RETENTION:-1000} + ETCD_QUOTA_BACKEND_BYTES: ${ETCD_QUOTA_BACKEND_BYTES:-4294967296} + ETCD_SNAPSHOT_COUNT: ${ETCD_SNAPSHOT_COUNT:-50000} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-S3User.MinIO} + ETCD_ENDPOINTS: ${ETCD_ENDPOINTS:-etcd:2379} + MINIO_ADDRESS: ${MINIO_ADDRESS:-minio:9000} + MILVUS_AUTHORIZATION_ENABLED: ${MILVUS_AUTHORIZATION_ENABLED:-true} + PGVECTOR_PGUSER: ${PGVECTOR_PGUSER:-postgres} + PGVECTOR_POSTGRES_PASSWORD: ${PGVECTOR_POSTGRES_PASSWORD:-difyai123456} + PGVECTOR_POSTGRES_DB: ${PGVECTOR_POSTGRES_DB:-dify} + PGVECTOR_PGDATA: ${PGVECTOR_PGDATA:-/var/lib/postgresql/data/pgdata} + OPENSEARCH_DISCOVERY_TYPE: ${OPENSEARCH_DISCOVERY_TYPE:-single-node} + OPENSEARCH_BOOTSTRAP_MEMORY_LOCK: ${OPENSEARCH_BOOTSTRAP_MEMORY_LOCK:-true} + OPENSEARCH_JAVA_OPTS_MIN: ${OPENSEARCH_JAVA_OPTS_MIN:-512m} + OPENSEARCH_JAVA_OPTS_MAX: ${OPENSEARCH_JAVA_OPTS_MAX:-1024m} + OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-Qazwsxedc!@#123} + OPENSEARCH_MEMLOCK_SOFT: ${OPENSEARCH_MEMLOCK_SOFT:--1} + OPENSEARCH_MEMLOCK_HARD: ${OPENSEARCH_MEMLOCK_HARD:--1} + OPENSEARCH_NOFILE_SOFT: ${OPENSEARCH_NOFILE_SOFT:-65536} + OPENSEARCH_NOFILE_HARD: ${OPENSEARCH_NOFILE_HARD:-65536} + NGINX_SERVER_NAME: ${NGINX_SERVER_NAME:-_} + NGINX_HTTPS_ENABLED: ${NGINX_HTTPS_ENABLED:-false} + NGINX_PORT: ${NGINX_PORT:-80} + NGINX_SSL_PORT: ${NGINX_SSL_PORT:-443} + NGINX_SSL_CERT_FILENAME: ${NGINX_SSL_CERT_FILENAME:-dify.crt} + NGINX_SSL_CERT_KEY_FILENAME: ${NGINX_SSL_CERT_KEY_FILENAME:-dify.key} + NGINX_SSL_PROTOCOLS: ${NGINX_SSL_PROTOCOLS:-TLSv1.1 TLSv1.2 TLSv1.3} + NGINX_WORKER_PROCESSES: ${NGINX_WORKER_PROCESSES:-auto} + NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-100M} + NGINX_KEEPALIVE_TIMEOUT: ${NGINX_KEEPALIVE_TIMEOUT:-65} + NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s} + NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s} + NGINX_ENABLE_CERTBOT_CHALLENGE: ${NGINX_ENABLE_CERTBOT_CHALLENGE:-false} + CERTBOT_EMAIL: ${CERTBOT_EMAIL:-your_email@example.com} + CERTBOT_DOMAIN: ${CERTBOT_DOMAIN:-your_domain.com} + CERTBOT_OPTIONS: ${CERTBOT_OPTIONS:-} + SSRF_HTTP_PORT: ${SSRF_HTTP_PORT:-3128} + SSRF_COREDUMP_DIR: ${SSRF_COREDUMP_DIR:-/var/spool/squid} + SSRF_REVERSE_PROXY_PORT: ${SSRF_REVERSE_PROXY_PORT:-8194} + SSRF_SANDBOX_HOST: ${SSRF_SANDBOX_HOST:-sandbox} + SSRF_DEFAULT_TIME_OUT: ${SSRF_DEFAULT_TIME_OUT:-5} + SSRF_DEFAULT_CONNECT_TIME_OUT: ${SSRF_DEFAULT_CONNECT_TIME_OUT:-5} + SSRF_DEFAULT_READ_TIME_OUT: ${SSRF_DEFAULT_READ_TIME_OUT:-5} + SSRF_DEFAULT_WRITE_TIME_OUT: ${SSRF_DEFAULT_WRITE_TIME_OUT:-5} + EXPOSE_NGINX_PORT: ${EXPOSE_NGINX_PORT:-80} + EXPOSE_NGINX_SSL_PORT: ${EXPOSE_NGINX_SSL_PORT:-443} + POSITION_TOOL_PINS: ${POSITION_TOOL_PINS:-} + POSITION_TOOL_INCLUDES: ${POSITION_TOOL_INCLUDES:-} + POSITION_TOOL_EXCLUDES: ${POSITION_TOOL_EXCLUDES:-} + POSITION_PROVIDER_PINS: ${POSITION_PROVIDER_PINS:-} + POSITION_PROVIDER_INCLUDES: ${POSITION_PROVIDER_INCLUDES:-} + POSITION_PROVIDER_EXCLUDES: ${POSITION_PROVIDER_EXCLUDES:-} + CSP_WHITELIST: ${CSP_WHITELIST:-} + CREATE_TIDB_SERVICE_JOB_ENABLED: ${CREATE_TIDB_SERVICE_JOB_ENABLED:-false} + MAX_SUBMIT_COUNT: ${MAX_SUBMIT_COUNT:-100} + TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-10} + DB_PLUGIN_DATABASE: ${DB_PLUGIN_DATABASE:-dify_plugin} + EXPOSE_PLUGIN_DAEMON_PORT: ${EXPOSE_PLUGIN_DAEMON_PORT:-5002} + PLUGIN_DAEMON_PORT: ${PLUGIN_DAEMON_PORT:-5002} + PLUGIN_DAEMON_KEY: ${PLUGIN_DAEMON_KEY:-lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi} + PLUGIN_DAEMON_URL: ${PLUGIN_DAEMON_URL:-http://plugin_daemon:5002} + PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + PLUGIN_PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false} + PLUGIN_DEBUGGING_HOST: ${PLUGIN_DEBUGGING_HOST:-0.0.0.0} + PLUGIN_DEBUGGING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003} + EXPOSE_PLUGIN_DEBUGGING_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost} + EXPOSE_PLUGIN_DEBUGGING_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003} + PLUGIN_DIFY_INNER_API_KEY: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + PLUGIN_DIFY_INNER_API_URL: ${PLUGIN_DIFY_INNER_API_URL:-http://api:5001} + ENDPOINT_URL_TEMPLATE: ${ENDPOINT_URL_TEMPLATE:-http://localhost/e/{hook_id}} + MARKETPLACE_ENABLED: ${MARKETPLACE_ENABLED:-true} + MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} + FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + PLUGIN_STDIO_BUFFER_SIZE: ${PLUGIN_STDIO_BUFFER_SIZE:-1024} + PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880} + PLUGIN_PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} + PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} + PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} + PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} + PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} + PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} + PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} + PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} + PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + PLUGIN_S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false} + PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} + PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} + PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} + PLUGIN_AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} + PLUGIN_AWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} + PLUGIN_AWS_REGION: ${PLUGIN_AWS_REGION:-} + PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-} + PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-} + PLUGIN_TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-} + PLUGIN_TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-} + PLUGIN_TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-} + PLUGIN_ALIYUN_OSS_REGION: ${PLUGIN_ALIYUN_OSS_REGION:-} + PLUGIN_ALIYUN_OSS_ENDPOINT: ${PLUGIN_ALIYUN_OSS_ENDPOINT:-} + PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID:-} + PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET:-} + PLUGIN_ALIYUN_OSS_AUTH_VERSION: ${PLUGIN_ALIYUN_OSS_AUTH_VERSION:-v4} + PLUGIN_ALIYUN_OSS_PATH: ${PLUGIN_ALIYUN_OSS_PATH:-} + PLUGIN_VOLCENGINE_TOS_ENDPOINT: ${PLUGIN_VOLCENGINE_TOS_ENDPOINT:-} + PLUGIN_VOLCENGINE_TOS_ACCESS_KEY: ${PLUGIN_VOLCENGINE_TOS_ACCESS_KEY:-} + PLUGIN_VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-} + PLUGIN_VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-} + ENABLE_OTEL: ${ENABLE_OTEL:-false} + OTLP_TRACE_ENDPOINT: ${OTLP_TRACE_ENDPOINT:-} + OTLP_METRIC_ENDPOINT: ${OTLP_METRIC_ENDPOINT:-} + OTLP_BASE_ENDPOINT: ${OTLP_BASE_ENDPOINT:-http://localhost:4318} + OTLP_API_KEY: ${OTLP_API_KEY:-} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-} + OTEL_EXPORTER_TYPE: ${OTEL_EXPORTER_TYPE:-otlp} + OTEL_SAMPLING_RATE: ${OTEL_SAMPLING_RATE:-0.1} + OTEL_BATCH_EXPORT_SCHEDULE_DELAY: ${OTEL_BATCH_EXPORT_SCHEDULE_DELAY:-5000} + OTEL_MAX_QUEUE_SIZE: ${OTEL_MAX_QUEUE_SIZE:-2048} + OTEL_MAX_EXPORT_BATCH_SIZE: ${OTEL_MAX_EXPORT_BATCH_SIZE:-512} + OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-60000} + OTEL_BATCH_EXPORT_TIMEOUT: ${OTEL_BATCH_EXPORT_TIMEOUT:-10000} + OTEL_METRIC_EXPORT_TIMEOUT: ${OTEL_METRIC_EXPORT_TIMEOUT:-30000} + ALLOW_EMBED: ${ALLOW_EMBED:-false} + QUEUE_MONITOR_THRESHOLD: ${QUEUE_MONITOR_THRESHOLD:-200} + QUEUE_MONITOR_ALERT_EMAILS: ${QUEUE_MONITOR_ALERT_EMAILS:-} + QUEUE_MONITOR_INTERVAL: ${QUEUE_MONITOR_INTERVAL:-30} + SWAGGER_UI_ENABLED: ${SWAGGER_UI_ENABLED:-true} + SWAGGER_UI_PATH: ${SWAGGER_UI_PATH:-/swagger-ui.html} + ENABLE_CLEAN_EMBEDDING_CACHE_TASK: ${ENABLE_CLEAN_EMBEDDING_CACHE_TASK:-false} + ENABLE_CLEAN_UNUSED_DATASETS_TASK: ${ENABLE_CLEAN_UNUSED_DATASETS_TASK:-false} + ENABLE_CREATE_TIDB_SERVERLESS_TASK: ${ENABLE_CREATE_TIDB_SERVERLESS_TASK:-false} + ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK: ${ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK:-false} + ENABLE_CLEAN_MESSAGES: ${ENABLE_CLEAN_MESSAGES:-false} + ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK: ${ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK:-false} + ENABLE_DATASETS_QUEUE_MONITOR: ${ENABLE_DATASETS_QUEUE_MONITOR:-false} + ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK: ${ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK:-true} + +services: + # API service + api: + image: langgenius/dify-api:1.8.1 + restart: always + environment: + # Use the shared environment variables. + <<: *shared-api-worker-env + # Startup mode, 'api' starts the API server. + MODE: api + SENTRY_DSN: ${API_SENTRY_DSN:-} + SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} + SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} + PLUGIN_REMOTE_INSTALL_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost} + PLUGIN_REMOTE_INSTALL_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003} + PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + depends_on: + redis: + condition: service_started + volumes: + # Mount the storage directory to the container, for storing user files. + #- ./volumes/app/storage:/app/api/storage + - ${DIFY_DATA}/app/storage:/app/api/storage + networks: + - ssrf_proxy_network + - default + + # worker service + # The Celery worker for processing the queue. + worker: + image: langgenius/dify-api:1.8.1 + restart: always + environment: + # Use the shared environment variables. + <<: *shared-api-worker-env + # Startup mode, 'worker' starts the Celery worker for processing the queue. + MODE: worker + SENTRY_DSN: ${API_SENTRY_DSN:-} + SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} + SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} + PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + depends_on: + redis: + condition: service_started + volumes: + # Mount the storage directory to the container, for storing user files. + - ${DIFY_DATA}/app/storage:/app/api/storage #- ./volumes/app/storage:/app/api/storage + networks: + - ssrf_proxy_network + - default + + # worker_beat service + # Celery beat for scheduling periodic tasks. + worker_beat: + image: langgenius/dify-api:1.8.1 + restart: always + environment: + # Use the shared environment variables. + <<: *shared-api-worker-env + # Startup mode, 'worker_beat' starts the Celery beat for scheduling periodic tasks. + MODE: beat + depends_on: + redis: + condition: service_started + networks: + - ssrf_proxy_network + - default + + # Frontend web application. + web: + image: langgenius/dify-web:1.8.1 + restart: always + environment: + CONSOLE_API_URL: ${CONSOLE_API_URL:-} + APP_API_URL: ${APP_API_URL:-} + SENTRY_DSN: ${WEB_SENTRY_DSN:-} + NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-0} + TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} + CSP_WHITELIST: ${CSP_WHITELIST:-} + ALLOW_EMBED: ${ALLOW_EMBED:-false} + ALLOW_UNSAFE_DATA_SCHEME: ${ALLOW_UNSAFE_DATA_SCHEME:-false} + MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} + MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai} + TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-} + INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-} + PM2_INSTANCES: ${PM2_INSTANCES:-2} + LOOP_NODE_MAX_COUNT: ${LOOP_NODE_MAX_COUNT:-100} + MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} + MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} + MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99} + MAX_TREE_DEPTH: ${MAX_TREE_DEPTH:-50} + ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} + ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} + ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} + + + # The redis cache. + redis: + image: redis:6-alpine + restart: always + environment: + REDISCLI_AUTH: ${REDIS_PASSWORD:-difyai123456} + volumes: + # Mount the redis data directory to the container. + - ./volumes/redis/data:/data + # Set the redis password when startup redis server. + command: redis-server --requirepass ${REDIS_PASSWORD:-difyai123456} + healthcheck: + test: ["CMD-SHELL", "redis-cli -a ${REDIS_PASSWORD:-difyai123456} ping | grep -q PONG"] + + # The DifySandbox + sandbox: + image: langgenius/dify-sandbox:0.2.12 + restart: always + environment: + # The DifySandbox configurations + # Make sure you are changing this key for your deployment with a strong key. + # You can generate a strong key using `openssl rand -base64 42`. + API_KEY: ${SANDBOX_API_KEY:-dify-sandbox} + GIN_MODE: ${SANDBOX_GIN_MODE:-release} + WORKER_TIMEOUT: ${SANDBOX_WORKER_TIMEOUT:-15} + ENABLE_NETWORK: ${SANDBOX_ENABLE_NETWORK:-true} + HTTP_PROXY: ${SANDBOX_HTTP_PROXY:-http://ssrf_proxy:3128} + HTTPS_PROXY: ${SANDBOX_HTTPS_PROXY:-http://ssrf_proxy:3128} + SANDBOX_PORT: ${SANDBOX_PORT:-8194} + PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + volumes: + - ./volumes/sandbox/dependencies:/dependencies + - ./volumes/sandbox/conf:/conf + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8194/health"] + networks: + - ssrf_proxy_network + + # plugin daemon + plugin_daemon: + image: langgenius/dify-plugin-daemon:0.2.0-local + restart: always + environment: + # Use the shared environment variables. + <<: *shared-api-worker-env + DB_DATABASE: ${DB_PLUGIN_DATABASE:-dify_plugin} + SERVER_PORT: ${PLUGIN_DAEMON_PORT:-5002} + SERVER_KEY: ${PLUGIN_DAEMON_KEY:-lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi} + MAX_PLUGIN_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false} + DIFY_INNER_API_URL: ${PLUGIN_DIFY_INNER_API_URL:-http://api:5001} + DIFY_INNER_API_KEY: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_DEBUGGING_HOST:-0.0.0.0} + PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003} + PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} + FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} + PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} + PLUGIN_STDIO_BUFFER_SIZE: ${PLUGIN_STDIO_BUFFER_SIZE:-1024} + PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880} + PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} + PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} + PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} + PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} + PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} + PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} + S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false} + S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} + AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} + AWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} + AWS_REGION: ${PLUGIN_AWS_REGION:-} + AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-} + AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-} + TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-} + TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-} + TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-} + ALIYUN_OSS_REGION: ${PLUGIN_ALIYUN_OSS_REGION:-} + ALIYUN_OSS_ENDPOINT: ${PLUGIN_ALIYUN_OSS_ENDPOINT:-} + ALIYUN_OSS_ACCESS_KEY_ID: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID:-} + ALIYUN_OSS_ACCESS_KEY_SECRET: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET:-} + ALIYUN_OSS_AUTH_VERSION: ${PLUGIN_ALIYUN_OSS_AUTH_VERSION:-v4} + ALIYUN_OSS_PATH: ${PLUGIN_ALIYUN_OSS_PATH:-} + VOLCENGINE_TOS_ENDPOINT: ${PLUGIN_VOLCENGINE_TOS_ENDPOINT:-} + VOLCENGINE_TOS_ACCESS_KEY: ${PLUGIN_VOLCENGINE_TOS_ACCESS_KEY:-} + VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-} + VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-} + SENTRY_ENABLED: ${PLUGIN_SENTRY_ENABLED:-false} + SENTRY_DSN: ${PLUGIN_SENTRY_DSN:-} + ports: + - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}" + volumes: + #- ./volumes/plugin_daemon:/app/storage + - ${DIFY_DATA}/plugin_daemon:/app/storage + + + # ssrf_proxy server + # for more information, please refer to + # https://docs.dify.ai/learn-more/faq/install-faq#18-why-is-ssrf-proxy-needed%3F + ssrf_proxy: + image: ubuntu/squid:latest + restart: always + volumes: + - ./ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template + - ./ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint-mount.sh + entrypoint: ["sh", "-c", "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh"] + environment: + # pls clearly modify the squid env vars to fit your network environment. + HTTP_PORT: ${SSRF_HTTP_PORT:-3128} + COREDUMP_DIR: ${SSRF_COREDUMP_DIR:-/var/spool/squid} + REVERSE_PROXY_PORT: ${SSRF_REVERSE_PROXY_PORT:-8194} + SANDBOX_HOST: ${SSRF_SANDBOX_HOST:-sandbox} + SANDBOX_PORT: ${SANDBOX_PORT:-8194} + networks: + - ssrf_proxy_network + - default + + # The nginx reverse proxy. + # used for reverse proxying the API service and Web service. + nginx: + image: nginx:latest + restart: always + volumes: + - ./nginx/nginx.conf.template:/etc/nginx/nginx.conf.template + - ./nginx/proxy.conf.template:/etc/nginx/proxy.conf.template + - ./nginx/https.conf.template:/etc/nginx/https.conf.template + - ./nginx/conf.d:/etc/nginx/conf.d + - ./nginx/docker-entrypoint.sh:/docker-entrypoint-mount.sh + - ./nginx/ssl:/etc/ssl # cert dir (legacy) + - ./volumes/certbot/conf/live:/etc/letsencrypt/live # cert dir (with certbot container) + - ./volumes/certbot/conf:/etc/letsencrypt + - ./volumes/certbot/www:/var/www/html + entrypoint: + [ + "sh", + "-c", + "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh", + ] + environment: + NGINX_SERVER_NAME: ${NGINX_SERVER_NAME:-_} + NGINX_HTTPS_ENABLED: ${NGINX_HTTPS_ENABLED:-false} + NGINX_SSL_PORT: ${NGINX_SSL_PORT:-443} + NGINX_PORT: ${NGINX_PORT:-80} + # You're required to add your own SSL certificates/keys to the `./nginx/ssl` directory + # and modify the env vars below in .env if HTTPS_ENABLED is true. + NGINX_SSL_CERT_FILENAME: ${NGINX_SSL_CERT_FILENAME:-dify.crt} + NGINX_SSL_CERT_KEY_FILENAME: ${NGINX_SSL_CERT_KEY_FILENAME:-dify.key} + NGINX_SSL_PROTOCOLS: ${NGINX_SSL_PROTOCOLS:-TLSv1.1 TLSv1.2 TLSv1.3} + NGINX_WORKER_PROCESSES: ${NGINX_WORKER_PROCESSES:-auto} + NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-100M} + NGINX_KEEPALIVE_TIMEOUT: ${NGINX_KEEPALIVE_TIMEOUT:-65} + NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s} + NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s} + NGINX_ENABLE_CERTBOT_CHALLENGE: ${NGINX_ENABLE_CERTBOT_CHALLENGE:-false} + CERTBOT_DOMAIN: ${CERTBOT_DOMAIN:-} + depends_on: + - api + - web + ports: # <------------ We expose the ports here + - '${DIFY_PORT:-80}:${NGINX_PORT:-80}' + #- '${EXPOSE_NGINX_SSL_PORT:-443}:${NGINX_SSL_PORT:-443}' + + +networks: + ssrf_proxy_network: + driver: bridge + internal: true diff --git a/app/dify/nginx/conf.d/default.conf.template b/app/dify/nginx/conf.d/default.conf.template new file mode 100644 index 0000000..a458412 --- /dev/null +++ b/app/dify/nginx/conf.d/default.conf.template @@ -0,0 +1,48 @@ +# Please do not directly edit this file. Instead, modify the .env variables related to NGINX configuration. + +server { + listen ${NGINX_PORT}; + server_name ${NGINX_SERVER_NAME}; + + location /console/api { + proxy_pass http://api:5001; + include proxy.conf; + } + + location /api { + proxy_pass http://api:5001; + include proxy.conf; + } + + location /v1 { + proxy_pass http://api:5001; + include proxy.conf; + } + + location /files { + proxy_pass http://api:5001; + include proxy.conf; + } + + location /explore { + proxy_pass http://web:3000; + include proxy.conf; + } + + location /e/ { + proxy_pass http://plugin_daemon:5002; + proxy_set_header Dify-Hook-Url $scheme://$host$request_uri; + include proxy.conf; + } + + location / { + proxy_pass http://web:3000; + include proxy.conf; + } + + # placeholder for acme challenge location + ${ACME_CHALLENGE_LOCATION} + + # placeholder for https config defined in https.conf.template + ${HTTPS_CONFIG} +} diff --git a/app/dify/nginx/docker-entrypoint.sh b/app/dify/nginx/docker-entrypoint.sh new file mode 100755 index 0000000..8e1110f --- /dev/null +++ b/app/dify/nginx/docker-entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +HTTPS_CONFIG='' + +if [ "${NGINX_HTTPS_ENABLED}" = "true" ]; then + # Check if the certificate and key files for the specified domain exist + if [ -n "${CERTBOT_DOMAIN}" ] && \ + [ -f "/etc/letsencrypt/live/${CERTBOT_DOMAIN}/${NGINX_SSL_CERT_FILENAME}" ] && \ + [ -f "/etc/letsencrypt/live/${CERTBOT_DOMAIN}/${NGINX_SSL_CERT_KEY_FILENAME}" ]; then + SSL_CERTIFICATE_PATH="/etc/letsencrypt/live/${CERTBOT_DOMAIN}/${NGINX_SSL_CERT_FILENAME}" + SSL_CERTIFICATE_KEY_PATH="/etc/letsencrypt/live/${CERTBOT_DOMAIN}/${NGINX_SSL_CERT_KEY_FILENAME}" + else + SSL_CERTIFICATE_PATH="/etc/ssl/${NGINX_SSL_CERT_FILENAME}" + SSL_CERTIFICATE_KEY_PATH="/etc/ssl/${NGINX_SSL_CERT_KEY_FILENAME}" + fi + export SSL_CERTIFICATE_PATH + export SSL_CERTIFICATE_KEY_PATH + + # set the HTTPS_CONFIG environment variable to the content of the https.conf.template + HTTPS_CONFIG=$(envsubst < /etc/nginx/https.conf.template) + export HTTPS_CONFIG + # Substitute the HTTPS_CONFIG in the default.conf.template with content from https.conf.template + envsubst '${HTTPS_CONFIG}' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf +fi +export HTTPS_CONFIG + +if [ "${NGINX_ENABLE_CERTBOT_CHALLENGE}" = "true" ]; then + ACME_CHALLENGE_LOCATION='location /.well-known/acme-challenge/ { root /var/www/html; }' +else + ACME_CHALLENGE_LOCATION='' +fi +export ACME_CHALLENGE_LOCATION + +env_vars=$(printenv | cut -d= -f1 | sed 's/^/$/g' | paste -sd, -) + +envsubst "$env_vars" < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf +envsubst "$env_vars" < /etc/nginx/proxy.conf.template > /etc/nginx/proxy.conf + +envsubst "$env_vars" < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf + +# Start Nginx using the default entrypoint +exec nginx -g 'daemon off;' \ No newline at end of file diff --git a/app/dify/nginx/https.conf.template b/app/dify/nginx/https.conf.template new file mode 100644 index 0000000..95ea36f --- /dev/null +++ b/app/dify/nginx/https.conf.template @@ -0,0 +1,9 @@ +# Please do not directly edit this file. Instead, modify the .env variables related to NGINX configuration. + +listen ${NGINX_SSL_PORT} ssl; +ssl_certificate ${SSL_CERTIFICATE_PATH}; +ssl_certificate_key ${SSL_CERTIFICATE_KEY_PATH}; +ssl_protocols ${NGINX_SSL_PROTOCOLS}; +ssl_prefer_server_ciphers on; +ssl_session_cache shared:SSL:10m; +ssl_session_timeout 10m; \ No newline at end of file diff --git a/app/dify/nginx/nginx.conf.template b/app/dify/nginx/nginx.conf.template new file mode 100644 index 0000000..32a5716 --- /dev/null +++ b/app/dify/nginx/nginx.conf.template @@ -0,0 +1,34 @@ +# Please do not directly edit this file. Instead, modify the .env variables related to NGINX configuration. + +user nginx; +worker_processes ${NGINX_WORKER_PROCESSES}; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout ${NGINX_KEEPALIVE_TIMEOUT}; + + #gzip on; + client_max_body_size ${NGINX_CLIENT_MAX_BODY_SIZE}; + + include /etc/nginx/conf.d/*.conf; +} \ No newline at end of file diff --git a/app/dify/nginx/proxy.conf.template b/app/dify/nginx/proxy.conf.template new file mode 100644 index 0000000..fc2daec --- /dev/null +++ b/app/dify/nginx/proxy.conf.template @@ -0,0 +1,11 @@ +# Please do not directly edit this file. Instead, modify the .env variables related to NGINX configuration. + +proxy_set_header Host $host; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; +proxy_set_header X-Forwarded-Port $server_port; +proxy_http_version 1.1; +proxy_set_header Connection ""; +proxy_buffering off; +proxy_read_timeout ${NGINX_PROXY_READ_TIMEOUT}; +proxy_send_timeout ${NGINX_PROXY_SEND_TIMEOUT}; \ No newline at end of file diff --git a/app/dify/ssrf_proxy/docker-entrypoint.sh b/app/dify/ssrf_proxy/docker-entrypoint.sh new file mode 100755 index 0000000..613897b --- /dev/null +++ b/app/dify/ssrf_proxy/docker-entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Modified based on Squid OCI image entrypoint + +# This entrypoint aims to forward the squid logs to stdout to assist users of +# common container related tooling (e.g., kubernetes, docker-compose, etc) to +# access the service logs. + +# Moreover, it invokes the squid binary, leaving all the desired parameters to +# be provided by the "command" passed to the spawned container. If no command +# is provided by the user, the default behavior (as per the CMD statement in +# the Dockerfile) will be to use Ubuntu's default configuration [1] and run +# squid with the "-NYC" options to mimic the behavior of the Ubuntu provided +# systemd unit. + +# [1] The default configuration is changed in the Dockerfile to allow local +# network connections. See the Dockerfile for further information. + +echo "[ENTRYPOINT] re-create snakeoil self-signed certificate removed in the build process" +if [ ! -f /etc/ssl/private/ssl-cert-snakeoil.key ]; then + /usr/sbin/make-ssl-cert generate-default-snakeoil --force-overwrite > /dev/null 2>&1 +fi + +tail -F /var/log/squid/access.log 2>/dev/null & +tail -F /var/log/squid/error.log 2>/dev/null & +tail -F /var/log/squid/store.log 2>/dev/null & +tail -F /var/log/squid/cache.log 2>/dev/null & + +# Replace environment variables in the template and output to the squid.conf +echo "[ENTRYPOINT] replacing environment variables in the template" +awk '{ + while(match($0, /\${[A-Za-z_][A-Za-z_0-9]*}/)) { + var = substr($0, RSTART+2, RLENGTH-3) + val = ENVIRON[var] + $0 = substr($0, 1, RSTART-1) val substr($0, RSTART+RLENGTH) + } + print +}' /etc/squid/squid.conf.template > /etc/squid/squid.conf + +/usr/sbin/squid -Nz +echo "[ENTRYPOINT] starting squid" +/usr/sbin/squid -f /etc/squid/squid.conf -NYC 1 diff --git a/app/dify/ssrf_proxy/squid.conf.template b/app/dify/ssrf_proxy/squid.conf.template new file mode 100644 index 0000000..676fe73 --- /dev/null +++ b/app/dify/ssrf_proxy/squid.conf.template @@ -0,0 +1,51 @@ +acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) +acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) +acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) +acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines +acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) +acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines +acl SSL_ports port 443 +# acl SSL_ports port 1025-65535 # Enable the configuration to resolve this issue: https://github.com/langgenius/dify/issues/12792 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http +acl CONNECT method CONNECT +http_access deny !Safe_ports +http_access deny CONNECT !SSL_ports +http_access allow localhost manager +http_access deny manager +http_access allow localhost +include /etc/squid/conf.d/*.conf +http_access deny all + +################################## Proxy Server ################################ +http_port ${HTTP_PORT} +coredump_dir ${COREDUMP_DIR} +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims +refresh_pattern \/InRelease$ 0 0% 0 refresh-ims +refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +refresh_pattern . 0 20% 4320 + + +# cache_dir ufs /var/spool/squid 100 16 256 +# upstream proxy, set to your own upstream proxy IP to avoid SSRF attacks +# cache_peer 172.1.1.1 parent 3128 0 no-query no-digest no-netdb-exchange default + +################################## Reverse Proxy To Sandbox ################################ +http_port ${REVERSE_PROXY_PORT} accel vhost +cache_peer ${SANDBOX_HOST} parent ${SANDBOX_PORT} 0 no-query originserver +acl src_all src all +http_access allow src_all diff --git a/app/dify/volumes/sandbox/conf/config.yaml b/app/dify/volumes/sandbox/conf/config.yaml new file mode 100644 index 0000000..f6b5803 --- /dev/null +++ b/app/dify/volumes/sandbox/conf/config.yaml @@ -0,0 +1,31 @@ +app: + port: 8194 + debug: True + key: dify-sandbox +max_workers: 4 +max_requests: 50 +worker_timeout: 5 +python_path: /usr/local/bin/python3 +python_lib_path: + - /usr/local/lib/python3.10 + - /usr/lib/python3.10 + - /usr/lib/python3 + - /usr/lib/x86_64-linux-gnu + - /etc/ssl/certs/ca-certificates.crt + - /etc/nsswitch.conf + - /etc/hosts + - /etc/resolv.conf + - /run/systemd/resolve/stub-resolv.conf + - /run/resolvconf/resolv.conf + - /etc/localtime + - /usr/share/zoneinfo + - /etc/timezone + # add more paths if needed +python_pip_mirror_url: https://pypi.tuna.tsinghua.edu.cn/simple +nodejs_path: /usr/local/bin/node +enable_network: True +allowed_syscalls: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499] +proxy: + socks5: '' + http: '' + https: '' diff --git a/app/dify/volumes/sandbox/conf/config.yaml.example b/app/dify/volumes/sandbox/conf/config.yaml.example new file mode 100644 index 0000000..f92c19e --- /dev/null +++ b/app/dify/volumes/sandbox/conf/config.yaml.example @@ -0,0 +1,35 @@ +app: + port: 8194 + debug: True + key: dify-sandbox +max_workers: 4 +max_requests: 50 +worker_timeout: 5 +python_path: /usr/local/bin/python3 +python_lib_path: + - /usr/local/lib/python3.10 + - /usr/lib/python3.10 + - /usr/lib/python3 + - /usr/lib/x86_64-linux-gnu + - /etc/ssl/certs/ca-certificates.crt + - /etc/nsswitch.conf + - /etc/hosts + - /etc/resolv.conf + - /run/systemd/resolve/stub-resolv.conf + - /run/resolvconf/resolv.conf + - /etc/localtime + - /usr/share/zoneinfo + - /etc/timezone + # add more paths if needed +python_pip_mirror_url: https://pypi.tuna.tsinghua.edu.cn/simple +nodejs_path: /usr/local/bin/node +enable_network: True +allowed_syscalls: + - 1 + - 2 + - 3 + # add all the syscalls which you require +proxy: + socks5: '' + http: '' + https: '' diff --git a/app/dify/volumes/sandbox/dependencies/python-requirements.txt b/app/dify/volumes/sandbox/dependencies/python-requirements.txt new file mode 100644 index 0000000..15bb34f --- /dev/null +++ b/app/dify/volumes/sandbox/dependencies/python-requirements.txt @@ -0,0 +1 @@ +#ssrf_proxy \ No newline at end of file diff --git a/app/electric/Makefile b/app/electric/Makefile new file mode 100644 index 0000000..a76302b --- /dev/null +++ b/app/electric/Makefile @@ -0,0 +1,41 @@ +default: up view + +view: + @echo "visit http://10.10.10.10:8002" + +# administration +up: + docker compose up -d +down: + docker compose down +stop: + docker compose stop +start: + docker compose start +restart: + docker compose restart +pause: + docker compose pause +resume: + docker compose unpause +pull: + docker compose pull +rm: + docker compose rm +log: + docker compose logs +logs: + docker compose logs -f +port: + docker compose port +conf: + docker compose config +info: + docker inspect electric | jq +save: + mkdir -p /tmp/docker + docker save electric/electric | gzip -9 -c > /tmp/docker/electric.tgz +load: + cat /tmp/docker/electric.tgz | gzip -d -c - | docker load + +.PHONY: default init view up down stop start restart pause resume pull rm logs port conf info save load \ No newline at end of file diff --git a/app/electric/README.md b/app/electric/README.md new file mode 100644 index 0000000..9372263 --- /dev/null +++ b/app/electric/README.md @@ -0,0 +1,6 @@ +# electric + +[Electric](https://electric-sql.com/), Sync Solved, sync postgres shape to frontend + +> Electric is a Postgres sync engine. It solves the hard problems of sync for you, including[ partial replication](https://electric-sql.com/docs/guides/shapes), [fan-out](https://electric-sql.com/docs/api/http#caching), and [data delivery](https://electric-sql.com/docs/api/http). + diff --git a/app/electric/docker-compose.yml b/app/electric/docker-compose.yml new file mode 100644 index 0000000..84d4109 --- /dev/null +++ b/app/electric/docker-compose.yml @@ -0,0 +1,14 @@ +#!/usr/bin/env docker compose +# please change the credentials in .env file! +# https://github.com/electric-sql/electric/blob/main/website/public/docker-compose.yaml +--- +services: + electric: + container_name: electric + image: electricsql/electric + env_file: + - .env + ports: + - "${ELECTRIC_PORT}:${ELECTRIC_PORT}" + - "${ELECTRIC_PROMETHEUS_PORT}:${ELECTRIC_PROMETHEUS_PORT}" +... \ No newline at end of file diff --git a/app/ferretdb/.env b/app/ferretdb/.env new file mode 100644 index 0000000..647f92f --- /dev/null +++ b/app/ferretdb/.env @@ -0,0 +1,2 @@ +# https://docs.ferretdb.io/configuration/flags/ +FERRETDB_POSTGRESQL_URL=postgres://dbuser_meta:DBUser.Meta@10.10.10.10:5432/meta \ No newline at end of file diff --git a/app/ferretdb/Makefile b/app/ferretdb/Makefile new file mode 100644 index 0000000..533dcf4 --- /dev/null +++ b/app/ferretdb/Makefile @@ -0,0 +1,34 @@ +default: up view + +up: view + docker compose up -d + +view: + @echo "docker run --rm -it --network=ferretdb --entrypoint=mongosh mongo:5 mongodb://ferretdb/" + +log: + docker logs -f ferretdb + +info: + docker inspect ferretdb | jq + +stop: + docker stop ferretdb; + +clean: + docker stop ferretdb; docker rm -f ferretdb + +pull: + docker pull ferretdb/ferretdb + +rmi: + docker rmi ferretdb/ferretdb + +save: + mkdir -p /tmp/docker + docker save quay.io/ferretdb/ferretdb:latest | gzip -9 -c > /tmp/docker/ferretdb.tgz + +load: + cat /tmp/docker/ferretdb.tgz | gzip -d -c - | docker load + +.PHONY: default up run view log info stop clean conf dump pull rmi save load \ No newline at end of file diff --git a/app/ferretdb/README.md b/app/ferretdb/README.md new file mode 100644 index 0000000..a388629 --- /dev/null +++ b/app/ferretdb/README.md @@ -0,0 +1,93 @@ +# FerretDB + +[FerretDB](https://www.ferretdb.io/) -- A truly Open Source MongoDB alternative, built on Postgres. + +```bash +cd app/ferretdb; docker compose up -d +``` + +```bash +make up # pull up ferretdb with docker compose in minimal mode +make run # launch ferretdb with docker , local data dir and external PostgreSQL +make view # print ferretdb access point +make log # tail -f ferretdb logs +make info # introspect ferretdb with jq +make stop # stop ferretdb container +make clean # remove ferretdb container +make pull # pull latest ferretdb image +make rmi # remove ferretdb image +make save # save ferretdb image to /tmp/ferretdb.tgz +make load # load ferretdb image from /tmp +``` + + +## Docker Compose + +```yaml + +services: + ferretdb: + image: quay.io/ferretdb/ferretdb:latest + container_name: ferretdb + restart: on-failure + environment: + - FERRETDB_POSTGRESQL_URL=postgres://dbuser_meta:DBUser.Meta@10.10.10.10:5432/meta + - FERRETDB_LISTEN_ADDR=:27017 + ports: + - 27017:27017 # expose default mongodb port + #- 28088:8080 # expose metrics port +``` + + + +## MongoDB Cli + +**Use mongosh inside container** + +```bash +docker run --rm -it --network=ferretdb --entrypoint=mongosh mongo:5 mongodb://10.10.10.10:27017/ +``` + +**Or Install it on bare metal** + +```bash +cat > /etc/yum.repos.d/mongo.repo < /tmp/docker/gitea.tgz + +load: + cat /tmp/docker/gitea.tgz | gzip -d -c - | docker load + +.PHONY: default up run view log info stop clean pull rmi save load \ No newline at end of file diff --git a/app/gitea/README.md b/app/gitea/README.md new file mode 100644 index 0000000..01d98de --- /dev/null +++ b/app/gitea/README.md @@ -0,0 +1,78 @@ +# gitea + +Gitea - Git with a cup of tea : https://gitea.io/ + +A painless self-hosted Git service. + +Gitea is a community managed lightweight code hosting solution written in Go. It is published under the MIT license. + +Check public demo on pigsty: http://git.pigsty.cc, you can register a new user and explore it yourself. + +## Get Started + +Check [`.env`](.env) file for configurable environment variables: + +```bash +# server +GITEA_PORT=8889 +GITEA_SSH_PORT=222 +GITEA_DATA=/data/gitea +GITEA_DOMAIN=http://git.pigsty + +# database +GITEA_DB_TYPE=postgres +GITEA_DB_USER=dbuser_gitea +GITEA_DB_PASSWORD=DBUser.Gitea +GITEA_DB_HOST=10.10.10.10:5432 +GITEA_DB_NAME=gitea +GITEA_DB_SSLMODE=disable +``` + +Then launch gitea with `docker compose`: + +```bash +cd app/gitea; make up +``` + +Visit [http://git.pigsty](http://git.pigsty) or http://10.10.10.10:8889 and follow the installation wizard. + +## Makefile + +```bash +make up # pull up gitea with docker compose in minimal mode +make run # launch gitea with docker , local data dir and external PostgreSQL +make view # print gitea access point +make log # tail -f gitea logs +make info # introspect gitea with jq +make stop # stop gitea container +make clean # remove gitea container +make rmdata # remove gitea data: /data/gitea +make pull # pull latest gitea image +make rmi # remove gitea image +make save # save gitea image to /tmp/docker/gitea.tgz +make load # load gitea image from /tmp/docker/gitea.tgz +``` + +## Use External PostgreSQL + +gitea use its internal sqlite database by default, You can use external PostgreSQL for higher durability. + +```yaml +# postgres://dbuser_gitea:DBUser.gitea@10.10.10.10:5432/gitea +pg_users: [ { name: dbuser_gitea ,password: DBUser.Gitea ,pgbouncer: true ,roles: [ dbrole_admin ] ,comment: admin user for gitea service } ] +pg_databases: [ { name: gitea ,owner: dbuser_gitea ,revokeconn: true ,comment: gitea meta database } ] +``` + +```bash +bin/pgsql-user pg-meta dbuser_gitea +bin/pgsql-db pg-meta gitea +``` + +## Expose Service + +Add `gitea` entry to `infra_portal` to expose gitea service with nginx. + +```yaml +infra_portal: # domain names and upstream servers + gitea : { domain: git.pigsty ,endpoint: "127.0.0.1:8889" } +``` \ No newline at end of file diff --git a/app/gitea/docker-compose.yml b/app/gitea/docker-compose.yml new file mode 100644 index 0000000..b2e481e --- /dev/null +++ b/app/gitea/docker-compose.yml @@ -0,0 +1,32 @@ +# check .env files before running +services: + server: + image: gitea/gitea + container_name: gitea + restart: always + environment: # # check https://docs.gitea.io/en-us/config-cheat-sheet/ + + # basic + USER_UID: ${GITEA_UID} # use default uid = 1000 + USER_GID: ${GITEA_UID} # use same gid 1000 as uid + + # server + GITEA__server__ROOT_URL: ${GITEA_DOMAIN} + GITEA__server__HTTP_PORT: ${GITEA_PORT} + GITEA__server__SSH_PORT: ${GITEA_SSH_PORT} + + # database + GITEA__database__DB_TYPE: ${GITEA_DB_TYPE} + GITEA__database__USER: ${GITEA_DB_USER} + GITEA__database__PASSWD: ${GITEA_DB_PASSWORD} + GITEA__database__HOST: ${GITEA_DB_HOST} + GITEA__database__NAME: ${GITEA_DB_NAME} + GITEA__database__SSL_MODE: ${GITEA_DB_SSLMODE} + + volumes: + - ${GITEA_DATA}:/data # gitea main data dir: /data/gitea by default + - /etc/timezone:/etc/timezone:ro # read system timezone + - /etc/localtime:/etc/localtime:ro # read system localtime + ports: + - "${GITEA_PORT}:${GITEA_PORT}" # expose web service on 8889 + - "${GITEA_SSH_PORT}:2222" # expose ssh service on port 2222 diff --git a/app/jupyter/.env b/app/jupyter/.env new file mode 100644 index 0000000..d26781c --- /dev/null +++ b/app/jupyter/.env @@ -0,0 +1,21 @@ +# https://jupyter-docker-stacks.readthedocs.io/en/latest/ +JUPYTER_IMAGE=jupyter/minimal-notebook:latest +#JUPYTER_IMAGE=jupyter/scipy-notebook:latest + +# https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html +#JUPYTER_IMAGE=jupyter/r-notebook:latest +#JUPYTER_IMAGE=jupyter/julia-notebook:latest +#JUPYTER_IMAGE=jupyter/tensorflow-notebook:latest +#JUPYTER_IMAGE=jupyter/pytorch-notebook:latest +#JUPYTER_IMAGE=jupyter/datascience-notebook:latest +#JUPYTER_IMAGE=jupyter/all-spark-notebook:latest + +# DEFAULT USER NAME +#NB_USER=jovyan +#NB_UID=1000 + +# DEFAULT NOTEBOOK PORT +JUPYTER_PORT=8888 + +# IMPORTANT! change this default password in prod env +JUPYTER_TOKEN=pigsty diff --git a/app/jupyter/Makefile b/app/jupyter/Makefile new file mode 100644 index 0000000..659ec2c --- /dev/null +++ b/app/jupyter/Makefile @@ -0,0 +1,46 @@ +default: up view + +up: view + docker compose up -d + +# make a /data/jupyter dir and change owner to default +dir: + sudo mkdir -p /data/jupyter; sudo chown -R 1000:100 /data/jupyter + +run: + docker run --init --name jupyter --restart always --detach --publish 8886:8081 sosedoff/jupyter + +view: + @echo "visit http://cli.pigsty or http://10.10.10.10:8886" + @echo "example url: postgres://dbuser_meta:DBUser.Meta@10.10.10.10:5432/meta?sslmode=disable" + +log: + docker logs -f jupyter + +info: + docker inspect jupyter | jq + +stop: + docker stop jupyter; + +clean: + docker stop jupyter; docker rm -f jupyter + +# clean data +purge: + sudo rm -rf /data/jupyter + +pull: + docker compose pull + +rmi: + docker rmi jupyter/minimal-notebook + +save: + mkdir -p /tmp/docker + docker save sosedoff/jupyter | gzip -9 -c > /tmp/docker/jupyter.tgz + +load: + cat /tmp/docker/jupyter.tgz | gzip -d -c - | docker load + +.PHONY: default up run view log info stop clean pull rmi save load \ No newline at end of file diff --git a/app/jupyter/README.md b/app/jupyter/README.md new file mode 100644 index 0000000..1ff97fb --- /dev/null +++ b/app/jupyter/README.md @@ -0,0 +1,81 @@ +# jupyter notebook + +Run jupyter notebook with docker, you have to: + +- 1. change the default password in [`.env`](.env): `JUPYTER_TOKEN` +- 2. create data dir with proper permission: `make dir`, owned by `1000:100` +- 3. `make up` to pull up jupyter with docker compose + + +```bash +cd ~/pigsty/app/jupyter ; make dir up +``` + +Visit [http://lab.pigsty](http://lab.pigsty) or http://10.10.10.10:8888, the default password is `pigsty` + +- [`http://lab.pigsty?token=pigsty`](http://lab.pigsty?token=pigsty) + + + +## Prepare + +Create a data directory `/data/jupyter`, with the default uid & gid `1000:100`: + +```bash +make dir # mkdir -p /data/jupyter; chown -R 1000:100 /data/jupyter +``` + + +## Connect to Postgres + +Use the jupyter terminal to install `psycopg2-binary` & `psycopg2` package. + +```bash +pip install psycopg2-binary psycopg2 + +# install with a mirror +pip install -i https://pypi.tuna.tsinghua.edu.cn/simple psycopg2-binary psycopg2 + +pip install --upgrade pip +pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple +``` + +Or installation with `conda`: + +```bash +conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/ +conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/ +conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/ +``` + +then use the driver in your notebook + +```python +import psycopg2 + +conn = psycopg2.connect('postgres://dbuser_dba:DBUser.DBA@10.10.10.10:5432/meta') +cursor = conn.cursor() +cursor.execute('SELECT * FROM pg_stat_activity') +for i in cursor.fetchall(): + print(i) +``` + + + + +## Alias + +```bash +make up # pull up jupyter with docker compose +make dir # create required /data/jupyter and set owner +make run # launch jupyter with docker +make view # print jupyter access point +make log # tail -f jupyter logs +make info # introspect jupyter with jq +make stop # stop jupyter container +make clean # remove jupyter container +make pull # pull latest jupyter image +make rmi # remove jupyter image +make save # save jupyter image to /tmp/docker/jupyter.tgz +make load # load jupyter image from /tmp/docker/jupyter.tgz +``` diff --git a/app/jupyter/docker-compose.yml b/app/jupyter/docker-compose.yml new file mode 100644 index 0000000..aaa44de --- /dev/null +++ b/app/jupyter/docker-compose.yml @@ -0,0 +1,15 @@ +# change .env files before running! + +services: + jupyter: + image: ${JUPYTER_IMAGE} + container_name: jupyter + ports: + - "${JUPYTER_PORT}:${JUPYTER_PORT}" + volumes: + - /data/jupyter:/home/jovyan/work + environment: + - JUPYTER_TOKEN=${JUPYTER_TOKEN} + +# mkdir -p /data/jupyter; chown -R 1000:100 /data/jupyter +# docker run -it -p 8888:8888 -v /data/jupyter:/home/jovyan/work -e JUPYTER_TOKEN=pigsty -name jupyter jupyter/minimal-notebook \ No newline at end of file diff --git a/app/kong/.env b/app/kong/.env new file mode 100644 index 0000000..7391eee --- /dev/null +++ b/app/kong/.env @@ -0,0 +1,9 @@ +#https://hub.docker.com/_/kong +KONG_DATABASE=postgres +KONG_PG_HOST=10.10.10.10 +KONG_PG_USER=dbuser_kong +KONG_PG_DATABASE=kong +KONG_PG_PASSWORD=DBUser.Kong +KONG_PORT=8000 +KONG_PORT_SSL=8443 +KONG_ADMIN_PORT=8001 \ No newline at end of file diff --git a/app/kong/Makefile b/app/kong/Makefile new file mode 100644 index 0000000..3f02fbe --- /dev/null +++ b/app/kong/Makefile @@ -0,0 +1,36 @@ +default: up view + docker compose up -d + +view: + @echo "admin url: http://10.10.10.10:8001" + +ui: + docker run --init --name pgweb --name swagger -p 8882:8080 -e API_URL=http://10.10.10.10:8001 swaggerapi/swagger-ui + +log: + docker logs -f kong + +info: + docker inspect kong | jq + +stop: + docker stop kong; + +clean: + docker stop kong; docker rm -f kong + + +pull: + docker pull kong + +rmi: + docker rmi kong + +save: + mkdir -p /tmp/docker + docker save kong | gzip -9 -c > /tmp/docker/kong.tgz + +load: + cat /tmp/docker/kong.tgz | gzip -d -c - | docker load + +.PHONY: default up run view log info stop clean pull rmi save load \ No newline at end of file diff --git a/app/kong/README.md b/app/kong/README.md new file mode 100644 index 0000000..1ad82a8 --- /dev/null +++ b/app/kong/README.md @@ -0,0 +1,58 @@ +# Kong + +Kong the API Gateway: https://konghq.com/ + +Build delightful customer experiences and unleash developer productivity with Kong, the fastest cloud native API +platform. + +## TL;DR + +Check [`.env](.env) for postgres credentials and ports: + +```bash +#https://hub.docker.com/_/kong +KONG_DATABASE=postgres +KONG_PG_HOST=10.10.10.10 +KONG_PG_USER=dbuser_kong +KONG_PG_DATABASE=kong +KONG_PG_PASSWORD=DBUser.Kong +KONG_PORT=8000 +KONG_PORT_SSL=8443 +KONG_ADMIN_PORT=8001 +``` + +Then launch kong with: + +```bash +cd app/kong ; make up +``` + + +## Makefile + +```bash +make up # pull up kong with docker compose +make ui # run swagger ui container +make log # tail -f kong logs +make info # introspect kong with jq +make stop # stop kong container +make clean # remove kong container +make rmui # remove swagger ui container +make pull # pull latest kong image +make rmi # remove kong image +make save # save kong image to /tmp/docker/kong.tgz +make load # load kong image from /tmp/docker/kong.tgz +``` + +## Scripts + +* Default Port: 8000 +* Default SSL Port: 8443 +* Default Admin Port: 8001 +* Default Postgres Database: `postgres://dbuser_kong:DBUser.Kong@10.10.10.10:5432/kong` + +```yaml +# postgres://dbuser_kong:DBUser.Kong@10.10.10.10:5432/kong +- { name: kong, owner: dbuser_kong, revokeconn: true , comment: kong the api gateway database } +- { name: dbuser_kong, password: DBUser.Kong , pgbouncer: true , roles: [ dbrole_admin ] } +``` diff --git a/app/kong/docker-compose.yml b/app/kong/docker-compose.yml new file mode 100644 index 0000000..9a87a3c --- /dev/null +++ b/app/kong/docker-compose.yml @@ -0,0 +1,42 @@ +# check .env files before running +services: + + # Kong: The API Gateway + kong: + container_name: kong + image: kong:latest + restart: always + environment: + KONG_DATABASE: ${KONG_DATABASE} + KONG_PG_HOST: ${KONG_PG_HOST} + KONG_PG_USER: ${KONG_PG_USER} + KONG_PG_DATABASE: ${KONG_PG_DATABASE} + KONG_PG_PASSWORD: ${KONG_PG_PASSWORD} + + KONG_PROXY_LISTEN: 0.0.0.0:${KONG_PORT} + KONG_PROXY_LISTEN_SSL: 0.0.0.0:${KONG_PORT_SSL} + KONG_ADMIN_LISTEN: 0.0.0.0:${KONG_ADMIN_PORT} + depends_on: + - kong-migration + healthcheck: + test: [ "CMD", "curl", "-f", "http://kong:${KONG_ADMIN_PORT}" ] + interval: 5s + timeout: 2s + retries: 15 + ports: + - "${KONG_PORT}:${KONG_PORT}" + - "${KONG_PORT_SSL}:${KONG_PORT_SSL}" + - "${KONG_ADMIN_PORT}:${KONG_ADMIN_PORT}" + + # Kong database migration task + kong-migration: # one time job + container_name: kong-migration + image: kong:latest + command: "kong migrations bootstrap" + restart: on-failure + environment: + KONG_DATABASE: ${KONG_DATABASE} + KONG_PG_HOST: ${KONG_PG_HOST} + KONG_PG_USER: ${KONG_PG_USER} + KONG_PG_DATABASE: ${KONG_PG_DATABASE} + KONG_PG_PASSWORD: ${KONG_PG_PASSWORD} \ No newline at end of file diff --git a/app/mattermost/.env b/app/mattermost/.env new file mode 100644 index 0000000..fe7480a --- /dev/null +++ b/app/mattermost/.env @@ -0,0 +1,48 @@ +# Domain of service +DOMAIN=mm.pigsty +APP_PORT=8065 + +# Container settings +## Timezone inside the containers. The value needs to be in the form 'Europe/Berlin'. +## A list of these tz database names can be looked up at Wikipedia +## https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +TZ=UTC + +# Postgres +POSTGRES_URL="postgres://dbuser_mattermost:DBUser.Mattermost@10.10.10.10:5432/mattermost?sslmode=disable&connect_timeout=10" + +# Mattermost settings +## Inside the container the uid and gid is 2000. The folder owner can be set with + +# mkdir -p volumes/{config,data,logs,plugins,client/plugins,bleve-indexes} +# chown -R 2000:2000 volumes +MATTERMOST_CONFIG_PATH=./volumes/config +MATTERMOST_DATA_PATH=./volumes/data +MATTERMOST_LOGS_PATH=./volumes/logs +MATTERMOST_PLUGINS_PATH=./volumes/plugins +MATTERMOST_CLIENT_PLUGINS_PATH=./volumes/client/plugins +MATTERMOST_BLEVE_INDEXES_PATH=./volumes/bleve-indexes +MM_BLEVESETTINGS_INDEXDIR=./volumes/bleve-indexes + +## This will be 'mattermost-enterprise-edition' or 'mattermost-team-edition' based on the version of Mattermost you're installing. +MATTERMOST_IMAGE=mattermost-team-edition + +## Update the image tag if you want to upgrade your Mattermost version. You may also upgrade to the latest one. The example is based on the latest Mattermost ESR version. +MATTERMOST_IMAGE_TAG=latest + +## Make Mattermost container readonly. This interferes with the regeneration of root.html inside the container. Only use +## it if you know what you're doing. +## See https://github.com/mattermost/docker/issues/18 +MATTERMOST_CONTAINER_READONLY=false + +## Configuration settings for Mattermost. Documentation on the variables and the settings itself can be found at +## https://docs.mattermost.com/administration/config-settings.html +## Keep in mind that variables set here will take precedence over the same setting in config.json. This includes +## the system console as well and settings set with env variables will be greyed out. + +## Below one can find necessary settings to spin up the Mattermost container +MM_SQLSETTINGS_DRIVERNAME=postgres +MM_SQLSETTINGS_DATASOURCE=${POSTGRES_URL} + +## Example settings (any additional setting added here also needs to be introduced in the docker-compose.yml) +MM_SERVICESETTINGS_SITEURL=https://${DOMAIN} \ No newline at end of file diff --git a/app/mattermost/Makefile b/app/mattermost/Makefile new file mode 100644 index 0000000..3c40946 --- /dev/null +++ b/app/mattermost/Makefile @@ -0,0 +1,44 @@ +default: up + +up: mkdir + docker compose up -d + +mkdir: + mkdir -p volumes/{config,data,logs,plugins,client/plugins,bleve-indexes} + chown -R 2000:2000 volumes + +view: + @echo "visit http://mm.pigsty or http://10.10.10.10:8065" + +log: + docker compose logs -f + +info: + docker inspect mattermost | jq + +stop: + docker stop mattermost; + +clean: + docker stop mattermost; docker rm -f mattermost + +edit: + vi .env + +restart: + docker compose restart + +pull: + docker compose pull + +rmi: + docker rmi mattermost/mattermost-team-edition + +save: + mkdir -p /tmp/docker + docker save mattermost/mattermost-team-edition | gzip -9 -c > /tmp/docker/mattermost.tgz + +load: + cat /tmp/docker/mattermost.tgz | gzip -d -c - | docker load + +.PHONY: default up mkdir run view log info stop clean edit pull rmi save load \ No newline at end of file diff --git a/app/mattermost/docker-compose.yml b/app/mattermost/docker-compose.yml new file mode 100644 index 0000000..4da8363 --- /dev/null +++ b/app/mattermost/docker-compose.yml @@ -0,0 +1,42 @@ +services: + + mattermost: + container_name: mattermost + image: mattermost/mattermost-team-edition:latest + restart: unless-stopped + security_opt: + - no-new-privileges:true + pids_limit: 200 + read_only: ${MATTERMOST_CONTAINER_READONLY} + tmpfs: + - /tmp + + # mkdir -p volumes/{config,data,logs,plugins,client/plugins,bleve-indexes} + # chown -R 2000:2000 volumes + + volumes: + - ${MATTERMOST_CONFIG_PATH}:/mattermost/config:rw + - ${MATTERMOST_DATA_PATH}:/mattermost/data:rw + - ${MATTERMOST_LOGS_PATH}:/mattermost/logs:rw + - ${MATTERMOST_PLUGINS_PATH}:/mattermost/plugins:rw + - ${MATTERMOST_CLIENT_PLUGINS_PATH}:/mattermost/client/plugins:rw + - ${MATTERMOST_BLEVE_INDEXES_PATH}:/mattermost/bleve-indexes:rw + + ports: + - ${APP_PORT}:8065 + + environment: + # timezone inside container + - TZ + + # necessary Mattermost options/variables (see env.example) + - MM_SQLSETTINGS_DRIVERNAME + - MM_SQLSETTINGS_DATASOURCE + + # necessary for bleve + - MM_BLEVESETTINGS_INDEXDIR + + # additional settings + - MM_SERVICESETTINGS_SITEURL + + diff --git a/app/metabase/.env b/app/metabase/.env new file mode 100644 index 0000000..b62f794 --- /dev/null +++ b/app/metabase/.env @@ -0,0 +1,9 @@ +# https://www.metabase.com/docs/latest/installation-and-operation/running-metabase-on-docker + +MB_PORT=9004 +MB_DB_TYPE=postgres +MB_DB_DBNAME=meta +MB_DB_PORT=5432 +MB_DB_USER=dbuser_meta +MB_DB_PASS=DBUser.Meta +MB_DB_HOST=10.10.10.10 \ No newline at end of file diff --git a/app/metabase/Makefile b/app/metabase/Makefile new file mode 100644 index 0000000..d6546b0 --- /dev/null +++ b/app/metabase/Makefile @@ -0,0 +1,45 @@ +default: up view + +up: view + docker compose up -d + +run: + docker run -d --name metabase \ + -p 9004:3000 \ + -e "MB_DB_TYPE=postgres" \ + -e "MB_DB_DBNAME=meta" \ + -e "MB_DB_PORT=5432" \ + -e "MB_DB_USER=dbuser_meta" \ + -e "MB_DB_PASS=DBUser.Meta" \ + -e "MB_DB_HOST=10.10.10.10" \ + --name metabase metabase/metabase + +view: + @echo "visit http://mtbs.pigsty or http://10.10.10.10:9004" + +log: + docker logs -f metabase + +info: + docker inspect metabase | jq + +stop: + docker stop metabase; + +clean: + docker stop metabase; docker rm -f metabase + +pull: + docker pull metabase/metabase:latest + +rmi: + docker rmi metabase/metabase:latest + +save: + mkdir -p /tmp/docker + docker save metabase/metabase:latest | gzip -9 -c > /tmp/docker/metabase.tgz + +load: + cat /tmp/docker/metabase.tgz | gzip -d -c - | docker load + +.PHONY: default up run view log info stop clean conf dump pull rmi save load \ No newline at end of file diff --git a/app/metabase/README.md b/app/metabase/README.md new file mode 100644 index 0000000..1b5b1e6 --- /dev/null +++ b/app/metabase/README.md @@ -0,0 +1,62 @@ +# metabase + +Fast analytics with the friendly UX and integrated tooling to let your company explore data on their own: https://metabase.com/ + +```bash +# do not forget to check app/metabase/.env before make up +cd app/metabase; make up +``` + + +```bash +make up # pull up metabase with docker compose +make run # launch metabase with docker, local data dir and external PostgreSQL +make view # print metabase access point +make log # tail -f metabase logs +make info # introspect metabase with jq +make stop # stop metabase container +make clean # remove metabase container +make pull # pull latest metabase image +make rmi # remove metabase image +make save # save metabase image to /tmp/metabase.tgz +make load # load metabase image from /tmp +``` + + + +## Expose Service + +Expose metabase UI with pigsty's Nginx via `infra_portal`: + +```yaml +infra_portal: # infra services exposed via portal + home : { domain: i.pigsty } # default domain name + # ADD METABASE ENTRY HERE, then apply with (./infra.yml -t nginx) + metabase : { domain: mtbs.pigsty ,endpoint: "10.10.10.10:9004", websocket: true } +``` + +Visit [http://mtbs.pigsty](http://mtbs.pigsty) or http://10.10.10.10:9004 with: + + + + +## Docker Compose + +```yaml + +services: + metabase: + container_name: metabase + image: metabase/metabase:latest + environment: + MB_DB_TYPE: "${MB_DB_TYPE}" + MB_DB_DBNAME: "${MB_DB_DBNAME}" + MB_DB_PORT: "${MB_DB_PORT}" + MB_DB_USER: "${MB_DB_USER}" + MB_DB_PASS: "${MB_DB_PASS}" + MB_DB_HOST: "${MB_DB_HOST}" + restart: always + ports: + - ${MB_PORT}:8080 + +``` diff --git a/app/metabase/docker-compose.yml b/app/metabase/docker-compose.yml new file mode 100644 index 0000000..aa275ed --- /dev/null +++ b/app/metabase/docker-compose.yml @@ -0,0 +1,14 @@ +services: + metabase: + container_name: metabase + image: metabase/metabase:latest + environment: + MB_DB_TYPE: "${MB_DB_TYPE}" + MB_DB_DBNAME: "${MB_DB_DBNAME}" + MB_DB_PORT: "${MB_DB_PORT}" + MB_DB_USER: "${MB_DB_USER}" + MB_DB_PASS: "${MB_DB_PASS}" + MB_DB_HOST: "${MB_DB_HOST}" + restart: always + ports: + - ${MB_PORT}:8080 diff --git a/app/minio/Makefile b/app/minio/Makefile new file mode 100644 index 0000000..8708954 --- /dev/null +++ b/app/minio/Makefile @@ -0,0 +1,40 @@ +default: up view + +up: view + docker compose up -d + +data: + sudo mkdir /data/minio + +run: + docker run -p 9000:9000 -p 8890:8890 -e "MINIO_ROOT_USER=admin" -e "MINIO_ROOT_PASSWORD=pigsty.minio" minio/minio server /data --console-address ":8890" + +view: + @echo "visit http://10.10.10.10:9000 with admin : pigsty.minio" + +log: + docker logs -f minio + +info: + docker inspect minio | jq + +stop: + docker stop minio; + +clean: + docker stop minio; docker rm -f minio + +pull: + docker pull minio/minio + +rmi: + docker rmi minio/minio + +save: + mkdir -p /tmp/docker + docker save minio | gzip -9 -c > /tmp/docker/minio.tgz + +load: + cat /tmp/docker/minio.tgz | gzip -d -c - | docker load + +.PHONY: default up run view log info stop clean pull rmi save load \ No newline at end of file diff --git a/app/minio/README.md b/app/minio/README.md new file mode 100644 index 0000000..2bddfd8 --- /dev/null +++ b/app/minio/README.md @@ -0,0 +1,32 @@ +# minio + +Launch minio (s3) service on 9000 & 9001, you can also use the built-in minio support. + +```bash +cd ~/pigsty/app/minio ; make up +``` + +```bash +docker run -p 9000:9000 -p 9001:9001 \ + -e "MINIO_ROOT_USER=admin" \ + -e "MINIO_ROOT_PASSWORD=pigsty.minio" \ + minio/minio server /data --console-address ":9001" +``` + +visit http://10.10.10.10:9000 with user `admin` and password `pigsty.minio` + +## Makefile + +```bash +make up # pull up minio with docker compose +make run # launch minio with docker +make view # print minio access point +make log # tail -f minio logs +make info # introspect minio with jq +make stop # stop minio container +make clean # remove minio container +make pull # pull latest minio image +make rmi # remove minio image +make save # save minio image to /tmp/docker/minio.tgz +make load # load minio image from /tmp/docker/minio.tgz +``` diff --git a/app/minio/docker-compose.yml b/app/minio/docker-compose.yml new file mode 100644 index 0000000..00585b4 --- /dev/null +++ b/app/minio/docker-compose.yml @@ -0,0 +1,14 @@ +# check .env files before running +services: + minio: + container_name: minio + image: minio/minio + restart: always + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: S3User.MinIO + ports: + - "9000:9000" + - "9001:9001" + command: | + server /data --console-address ":9001" diff --git a/app/nocodb/.env b/app/nocodb/.env new file mode 100644 index 0000000..12cc89c --- /dev/null +++ b/app/nocodb/.env @@ -0,0 +1,5 @@ +# https://docs.nocodb.com/getting-started/environment-variables + +DATABASE_URL="postgres://dbuser_noco:DBUser.Noco@10.10.10.10:5432/noco" +NC_PORT=9003 +NC_AUTH_JWT_SECRET="some-secret-password" diff --git a/app/nocodb/Makefile b/app/nocodb/Makefile new file mode 100644 index 0000000..079dbdc --- /dev/null +++ b/app/nocodb/Makefile @@ -0,0 +1,42 @@ +default: up view + +up: view + docker compose up -d + +run: + docker run -d --name nocodb \ + -v "/data/nocodb:/usr/app/data/" \ + -p 9003:8080 \ + -e DATABASE_URL="postgres://dbuser_noco:DBUser.Noco@10.10.10.10:5432/noco" \ + -e NC_AUTH_JWT_SECRET="some-secret-password" \ + nocodb/nocodb:latest + +view: + @echo "visit http://noco.pigsty or http://10.10.10.10:9003" + +log: + docker logs -f nocodb + +info: + docker inspect nocodb | jq + +stop: + docker stop nocodb; + +clean: + docker stop nocodb; docker rm -f nocodb + +pull: + docker pull nocodb/nocodb:latest + +rmi: + docker rmi nocodb/nocodb:latest + +save: + mkdir -p /tmp/docker + docker save nocodb/nocodb:latest | gzip -9 -c > /tmp/docker/nocodb.tgz + +load: + cat /tmp/docker/nocodb.tgz | gzip -d -c - | docker load + +.PHONY: default up run view log info stop clean conf dump pull rmi save load \ No newline at end of file diff --git a/app/nocodb/README.md b/app/nocodb/README.md new file mode 100644 index 0000000..e9c146d --- /dev/null +++ b/app/nocodb/README.md @@ -0,0 +1,68 @@ +# NocoDB + +Open Source Airtable Alternative: https://nocodb.com/ + +```bash +# do not forget to check app/nocodb/.env before make up +cd app/nocodb; make up +``` + + +```bash +make up # pull up nocodb with docker compose +make run # launch nocodb with docker, local data dir and external PostgreSQL +make view # print nocodb access point +make log # tail -f nocodb logs +make info # introspect nocodb with jq +make stop # stop nocodb container +make clean # remove nocodb container +make pull # pull latest nocodb image +make rmi # remove nocodb image +make save # save nocodb image to /tmp/nocodb.tgz +make load # load nocodb image from /tmp +``` + + + +## Database + +Database & User definition + +```yaml +- {name: dbuser_noco ,password: DBUser.Noco ,pgbouncer: true ,roles: [dbrole_admin] ,comment: admin user for nocodb service } +- {name: noco ,owner: dbuser_noco ,revokeconn: true ,comment: nocodb database } +``` + +```bash +bin/pgsql-user pg-meta dbuser_noco +bin/pgsql-db pg-meta noco +# postgres://dbuser_noco:DBUser.Noco@10.10.10.10:5432/noco +``` + + +## Expose Service + + +Visit [http://noco.pigsty](http://noco.pigsty) or http://10.10.10.10:9003 with: + +Public demo: http://noco.pigsty.cc , username: `admin@pigsty.cc` and password: `pigsty.cc` + + + +## Docker Compose + +```yaml + +services: + nocodb: + container_name: nocodb + image: nocodb/nocodb:latest + environment: + DATABASE_URL: "${DATABASE_URL}" + NC_AUTH_JWT_SECRET: "${NC_AUTH_JWT_SECRET}" + restart: always + ports: + - ${NC_PORT}:8080 + volumes: + - /data/nocodb:/usr/app/data/ +``` diff --git a/app/nocodb/docker-compose.yml b/app/nocodb/docker-compose.yml new file mode 100644 index 0000000..ecd3a92 --- /dev/null +++ b/app/nocodb/docker-compose.yml @@ -0,0 +1,12 @@ +services: + nocodb: + container_name: nocodb + image: nocodb/nocodb:latest + environment: + DATABASE_URL: "${DATABASE_URL}" + NC_AUTH_JWT_SECRET: "${NC_AUTH_JWT_SECRET}" + restart: always + ports: + - ${NC_PORT}:8080 + volumes: + - /data/nocodb:/usr/app/data/ diff --git a/app/odoo/.env b/app/odoo/.env new file mode 100644 index 0000000..c2582c8 --- /dev/null +++ b/app/odoo/.env @@ -0,0 +1,11 @@ +# https://hub.docker.com/_/odoo# +PG_HOST=10.10.10.10 +PG_PORT=5432 +PG_USERNAME=odoo +PG_DATABASE=odoo +PG_PASSWORD=DBUser.Odoo + +ODOO_PORT=8069 +ODOO_DATA=/data/odoo/webdata +ODOO_ADDONS=/data/odoo/addons +ODOO_VERSION=19.0 \ No newline at end of file diff --git a/app/odoo/Makefile b/app/odoo/Makefile new file mode 100644 index 0000000..059e37a --- /dev/null +++ b/app/odoo/Makefile @@ -0,0 +1,47 @@ +default: init up view + +# init odoo dir & database schema +init: + docker compose --profile init up odoo-init +dir: + mkdir -p /data/odoo /data/odoo/webdata /data/odoo/addons + chown 100:101 /data/odoo /data/odoo/webdata /data/odoo/addons +view: + @echo "visit http://odoo.pigsty or http://10.10.10.10:8069" + +# administration +up: + docker compose up -d +down: + docker compose down +stop: + docker compose stop +start: + docker compose start +restart: + docker compose restart +pause: + docker compose pause +resume: + docker compose unpause +pull: + docker compose pull +rm: + docker compose rm +logs: + docker compose logs -f +port: + docker compose port +conf: + docker compose config +info: + docker inspect odoo | jq + + +save: + mkdir -p /tmp/docker + docker save odoo/odoo:19.0 | gzip -9 -c > /tmp/docker/odoo.tgz +load: + cat /tmp/docker/odoo.tgz | gzip -d -c - | docker load + +.PHONY: default init view up down stop start restart pause resume pull rm logs port conf info save load \ No newline at end of file diff --git a/app/odoo/README.md b/app/odoo/README.md new file mode 100644 index 0000000..8f06f7f --- /dev/null +++ b/app/odoo/README.md @@ -0,0 +1,111 @@ +# Odoo + +[Odoo](https://www.odoo.com/), Open Source ERP. + +> All your business on one platform, Simple, efficient, yet affordable + +Odoo pigsty installation tutorial: [Self-hosting Odoo](https://pigsty.io/docs/app/odoo) + +Check public demo: http://odoo.pigsty.cc, username: `test@pigsty.cc`, password: `pigsty` + +> The default credentials are `admin` : `admin` + + +-------- + +## Get Started + +First, follow the standard pigsty single (or multiple if you want) node installation, using the [`conf/app/odoo`](https://github.com/pgsty/pigsty/blob/main/conf/app/odoo.yml) config template. + +```bash +curl -fsSL https://repo.pigsty.io/pig | bash +pig sty init # init pigsty directory +pig sty boot # prepare local repo & ansible +pig sty conf -c app/odoo # use the odoo 1-node config template +pig sty install # begin installation +``` + +Then install docker and launch odoo app: + +``` +./docker.yml # install & configure docker & docker-compose +./app.yml # install and launch odoo with docker-compose +``` + +That's it, YOU ARE ALL SET! Odoo is serving on port 8069 by default, you can access it via `http://:8069`. + +You can add a static entry to your `/etc/hosts` file to access odoo via `http://odoo.pigsty` through the nginx 80/443 portal + +> To create another odoo database, you'll have to alter the `odoo` db user to superuser to do so. + + + +-------- + +## Configuration + +There's a config template [`conf/app/odoo`](https://github.com/pgsty/pigsty/blob/main/conf/app/odoo.yml), you'd better change some credentials and make your modifications there: + +```yaml + +all: + children: + + # the odoo application (default username & password: admin/admin) + odoo: + hosts: { 10.10.10.10: {} } + vars: + app: odoo # specify app name to be installed (in the apps) + apps: # define all applications + odoo: # app name, should have corresponding ~/app/odoo folder + file: # optional directory to be created + - { path: /data/odoo ,state: directory, owner: 100, group: 101 } + - { path: /data/odoo/webdata ,state: directory, owner: 100, group: 101 } + - { path: /data/odoo/addons ,state: directory, owner: 100, group: 101 } + conf: # override /opt//.env config file + PG_HOST: 10.10.10.10 # postgres host + PG_PORT: 5432 # postgres port + PG_USERNAME: odoo # postgres user + PG_PASSWORD: DBUser.Odoo # postgres password + ODOO_PORT: 8069 # odoo app port + ODOO_DATA: /data/odoo/webdata # odoo webdata + ODOO_ADDONS: /data/odoo/addons # odoo plugins + ODOO_DBNAME: odoo # odoo database name + ODOO_VERSION: 19.0 # odoo image version + + # the odoo database + pg-odoo: + hosts: { 10.10.10.10: { pg_seq: 1, pg_role: primary } } + vars: + pg_cluster: pg-odoo + pg_users: + - { name: odoo ,password: DBUser.Odoo ,pgbouncer: true ,roles: [ dbrole_admin ] ,createdb: true ,comment: admin user for odoo service } + - { name: odoo_ro ,password: DBUser.Odoo ,pgbouncer: true ,roles: [ dbrole_readonly ] ,comment: read only user for odoo service } + - { name: odoo_rw ,password: DBUser.Odoo ,pgbouncer: true ,roles: [ dbrole_readwrite ] ,comment: read write user for odoo service } + pg_databases: + - { name: odoo ,owner: odoo ,revokeconn: true ,comment: odoo main database } + pg_hba_rules: + - { user: all ,db: all ,addr: 172.17.0.0/16 ,auth: pwd ,title: 'allow access from local docker network' } + - { user: dbuser_view , db: all ,addr: infra ,auth: pwd ,title: 'allow grafana dashboard access cmdb from infra nodes' } + +``` + + + +------- + +## Odoo Addons + +There are lots of Odoo modules available in the community, you can install them by downloading and placing them in the `addons` folder (`/data/odoo/addons` by default). + +```bash +mkdir -p /data/odoo/addons; chown 100:101 /data/odoo/addons +``` + +To enable addon module, first entering the [Developer mode](https://www.odoo.com/documentation/19.0/applications/general/developer_mode.html) + +> Settings -> Generic Settings -> Developer Tools -> Activate the developer Mode + +Then goes to the > Apps -> Update Apps List, then you can find the extra addons and install from the panel. + +Frequently used [free](https://apps.odoo.com/apps/modules/browse?order=Downloads) addons: [Accounting Kit](https://apps.odoo.com/apps/modules/19.0/base_accounting_kit/) diff --git a/app/odoo/docker-compose.yml b/app/odoo/docker-compose.yml new file mode 100644 index 0000000..7928fdb --- /dev/null +++ b/app/odoo/docker-compose.yml @@ -0,0 +1,47 @@ +# please change the credentials in .env file! +#!/usr/bin/env docker compose +--- +#==============================================================# +# File : docker-compose.yml +# Desc : odoo docker compose template +# Ctime : 2024-09-19 +# Mtime : 2025-01-12 +# Path : app/odoo/docker-compose.yml +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +#==============================================================# +# Image: https://hub.docker.com/_/odoo +# Tutorial: https://pigsty.io/docs/app/odoo + + +name: odoo +services: + + # the main odoo container + odoo: + container_name: odoo + image: odoo:${ODOO_VERSION} + ports: + - ${ODOO_PORT}:8069 + environment: + - HOST=${PG_HOST} + - PORT=${PG_PORT} + - USER=${PG_USERNAME} + - PASSWORD=${PG_PASSWORD} + volumes: + - ${ODOO_DATA}:/var/lib/odoo + - ${ODOO_ADDONS}:/mnt/extra-addons + #- ./config:/etc/odoo + + # run this init schema migration with: docker compose --profile init up odoo-init + odoo-init: + image: odoo:${ODOO_VERSION} + profiles: ["init"] + command: [ odoo, --stop-after-init , --init=base , -d, "${ODOO_DBNAME}" ] #--without-demo + environment: + - HOST=${PG_HOST} + - PORT=${PG_PORT} + - USER=${PG_USERNAME} + - PASSWORD=${PG_PASSWORD} + restart: "no" +... \ No newline at end of file diff --git a/app/pg_exporter/.env b/app/pg_exporter/.env new file mode 100644 index 0000000..4aa4daa --- /dev/null +++ b/app/pg_exporter/.env @@ -0,0 +1,16 @@ +# https://openpolardb.com/document?type=PolarDB-PG +PG_EXPORTER_PORT: "9632" +PG_EXPORTER_URL: 'postgres://dbuser_dba:DBUser.DBA@10.10.10.10:5432/postgres?sslmode=disable' +PG_EXPORTER_CONFIG: '/etc/pg_exporter.yml' +PG_EXPORTER_LABEL: "" +PG_EXPORTER_TAG: "" +PG_EXPORTER_DISABLE_CACHE: 'false' +PG_EXPORTER_DISABLE_DEFAULT_METRICS: 'false' +PG_EXPORTER_AUTO_DISCOVERY: 'true' +PG_EXPORTER_EXCLUDE_DATABASE: "template0,template1,postgres" +PG_EXPORTER_INCLUDE_DATABASE: "" +PG_EXPORTER_NAMESPACE: "pg" +PG_EXPORTER_FAIL_FAST: 'false' +PG_EXPORTER_CONNECT_TIMEOUT: '100' +PG_EXPORTER_LISTEN_ADDRESS: ":9630" +PG_EXPORTER_TELEMETRY_PATH: "/metrics" diff --git a/app/pg_exporter/Makefile b/app/pg_exporter/Makefile new file mode 100644 index 0000000..d862876 --- /dev/null +++ b/app/pg_exporter/Makefile @@ -0,0 +1,37 @@ +default: up + +up: + docker compose up -d + +run: + docker run --init --name pg_exporter -p 9632:9630 -e PG_EXPORTER_URL='postgres://dbuser_dba:DBUser.DBA@10.10.10.10:5432/postgres?sslmode=disable' vonng/pg_exporter + +view: + curl http://localhost:9632/metrics + +log: + docker logs -f pg_exporter + +info: + docker inspect pg_exporter | jq + +stop: + docker stop pg_exporter; + +clean: + docker stop pg_exporter; docker rm -f pg_exporter + +pull: + docker pull vonng/pg_exporter + +rmi: + docker rmi vonng/pg_exporter + +save: + mkdir -p /tmp/docker + docker save Vonng/pg_exporter | gzip -9 -c > /tmp/docker/pg_exporter.tgz + +load: + cat /tmp/docker/pg_exporter.tgz | gzip -d -c - | docker load + +.PHONY: default up run view log info stop clean pull rmi save load \ No newline at end of file diff --git a/app/pg_exporter/README.md b/app/pg_exporter/README.md new file mode 100644 index 0000000..a371549 --- /dev/null +++ b/app/pg_exporter/README.md @@ -0,0 +1,43 @@ +# PG Exporter + +This is an example of running pg_exporter inside docker + +Which expose meta sample database on (10.10.10.10) on local port 9632. + +Change `.env` file to adapt to your own postgres targets + +```bash +cd ~/pigsty/app/pg_exporter ; docker compose up -d +``` + +```bash +make up # pull up pg_exporter with docker compose +make run # launch pg_exporter with docker +make view # curl pg_exporter metrics +make log # tail -f pg_exporter logs +make info # introspect pg_exporter with jq +make stop # stop pg_exporter container +make clean # remove pg_exporter container +make pull # pull latest pg_exporter image +make rmi # remove pg_exporter image +make save # save pg_exporter image to /tmp/pg_exporter.tgz +make load # load pg_exporter image from /tmp +``` + + +## Usage + +Replace bare metal pg_exporter with docker: + +```bash +systemctl stop pg_exporter # stop existing pg_exporter on bare metal +docker run --init --name pg_exporter -p 9632:9630 \ + -e PG_EXPORTER_URL='postgres://dbuser_dba:DBUser.DBA@10.10.10.10:5432/postgres?sslmode=disable' \ + -e PG_EXPORTER_AUTO_DISCOVERY='true' \ + vonng/pg_exporter +``` + +```bash +curl http://127.0.0.1:9630/metrics # get metrics from pg_exporter inside docker +make view # or use the shortcut +``` \ No newline at end of file diff --git a/app/pg_exporter/docker-compose.yml b/app/pg_exporter/docker-compose.yml new file mode 100644 index 0000000..d52964e --- /dev/null +++ b/app/pg_exporter/docker-compose.yml @@ -0,0 +1,37 @@ +services: + pg_exporter: + container_name: pg_exporter + image: vonng/pg_exporter + restart: always + environment: + PG_EXPORTER_URL: ${PG_EXPORTER_URL} + PG_EXPORTER_CONFIG: ${PG_EXPORTER_CONFIG} + PG_EXPORTER_LABEL: ${PG_EXPORTER_LABEL} + PG_EXPORTER_TAG: ${PG_EXPORTER_TAG} + PG_EXPORTER_DISABLE_CACHE: ${PG_EXPORTER_DISABLE_CACHE} + PG_EXPORTER_DISABLE_DEFAULT_METRICS: ${PG_EXPORTER_DISABLE_DEFAULT_METRICS} + PG_EXPORTER_AUTO_DISCOVERY: ${PG_EXPORTER_AUTO_DISCOVERY} + PG_EXPORTER_EXCLUDE_DATABASE: ${PG_EXPORTER_EXCLUDE_DATABASE} + PG_EXPORTER_INCLUDE_DATABASE: ${PG_EXPORTER_INCLUDE_DATABASE} + PG_EXPORTER_NAMESPACE: ${PG_EXPORTER_NAMESPACE} + PG_EXPORTER_FAIL_FAST: ${PG_EXPORTER_FAIL_FAST} + PG_EXPORTER_CONNECT_TIMEOUT: ${PG_EXPORTER_CONNECT_TIMEOUT} + PG_EXPORTER_LISTEN_ADDRESS: ${PG_EXPORTER_LISTEN_ADDRESS} + PG_EXPORTER_TELEMETRY_PATH: ${PG_EXPORTER_TELEMETRY_PATH} + + #PG_EXPORTER_URL: 'postgres://dbuser_monitor:DBUser.Monitor@10.10.10.10:5432/postgres?sslmode=disable' + #PG_EXPORTER_CONFIG: '/etc/pg_exporter.yml' + #PG_EXPORTER_LABEL: "" + #PG_EXPORTER_TAG: "" + #PG_EXPORTER_DISABLE_CACHE: 'false' + #PG_EXPORTER_DISABLE_DEFAULT_METRICS: 'false' + #PG_EXPORTER_AUTO_DISCOVERY: 'true' + #PG_EXPORTER_EXCLUDE_DATABASE: "template0,template1,postgres" + #PG_EXPORTER_INCLUDE_DATABASE: "" + #PG_EXPORTER_NAMESPACE: "pg" + #PG_EXPORTER_FAIL_FAST: 'false' + #PG_EXPORTER_CONNECT_TIMEOUT: '100' + #PG_EXPORTER_LISTEN_ADDRESS: ":9630" + #PG_EXPORTER_TELEMETRY_PATH: "/metrics" + ports: + - "${PG_EXPORTER_PORT}:9630" diff --git a/app/pgadmin/.env b/app/pgadmin/.env new file mode 100644 index 0000000..a00bc47 --- /dev/null +++ b/app/pgadmin/.env @@ -0,0 +1,9 @@ +# https://www.pgadmin.org/ +# https://www.pgadmin.org/docs/pgadmin4/latest/container_deployment.html + +PGADMIN_DEFAULT_EMAIL=admin@pigsty.cc +PGADMIN_DEFAULT_PASSWORD=pigsty +PGADMIN_LISTEN_ADDRESS=0.0.0.0 +PGADMIN_PORT=8885 +PGADMIN_SERVER_JSON_FILE=/pgadmin4/servers.json +PGADMIN_REPLACE_SERVERS_ON_STARTUP=true \ No newline at end of file diff --git a/app/pgadmin/Makefile b/app/pgadmin/Makefile new file mode 100644 index 0000000..9cb978c --- /dev/null +++ b/app/pgadmin/Makefile @@ -0,0 +1,60 @@ +default: up view + +up: + docker compose up -d +down: + docker compose down +run: + docker run --init --name pgadmin --restart always --detach --publish 8885:80 \ + -e PGADMIN_DEFAULT_EMAIL=admin@pigsty.cc -e PGADMIN_DEFAULT_PASSWORD=pigsty \ + dpage/pgadmin4 +view: + @echo "visit http://adm.pigsty or http://10.10.10.10:8885" + @echo "cred: admin@pigsty.cc / pigsty" +log: + docker logs -f pgadmin +info: + docker inspect pgadmin | jq +clean: + docker stop pgadmin; docker rm -f pgadmin +edit: + vi .env +pull: + docker compose pull +rmi: + docker rmi dpage/pgadmin4 +start: + docker compose start +stop: + docker compose stop +restart: + docker compose restart +exec: + docker exec -u 0 -it pgadmin /bin/bash + +reload: list pass +conf: list pass +pass: + docker exec pgadmin cp /pgadmin4/pgpass /var/lib/pgadmin/storage/admin_pigsty.cc/pgpass; +list: + docker exec pgadmin /venv/bin/python3 /pgadmin4/setup.py load-servers --replace --user admin@pigsty.cc /pgadmin4/servers.json + +# load pigsty server list manually +servers: + docker cp /infra/pgadmin/servers.json pgadmin:/tmp/servers.json; + docker exec -u 0 -it pgadmin chown pgadmin /tmp/servers.json; + docker exec -it pgadmin /venv/bin/python3 /pgadmin4/setup.py load-servers --replace --user admin@pigsty.cc /tmp/servers.json +# load pgpass manually +pgpass: + docker cp /infra/pgadmin/pgpass pgadmin:/var/lib/pgadmin/storage/admin_pigsty.cc/pgpass; + docker exec -u 0 -it pgadmin chown pgadmin /var/lib/pgadmin/storage/admin_pigsty.cc/pgpass; + + +dump: save +save: + mkdir -p /tmp/docker + docker save dpage/pgadmin4 | gzip -9 -c > /tmp/docker/pgadmin4.tgz +load: + cat /tmp/docker/pgadmin4.tgz | gzip -d -c - | docker load + +.PHONY: default up down run view log info clean edit pull start stop restart exec reload conf list pass servers pgpass rmi dump save load \ No newline at end of file diff --git a/app/pgadmin/README.md b/app/pgadmin/README.md new file mode 100644 index 0000000..b350c8f --- /dev/null +++ b/app/pgadmin/README.md @@ -0,0 +1,86 @@ +# PgAdmin4 + +PGADMIN4: https://www.pgadmin.org/ + +Tutorial: https://pigsty.io/docs/app/pgadmin + +pgAdmin is the most popular and feature rich Open Source administration and development platform for PostgreSQL. + +-------- + +## TL;DR + +launch pgadmin4 with: + +```bash +cd ~/pigsty; ./app.yml -e app=pgadmin +``` + +Then visit [http://adm.pigsty](http://adm.pigsty) or http://10.10.10.10:8885 with: + +username: `admin@pigsty.cc` and password: `pigsty` + + + +--------- + +## Reload + +To reload server list, just re-generate `servers.json` file and restart pgadmin + +```bash +./infra.yml -t env_pgadmin # regenerate pgadmin server list +./app.yml -e app=pgadmin -t app_launch -e app_args=reload # reload pgadmin server list +``` + +> PgAdmin launch is slow, may take 30+ seconds to start, please reload after server started. + + + +-------- + +## Configure + +If you want to change configuration, you can edit [`~/pigsty/app/pgadmin/.env`](.env) + +```bash +PGADMIN_DEFAULT_EMAIL=admin@pigsty.cc +PGADMIN_DEFAULT_PASSWORD=pigsty # IMPORTANT!!! CHANGE PASSWORD +PGADMIN_LISTEN_ADDRESS=0.0.0.0 +PGADMIN_PORT=8885 +PGADMIN_SERVER_JSON_FILE=/pgadmin4/servers.json +PGPASS_FILE=/pgadmin4/pgpass +PGADMIN_REPLACE_SERVERS_ON_STARTUP=true +``` + +Or configure them in the `pigsty.yml` config file + +```yaml + +all: + children: + pgadmin: + hosts: { 10.10.10.10: {} } + vars: + app: pgadmin + apps: + pgadmin: + conf: + PGADMIN_DEFAULT_EMAIL: admin@pigsty.cc # CHANGE USERNAME + PGADMIN_DEFAULT_PASSWORD: pigsty # CHANGE PASSWORD (IMPORTANT!) + PGADMIN_PORT: 8885 + + vars: + infra_portal: + pgadmin: + domain: adm.pigsty # CHANGE DOMAIN + endpoint: "10.10.10.10:8885" # CHANGE ENDPOINT + certbot: adm.pigsty # if using real domain & https +``` + +Then run `app.yml` and `infra.yml` playbook to take effect: + +```bash +./app.yml # install pgadmin according to the config +./infra.yml -t nginx # re-configure nginx to proxy pgadmin server +``` diff --git a/app/pgadmin/docker-compose.yml b/app/pgadmin/docker-compose.yml new file mode 100644 index 0000000..42e8afc --- /dev/null +++ b/app/pgadmin/docker-compose.yml @@ -0,0 +1,13 @@ +# check .env files before running +services: + pgadmin: + container_name: pgadmin + image: dpage/pgadmin4 + restart: always + env_file: + - .env + volumes: + - /infra/pgadmin/servers.json:/pgadmin4/servers.json + - /infra/pgadmin/pgpass:/pgadmin4/pgpass:ro + ports: + - "${PGADMIN_PORT}:80" diff --git a/app/pgweb/Makefile b/app/pgweb/Makefile new file mode 100644 index 0000000..493e567 --- /dev/null +++ b/app/pgweb/Makefile @@ -0,0 +1,38 @@ +default: up view + +up: view + docker compose up -d + +run: + docker run --init --name pgweb --restart always --detach --publish 8886:8081 sosedoff/pgweb + +view: + @echo "visit http://cli.pigsty or http://10.10.10.10:8886" + @echo "example url: postgres://dbuser_meta:DBUser.Meta@10.10.10.10:5432/meta?sslmode=disable" + +log: + docker logs -f pgweb + +info: + docker inspect pgweb | jq + +stop: + docker stop pgweb; + +clean: + docker stop pgweb; docker rm -f pgweb + +pull: + docker pull sosedoff/pgweb + +rmi: + docker rmi sosedoff/pgweb + +save: + mkdir -p /tmp/docker + docker save sosedoff/pgweb | gzip -9 -c > /tmp/docker/pgweb.tgz + +load: + cat /tmp/docker/pgweb.tgz | gzip -d -c - | docker load + +.PHONY: default up run view log info stop clean pull rmi save load \ No newline at end of file diff --git a/app/pgweb/README.md b/app/pgweb/README.md new file mode 100644 index 0000000..5d82a4f --- /dev/null +++ b/app/pgweb/README.md @@ -0,0 +1,33 @@ +# PGWEB + +PGWEB: https://github.com/sosedoff/pgweb + +Simple web-based and cross platform PostgreSQL database explorer. + + +```bash +cd ~/pigsty/app/pgweb ; make up +``` + +Visit [http://cli.pigsty](http://cli.pigsty) or http://10.10.10.10:8886 + +Try connecting with example URLs: + +```bash +postgres://dbuser_meta:DBUser.Meta@10.10.10.10:5432/meta?sslmode=disable +postgres://test:test@10.10.10.11:5432/test?sslmode=disable +``` + +```bash +make up # pull up pgweb with docker compose +make run # launch pgweb with docker +make view # print pgweb access point +make log # tail -f pgweb logs +make info # introspect pgweb with jq +make stop # stop pgweb container +make clean # remove pgweb container +make pull # pull latest pgweb image +make rmi # remove pgweb image +make save # save pgweb image to /tmp/docker/pgweb.tgz +make load # load pgweb image from /tmp/docker/pgweb.tgz +``` diff --git a/app/pgweb/docker-compose.yml b/app/pgweb/docker-compose.yml new file mode 100644 index 0000000..b6f7f24 --- /dev/null +++ b/app/pgweb/docker-compose.yml @@ -0,0 +1,8 @@ +services: + pgweb: + container_name: pgweb + image: sosedoff/pgweb + restart: always + ports: [ "8886:8081" ] + +# docker run --init --name pgweb --restart always --detach --publish 8886:8081 sosedoff/pgweb \ No newline at end of file diff --git a/app/postgrest/.env b/app/postgrest/.env new file mode 100644 index 0000000..3401ab7 --- /dev/null +++ b/app/postgrest/.env @@ -0,0 +1,6 @@ +# https://postgrest.org/en/stable/install.html#docker +POSTGREST_SERVER_PORT=8884 +POSTGREST_DB_URI=postgres://dbuser_dba:DBUser.DBA@10.10.10.10:5432/meta +POSTGREST_DB_SCHEMA=pigsty +POSTGREST_DB_ANON_ROLE=dbuser_dba +POSTGREST_JWT_SECRET=some-random-secret \ No newline at end of file diff --git a/app/postgrest/Makefile b/app/postgrest/Makefile new file mode 100644 index 0000000..2012a6f --- /dev/null +++ b/app/postgrest/Makefile @@ -0,0 +1,43 @@ +default: up view + +up: view + docker compose up -d + +run: + docker run --init --name postgrest --restart always --detach --publish 8884:8081 postgrest/postgrest + +ui: + docker run --init --name pgweb --name swagger -p 8882:8080 -e API_URL=http://10.10.10.10:8884 swaggerapi/swagger-ui + +view: + @echo "visit http://10.10.10.10:8884 for API Spec" + +log: + docker logs -f postgrest + +info: + docker inspect postgrest | jq + +stop: + docker stop postgrest; + +clean: + docker stop postgrest; docker rm -f postgrest + +rmui: + docker stop swagger; docker rm swagger + +pull: + docker pull postgrest/postgrest + +rmi: + docker rmi postgrest/postgrest + +save: + mkdir -p /tmp/docker + docker save postgrest/postgrest | gzip -9 -c > /tmp/docker/postgrest.tgz + +load: + cat /tmp/docker/postgrest.tgz | gzip -d -c - | docker load + +.PHONY: default up run ui rmui view log info stop clean pull rmi save load \ No newline at end of file diff --git a/app/postgrest/README.md b/app/postgrest/README.md new file mode 100644 index 0000000..0fb7f91 --- /dev/null +++ b/app/postgrest/README.md @@ -0,0 +1,46 @@ +# PostgREST + +PostgREST: https://postgrest.org/en/stable/index.html + +Serve a RESTful API from any Postgres database + +This is an example of creating pigsty cmdb API with PostgREST + +```bash +cd ~/pigsty/app/postgrest; make up +``` + +http://10.10.10.10:8884 is the default endpoint for PostgREST + +Public API: http://api.pigsty.cc + + +## Makefile + +```bash +make up # pull up postgrest with docker compose +make run # launch postgrest with docker +make ui # run swagger ui container +make view # print postgrest access point +make log # tail -f postgrest logs +make info # introspect postgrest with jq +make stop # stop postgrest container +make clean # remove postgrest container +make rmui # remove swagger ui container +make pull # pull latest postgrest image +make rmi # remove postgrest image +make save # save postgrest image to /tmp/docker/postgrest.tgz +make load # load postgrest image from /tmp/docker/postgrest.tgz +``` + + +## Swagger UI + +Launch a swagger OpenAPI UI and visualize PostgREST API on 8883 with: + +```bash +docker run --init --name postgrest --name swagger -p 8883:8080 -e API_URL=http://10.10.10.10:8884 swaggerapi/swagger-ui +# docker run -d -e API_URL=http://10.10.10.10:8884 -p 8883:8080 swaggerapi/swagger-editor # swagger editor +``` + +Check [http://10.10.10.10:8883/](http://10.10.10.10:8883/) diff --git a/app/postgrest/docker-compose.yml b/app/postgrest/docker-compose.yml new file mode 100644 index 0000000..4db27ba --- /dev/null +++ b/app/postgrest/docker-compose.yml @@ -0,0 +1,15 @@ +# check .env files before running + +services: + postgrest: + container_name: postgrest + image: postgrest/postgrest + restart: always + environment: + PGRST_DB_URI : ${POSTGREST_DB_URI} + PGRST_DB_SCHEMA : ${POSTGREST_DB_SCHEMA} + PGRST_DB_ANON_ROLE : ${POSTGREST_DB_ANON_ROLE} + PGRST_SERVER_PORT : ${POSTGREST_SERVER_PORT} + PGRST_JWT_SECRET : ${POSTGREST_JWT_SECRET} + ports: + - "${POSTGREST_SERVER_PORT}:${POSTGREST_SERVER_PORT}" \ No newline at end of file diff --git a/app/registry/Makefile b/app/registry/Makefile new file mode 100644 index 0000000..3745ec1 --- /dev/null +++ b/app/registry/Makefile @@ -0,0 +1,91 @@ +#!/usr/bin/make + +# Registry app management commands +APP = registry +APP_DIR = /opt/$(APP) +DATA_DIR = /data/$(APP) + +#==============================================================# +# Docker Registry Mirror Management +#==============================================================# + +default: up + +# create required directories +dir: + mkdir -p $(DATA_DIR) + + +# launch the registry service +up: dir conf + cd $(APP_DIR) && docker compose up -d + +# stop the registry service +down: + cd $(APP_DIR) && docker compose down + +# restart the registry service +restart: down up + +# remove the registry service and data +clean: down + cd $(APP_DIR) && docker compose down -v + rm -rf $(DATA_DIR)/* + +# show registry service status +status: + cd $(APP_DIR) && docker compose ps + +# show registry service logs +log: + cd $(APP_DIR) && docker compose logs -f + +# pull the registry image +pull: + cd $(APP_DIR) && docker compose pull + +# test registry health +health: + @echo "Testing registry health..." + @curl -f http://localhost:5000/v2/ && echo "Registry is healthy" || echo "Registry is not responding" + +# show registry catalog +catalog: + @echo "Registry catalog:" + @curl -s http://localhost:5000/v2/_catalog | jq '.' 2>/dev/null || curl -s http://localhost:5000/v2/_catalog + +# backup registry data +backup: + @echo "Backing up registry data..." + tar -czf $(DATA_DIR)_backup_$(shell date +%Y%m%d_%H%M%S).tar.gz -C $(DATA_DIR) . + +# show disk usage +du: + @echo "Registry data disk usage:" + @du -sh $(DATA_DIR) 2>/dev/null || echo "Data directory not found" + +# clean up unused data +prune: + cd $(APP_DIR) && docker system prune -f + cd $(APP_DIR) && docker volume prune -f + +#==============================================================# +# Helper commands +#==============================================================# + +# display help information +help: + @echo "Available commands:" + @echo " up - Start registry service" + @echo " down - Stop registry service" + @echo " restart - Restart registry service" + @echo " status - Show service status" + @echo " log - Show service logs" + @echo " clean - Remove service and data" + @echo " health - Test registry health" + @echo " catalog - Show registry catalog" + @echo " backup - Backup registry data" + @echo " du - Show disk usage" + @echo " prune - Clean up unused data" + +.PHONY: default dir conf up down restart clean status log pull health catalog backup du prune help \ No newline at end of file diff --git a/app/registry/README.md b/app/registry/README.md new file mode 100644 index 0000000..4a1d006 --- /dev/null +++ b/app/registry/README.md @@ -0,0 +1,254 @@ +# Docker Registry Mirror + +This is a Docker Registry mirror service for caching Docker images, particularly useful for users in China or regions with slow Docker Hub access. + +## Features + +- **Registry Mirror**: Cache Docker images from Docker Hub and other registries +- **Web UI**: Optional web interface for managing cached images +- **High Performance**: Local caching reduces pull times significantly +- **Storage Management**: Configurable storage cleanup and management +- **Health Monitoring**: Built-in health check endpoints + +## Quick Start + +### Prerequisites + +- Docker and Docker Compose installed +- Sufficient disk space for image caching (recommend 100GB+) + +### Deployment + +1. **Create data directory:** + ```bash + sudo mkdir -p /data/registry + sudo chown -R $(whoami):$(whoami) /data/registry + ``` + +2. **Start the service:** + ```bash + make up + ``` + +3. **Check service status:** + ```bash + make status + ``` + +4. **Test the registry:** + ```bash + # Test health + make health + + # Configure Docker daemon to use the mirror + # Add to /etc/docker/daemon.json: + { + "registry-mirrors": ["http://your-server-ip:5000"] + } + + # Restart Docker + sudo systemctl restart docker + ``` + +### Usage Examples + +#### Configure Docker to use the mirror + +1. **Edit Docker daemon config:** + ```bash + sudo nano /etc/docker/daemon.json + ``` + +2. **Add registry mirror:** + ```json + { + "registry-mirrors": ["http://your-domain.com:5000"], + "insecure-registries": ["your-domain.com:5000"] + } + ``` + +3. **Restart Docker:** + ```bash + sudo systemctl restart docker + ``` + +#### Pull images through the mirror + +Once configured, all `docker pull` commands will automatically use the mirror: + +```bash +# These commands will now use your mirror +docker pull nginx:latest +docker pull redis:alpine +docker pull postgres:15 +``` + +#### Direct registry access + +You can also access the registry directly: + +```bash +# Check registry health +curl http://your-domain.com:5000/v2/ + +# List cached repositories +curl http://your-domain.com:5000/v2/_catalog + +# Check specific image tags +curl http://your-domain.com:5000/v2/nginx/tags/list +``` + +## Configuration + +### Environment Variables + +- `REGISTRY_DATA`: Data storage path (default: `/data/registry`) +- `REGISTRY_PORT`: Registry service port (default: `5000`) +- `REGISTRY_UI_PORT`: Web UI port (default: `5080`) + +### Advanced Configuration + +Edit `config.yml` to customize: + +- **Storage backend**: Filesystem, S3, GCS, etc. +- **Cache TTL**: How long to cache images +- **Authentication**: Enable user authentication +- **Proxy settings**: Configure upstream registries + +## Management Commands + +```bash +# Service management +make up # Start services +make down # Stop services +make restart # Restart services +make status # Show status +make log # Show logs + +# Maintenance +make health # Test health +make catalog # Show cached images +make backup # Backup data +make du # Show disk usage +make prune # Clean up unused data +make clean # Remove all data +``` + +## Monitoring + +### Health Check + +The registry provides health endpoints: + +- `GET /v2/` - Registry API version +- `GET /debug/health` - Detailed health status + +### Web UI + +Access the web interface at `http://your-domain.com:5080` to: + +- Browse cached images +- View image details and tags +- Delete unused images +- Monitor storage usage + +### Metrics + +For production deployments, consider adding: + +- Prometheus metrics collection +- Grafana dashboards +- Log aggregation +- Disk usage monitoring + +## Nginx Reverse Proxy + +For HTTPS and domain-based access, configure Nginx: + +```nginx +server { + listen 80; + server_name registry.your-domain.com; + + # Redirect to HTTPS + return 301 https://$server_name$request_uri; +} + +server { + listen 443 ssl; + server_name registry.your-domain.com; + + ssl_certificate /path/to/cert.pem; + ssl_certificate_key /path/to/key.pem; + + client_max_body_size 0; + chunked_transfer_encoding on; + + location / { + proxy_pass http://127.0.0.1:5000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + } +} +``` + +## Troubleshooting + +### Common Issues + +1. **Permission denied:** + ```bash + sudo chown -R $(whoami):$(whoami) /data/registry + ``` + +2. **Port already in use:** + ```bash + # Check what's using the port + sudo netstat -tulpn | grep :5000 + + # Modify port in docker-compose.yml + ``` + +3. **Docker daemon not using mirror:** + ```bash + # Check Docker configuration + docker info | grep -A5 "Registry Mirrors" + + # Restart Docker after config changes + sudo systemctl restart docker + ``` + +4. **Disk space issues:** + ```bash + # Check disk usage + make du + + # Clean up unused images + make prune + ``` + +### Performance Tuning + +- **Storage**: Use SSD storage for better performance +- **Memory**: Increase available memory for caching +- **Network**: Ensure good bandwidth to upstream registries +- **Cleanup**: Regular cleanup of unused images + +## Security Considerations + +- Use HTTPS in production +- Configure authentication if needed +- Restrict network access appropriately +- Regular security updates +- Monitor access logs + +## Contributing + +Contributions are welcome! Please feel free to submit issues and pull requests. + +## License + +This project is licensed under the same license as Pigsty. \ No newline at end of file diff --git a/app/registry/config.yml b/app/registry/config.yml new file mode 100644 index 0000000..e8b5194 --- /dev/null +++ b/app/registry/config.yml @@ -0,0 +1,49 @@ +version: 0.1 +log: + fields: + service: registry + level: info + +storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry + delete: + enabled: true + +http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] + Access-Control-Allow-Origin: ['*'] + Access-Control-Allow-Methods: ['HEAD', 'GET', 'OPTIONS', 'DELETE'] + Access-Control-Allow-Headers: ['Authorization', 'Accept', 'Cache-Control'] + Access-Control-Max-Age: [1728000] + Access-Control-Allow-Credentials: [true] + Access-Control-Expose-Headers: ['Docker-Content-Digest'] + +# Proxy configuration for multiple registries +proxy: + remoteurl: https://registry-1.docker.io + username: ~ + password: ~ + ttl: 168h + +# Alternative: Use specific proxy configurations +# This config supports pulling from different registries by URL path +# proxy: +# remoteurl: https://registry-1.docker.io # Default to Docker Hub + +# Health check endpoint +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 + +# Optional: Authentication (uncomment if needed) +# auth: +# htpasswd: +# realm: basic-realm +# path: /etc/docker/registry/htpasswd \ No newline at end of file diff --git a/app/registry/docker-compose.yml b/app/registry/docker-compose.yml new file mode 100644 index 0000000..99aa57a --- /dev/null +++ b/app/registry/docker-compose.yml @@ -0,0 +1,42 @@ +# Docker Registry Mirror for China Users +# Port: 5000 - Registry Service +# Port: 5080 - Registry Admin UI (optional) + +services: + registry: + container_name: registry + image: registry:2.8 + restart: always + environment: + REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY: /var/lib/registry + REGISTRY_HTTP_ADDR: 0.0.0.0:5000 + REGISTRY_LOG_LEVEL: info + REGISTRY_STORAGE_DELETE_ENABLED: true + ports: + - "5000:5000" + volumes: + - ${REGISTRY_DATA:-/data/registry}:/var/lib/registry + - ./config.yml:/etc/docker/registry/config.yml:ro + command: ["/etc/docker/registry/config.yml"] + + # Optional: Registry UI for management + registry-ui: + container_name: registry-ui + image: joxit/docker-registry-ui:2.5.7 + restart: always + environment: + SINGLE_REGISTRY: true + REGISTRY_TITLE: "Docker Registry Mirror" + DELETE_IMAGES: true + SHOW_CONTENT_DIGEST: true + NGINX_PROXY_PASS_URL: http://registry:5000 + SHOW_CATALOG_NB_TAGS: true + CATALOG_MIN_BRANCHES: 1 + CATALOG_MAX_BRANCHES: 1 + TAGLIST_PAGE_SIZE: 100 + REGISTRY_SECURED: false + CATALOG_ELEMENTS_LIMIT: 1000 + ports: + - "5080:80" + depends_on: + - registry \ No newline at end of file diff --git a/app/supabase/Makefile b/app/supabase/Makefile new file mode 100644 index 0000000..239c416 --- /dev/null +++ b/app/supabase/Makefile @@ -0,0 +1,50 @@ +default: up view + +IMG_DIR=/tmp/docker/supabase + +up: + docker compose up -d +start: up + +stop: down +down: + docker compose down + +view: + @echo "visit http://supa.pigsty for supabase studio" + +log: + docker compose logs -f + +info: + docker inspect supabase-studio | jq + +restart: + docker compose restart + +clean: + docker compose rm + +edit: + vi .env + +pull: + docker compose pull + +dump: save +save: + bin/dump + +# copy to target machine: scp -r /tmp/docker xxx:/tmp/docker +tz: tarball +tarball: save + tar -cvf /tmp/supabase.tgz -C /tmp/docker supabase + +copy: + ssh 10.10.10.10 'mkdir -p /tmp/docker/supabase' + rsync -avz /tmp/docker/supabase/ 10.10.10.10:/tmp/docker/supabase/ + +load: + bin/load + +.PHONY: default up down start stop view log info restart clean edit pull dump save load tz tarball \ No newline at end of file diff --git a/app/supabase/README.md b/app/supabase/README.md new file mode 100644 index 0000000..2e22786 --- /dev/null +++ b/app/supabase/README.md @@ -0,0 +1,48 @@ +# Supabase + +> [Supabase](https://supabase.com/) —— Build in a weekend, Scale to millions + +Pigsty allow you to self-host **supabase** with existing managed HA postgres cluster, and launch the stateless part of supabase with docker-compose. +Check the official tutorial for details: [Self-Hosting Supabase](https://pigsty.io/docs/app/supabase) + +Supabase is the open-source Firebase alternative built upon PostgreSQL. +It provides authentication, API, edge functions, real-time subscriptions, object storage, and vector embedding capabilities out of the box. +All you need to do is to design the database schema and frontend, and you can quickly get things done without worrying about the backend development. + +Supabase's slogan is: "**Build in a weekend, Scale to millions**". Supabase has great cost-effectiveness in small scales (4c8g) indeed. +But there is no doubt that when you really grow to millions of users, some may choose to self-hosting their own Supabase —— for functionality, performance, cost, and other reasons. + +That's where Pigsty comes in. Pigsty provides a complete one-click self-hosting solution for Supabase. +Self-hosted Supabase can enjoy full PostgreSQL monitoring, IaC, PITR, and high availability, the new PG 18 kernels (and 15~18), +and [437](https://pgext.cloud/list) PostgreSQL extensions ready to use, and can take full advantage of the performance and cost advantages of modern hardware. + + + +------- + +## Quick Start + +First, download & [install](https://pigsty.io/docs/setup/install) pigsty as usual, with the `supa` config template: + +```bash + curl -fsSL https://repo.pigsty.io/get | bash + cd pigsty +./configure -c supabase # use app/supa config template (IMPORTANT: CHANGE PASSWORDS!) +./deploy.yml # install pigsty, create ha postgres & minio clusters +``` + +Please change the `pigsty.yml` config file according to your need before deploying Supabase. (Credentials) + +Then, run the [`docker.yml`](https://github.com/pgsty/pigsty/blob/main/docker.yml) and [`app.yml`](https://github.com/pgsty/pigsty/blob/main/app.yml) to install supabase with docker. + +```bash +./docker.yml # install docker & docker compose +./app.yml # launch supabase stateless part with docker compose +``` + +You can access the supabase API / Web UI through the `80/443` infra portal, +with configured DNS for public domain, or a local `/etc/hosts` record with `supa.pigsty` pointing to the node also works. + +> Default username & password: `supabase` : `pigsty` + +> Beware the storage API require MinIO/S3 to work, and you have to access it via a valid domain name (`/etc/hosts` or real domain) \ No newline at end of file diff --git a/app/supabase/bin/dump b/app/supabase/bin/dump new file mode 100755 index 0000000..4d0cab2 --- /dev/null +++ b/app/supabase/bin/dump @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail +#==============================================================# +# File : app/supabase/bin/dump +# Desc : Dump Supabase Docker images to local dir in parallel +# Ctime : 2025-07-01 +# Mtime : 2025-07-01 +# Args : [directory] (default: /tmp/docker/supabase/) +# Usage : bin/load [/tmp/docker/supabase/] +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +#==============================================================# + +IMG_DIR=${1:-/tmp/docker/supabase} +mkdir -p ${IMG_DIR} +echo "Dumping Supabase Docker images to: $IMG_DIR" + +# Dump all images in parallel +docker save supabase/storage-api | gzip -c -9 > ${IMG_DIR}/storage.tgz & +docker save darthsim/imgproxy | gzip -c -9 > ${IMG_DIR}/imgproxy.tgz & +docker save supabase/studio | gzip -c -9 > ${IMG_DIR}/studio.tgz & +docker save supabase/realtime | gzip -c -9 > ${IMG_DIR}/realtime.tgz & +docker save supabase/edge-runtime | gzip -c -9 > ${IMG_DIR}/edge.tgz & +docker save supabase/gotrue | gzip -c -9 > ${IMG_DIR}/gotrue.tgz & +docker save supabase/postgres-meta | gzip -c -9 > ${IMG_DIR}/postgres.tgz & +docker save supabase/logflare | gzip -c -9 > ${IMG_DIR}/logflare.tgz & +docker save postgrest/postgrest | gzip -c -9 > ${IMG_DIR}/postgrest.tgz & +docker save timberio/vector | gzip -c -9 > ${IMG_DIR}/vector.tgz & +docker save kong | gzip -c -9 > ${IMG_DIR}/kong.tgz & + +# Wait for all background jobs to complete +wait +echo "All Supabase Docker images dump successfully!" +ls -alh ${IMG_DIR}/*.tgz diff --git a/app/supabase/bin/load b/app/supabase/bin/load new file mode 100755 index 0000000..88c9e4b --- /dev/null +++ b/app/supabase/bin/load @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail +#==============================================================# +# File : app/supabase/bin/load +# Desc : Load Supabase Docker images in parallel from local +# Ctime : 2025-07-01 +# Mtime : 2025-07-01 +# Args : [directory] - directory containing tgz files (default: /tmp/docker/supabase/) +# Usage : bin/load [/tmp/docker/supabase/] +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +#==============================================================# + +IMG_DIR=${1:-/tmp/docker/supabase} + +echo "Loading Supabase Docker images from: $IMG_DIR" +echo "Starting parallel image loading..." + +# Load all images in parallel +cat "$IMG_DIR/storage.tgz" | gzip -d -c - | docker load & +cat "$IMG_DIR/imgproxy.tgz" | gzip -d -c - | docker load & +cat "$IMG_DIR/studio.tgz" | gzip -d -c - | docker load & +cat "$IMG_DIR/realtime.tgz" | gzip -d -c - | docker load & +cat "$IMG_DIR/edge.tgz" | gzip -d -c - | docker load & +cat "$IMG_DIR/gotrue.tgz" | gzip -d -c - | docker load & +cat "$IMG_DIR/postgres.tgz" | gzip -d -c - | docker load & +cat "$IMG_DIR/logflare.tgz" | gzip -d -c - | docker load & +cat "$IMG_DIR/postgrest.tgz" | gzip -d -c - | docker load & +cat "$IMG_DIR/vector.tgz" | gzip -d -c - | docker load & +cat "$IMG_DIR/kong.tgz" | gzip -d -c - | docker load & + +# Wait for all background jobs to complete +wait + +echo "All Supabase Docker images loaded successfully!" \ No newline at end of file diff --git a/app/supabase/docker-compose.yml b/app/supabase/docker-compose.yml new file mode 100644 index 0000000..e46ba4a --- /dev/null +++ b/app/supabase/docker-compose.yml @@ -0,0 +1,392 @@ +#!/usr/bin/env docker compose +--- +#==============================================================# +# File : docker-compose.yml +# Desc : modified supabase docker compose for pigsty pg +# Ctime : 2023-09-19 +# Mtime : 2025-07-01 +# Path : app/supabase/docker-compose.yml +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +#==============================================================# +# Dir: https://github.com/supabase/supabase/blob/master/docker/ +# Source: https://github.com/supabase/supabase/blob/master/docker/docker-compose.yml +# Tutorial: https://supabase.com/docs/guides/self-hosting/docker + + +name: supabase +services: + + #--------------------------------------------------------------# + # Studio: the admin GUI dashboard for supabase + #--------------------------------------------------------------# + # https://supabase.com/docs/guides/getting-started/architecture#studio-dashboard + studio: + container_name: supabase-studio + image: supabase/studio:2025.11.10-sha-5291fe3 + restart: unless-stopped + healthcheck: + test: [ "CMD", "node", "-e", "fetch('http://studio:3000/api/platform/profile').then((r) => {if (r.status !== 200) throw new Error(r.status)})"] + timeout: 10s + interval: 5s + retries: 3 + depends_on: + analytics: + condition: service_healthy + environment: + # Binds nestjs listener to both IPv4 and IPv6 network interfaces + HOSTNAME: "::" + STUDIO_PG_META_URL: http://meta:8080 + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + PG_META_CRYPTO_KEY: ${PG_META_CRYPTO_KEY} + DEFAULT_ORGANIZATION_NAME: ${STUDIO_DEFAULT_ORGANIZATION} + DEFAULT_PROJECT_NAME: ${STUDIO_DEFAULT_PROJECT} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + SUPABASE_URL: http://kong:8000 + SUPABASE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} + SUPABASE_ANON_KEY: ${ANON_KEY} + SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} + AUTH_JWT_SECRET: ${JWT_SECRET} + LOGFLARE_PRIVATE_ACCESS_TOKEN: ${LOGFLARE_PRIVATE_ACCESS_TOKEN} + LOGFLARE_URL: http://analytics:4000 + NEXT_PUBLIC_ENABLE_LOGS: true + NEXT_ANALYTICS_BACKEND_PROVIDER: postgres + extra_hosts: + - "${POSTGRES_DOMAIN}:${POSTGRES_HOST}" + + + #--------------------------------------------------------------# + # Kong: the API gateway for all supabase services + #--------------------------------------------------------------# + # https://supabase.com/docs/guides/getting-started/architecture#kong-api-gateway + kong: + container_name: supabase-kong + image: kong:2.8.1 + restart: unless-stopped + ports: + - ${KONG_HTTP_PORT}:8000/tcp + - ${KONG_HTTPS_PORT}:8443/tcp + volumes: + - ./volumes/api/kong.yml:/home/kong/temp.yml:ro + depends_on: + analytics: + condition: service_healthy + environment: + KONG_DATABASE: "off" + KONG_DECLARATIVE_CONFIG: /home/kong/kong.yml + # https://github.com/supabase/cli/issues/14 + KONG_DNS_ORDER: LAST,A,CNAME + KONG_PLUGINS: request-transformer,cors,key-auth,acl,basic-auth,request-termination,ip-restriction + KONG_NGINX_PROXY_PROXY_BUFFER_SIZE: 160k + KONG_NGINX_PROXY_PROXY_BUFFERS: 64 160k + SUPABASE_ANON_KEY: ${ANON_KEY} + SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} + DASHBOARD_USERNAME: ${DASHBOARD_USERNAME} + DASHBOARD_PASSWORD: ${DASHBOARD_PASSWORD} + entrypoint: bash -c 'eval "echo \"$$(cat ~/temp.yml)\"" > ~/kong.yml && /docker-entrypoint.sh kong docker-start' + + + #--------------------------------------------------------------# + # Auth: JWT-based API for authentication using GoTrue + #--------------------------------------------------------------# + # https://supabase.com/docs/guides/getting-started/architecture#gotrue-auth + auth: + container_name: supabase-auth + image: supabase/gotrue:v2.182.1 + restart: unless-stopped + depends_on: + analytics: + condition: service_healthy + healthcheck: + test: [ "CMD","wget","--no-verbose","--tries=1","--spider","http://localhost:9999/health" ] + timeout: 5s + interval: 5s + retries: 3 + environment: + GOTRUE_API_HOST: 0.0.0.0 + GOTRUE_API_PORT: 9999 + API_EXTERNAL_URL: ${API_EXTERNAL_URL} + GOTRUE_DB_DRIVER: postgres + GOTRUE_DB_DATABASE_URL: postgres://supabase_auth_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + GOTRUE_SITE_URL: ${SITE_URL} + GOTRUE_URI_ALLOW_LIST: ${ADDITIONAL_REDIRECT_URLS} + GOTRUE_DISABLE_SIGNUP: ${DISABLE_SIGNUP} + GOTRUE_JWT_ADMIN_ROLES: service_role + GOTRUE_JWT_AUD: authenticated + GOTRUE_JWT_DEFAULT_GROUP_NAME: authenticated + GOTRUE_JWT_EXP: ${JWT_EXPIRY} + GOTRUE_JWT_SECRET: ${JWT_SECRET} + GOTRUE_EXTERNAL_EMAIL_ENABLED: ${ENABLE_EMAIL_SIGNUP} + GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED: ${ENABLE_ANONYMOUS_USERS} + GOTRUE_MAILER_AUTOCONFIRM: ${ENABLE_EMAIL_AUTOCONFIRM} + # Uncomment to bypass nonce check in ID Token flow. Commonly set to true when using Google Sign In on mobile. + # GOTRUE_EXTERNAL_SKIP_NONCE_CHECK: true + # GOTRUE_MAILER_SECURE_EMAIL_CHANGE_ENABLED: true + # GOTRUE_SMTP_MAX_FREQUENCY: 1s + GOTRUE_SMTP_ADMIN_EMAIL: ${SMTP_ADMIN_EMAIL} + GOTRUE_SMTP_HOST: ${SMTP_HOST} + GOTRUE_SMTP_PORT: ${SMTP_PORT} + GOTRUE_SMTP_USER: ${SMTP_USER} + GOTRUE_SMTP_PASS: ${SMTP_PASS} + GOTRUE_SMTP_SENDER_NAME: ${SMTP_SENDER_NAME} + # use absolute URLs for mailer URL paths + GOTRUE_MAILER_URLPATHS_INVITE: "${SUPABASE_PUBLIC_URL}/${MAILER_URLPATHS_INVITE}" + GOTRUE_MAILER_URLPATHS_CONFIRMATION: "${SUPABASE_PUBLIC_URL}/${MAILER_URLPATHS_CONFIRMATION}" + GOTRUE_MAILER_URLPATHS_RECOVERY: "${SUPABASE_PUBLIC_URL}/${MAILER_URLPATHS_RECOVERY}" + GOTRUE_MAILER_URLPATHS_EMAIL_CHANGE: "${SUPABASE_PUBLIC_URL}/${MAILER_URLPATHS_EMAIL_CHANGE}" + GOTRUE_EXTERNAL_PHONE_ENABLED: ${ENABLE_PHONE_SIGNUP} + GOTRUE_SMS_AUTOCONFIRM: ${ENABLE_PHONE_AUTOCONFIRM} + # Uncomment to enable custom access token hook. Please see: https://supabase.com/docs/guides/auth/auth-hooks for full list of hooks and additional details about custom_access_token_hook + # GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_ENABLED: "true" + # GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_URI: "pg-functions://postgres/public/custom_access_token_hook" + # GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_SECRETS: "" + # GOTRUE_HOOK_MFA_VERIFICATION_ATTEMPT_ENABLED: "true" + # GOTRUE_HOOK_MFA_VERIFICATION_ATTEMPT_URI: "pg-functions://postgres/public/mfa_verification_attempt" + # GOTRUE_HOOK_PASSWORD_VERIFICATION_ATTEMPT_ENABLED: "true" + # GOTRUE_HOOK_PASSWORD_VERIFICATION_ATTEMPT_URI: "pg-functions://postgres/public/password_verification_attempt" + # GOTRUE_HOOK_SEND_SMS_ENABLED: "false" + # GOTRUE_HOOK_SEND_SMS_URI: "pg-functions://postgres/public/custom_access_token_hook" + # GOTRUE_HOOK_SEND_SMS_SECRETS: "v1,whsec_VGhpcyBpcyBhbiBleGFtcGxlIG9mIGEgc2hvcnRlciBCYXNlNjQgc3RyaW5n" + # GOTRUE_HOOK_SEND_EMAIL_ENABLED: "false" + # GOTRUE_HOOK_SEND_EMAIL_URI: "http://host.docker.internal:54321/functions/v1/email_sender" + # GOTRUE_HOOK_SEND_EMAIL_SECRETS: "v1,whsec_VGhpcyBpcyBhbiBleGFtcGxlIG9mIGEgc2hvcnRlciBCYXNlNjQgc3RyaW5n" + extra_hosts: + - "${POSTGRES_DOMAIN}:${POSTGRES_HOST}" + + #--------------------------------------------------------------# + # Rest: Turn PostgreSQL into a RESTful API using PostgREST + #--------------------------------------------------------------# + # https://supabase.com/docs/guides/getting-started/architecture#postgrest-api + rest: + container_name: supabase-rest + image: postgrest/postgrest:v13.0.7 + restart: unless-stopped + depends_on: + analytics: + condition: service_healthy + environment: + PGRST_DB_URI: postgres://authenticator:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + PGRST_DB_SCHEMAS: ${PGRST_DB_SCHEMAS} + PGRST_DB_ANON_ROLE: anon + PGRST_JWT_SECRET: ${JWT_SECRET} + PGRST_DB_USE_LEGACY_GUCS: "false" + PGRST_APP_SETTINGS_JWT_SECRET: ${JWT_SECRET} + PGRST_APP_SETTINGS_JWT_EXP: ${JWT_EXPIRY} + command: "postgrest" + extra_hosts: + - "${POSTGRES_DOMAIN}:${POSTGRES_HOST}" + + #--------------------------------------------------------------# + # Realtime: A scalable websocket engine for realtime msg + #--------------------------------------------------------------# + # https://supabase.com/docs/guides/getting-started/architecture#realtime-api--multiplayer + realtime: + # This container name looks inconsistent but is correct because realtime constructs tenant id by parsing the subdomain + container_name: realtime-dev.supabase-realtime + image: supabase/realtime:v2.63.0 + restart: unless-stopped + depends_on: + analytics: + condition: service_healthy + healthcheck: + test: [ "CMD","curl","-sSfL","--head","-o","/dev/null","-H","Authorization: Bearer ${ANON_KEY}","http://localhost:4000/api/tenants/realtime-dev/health" ] + timeout: 5s + interval: 5s + retries: 3 + environment: + PORT: 4000 + DB_HOST: ${POSTGRES_HOST} + DB_PORT: ${POSTGRES_PORT} + DB_USER: supabase_admin + DB_PASSWORD: ${POSTGRES_PASSWORD} + DB_NAME: ${POSTGRES_DB} + DB_AFTER_CONNECT_QUERY: 'SET search_path TO _realtime' + DB_ENC_KEY: supabaserealtime + API_JWT_SECRET: ${JWT_SECRET} + SECRET_KEY_BASE: ${SECRET_KEY_BASE} + ERL_AFLAGS: -proto_dist inet_tcp + DNS_NODES: "''" + RLIMIT_NOFILE: "10000" + APP_NAME: realtime + SEED_SELF_HOST: true + RUN_JANITOR: true + extra_hosts: + - "${POSTGRES_DOMAIN}:${POSTGRES_HOST}" + + #--------------------------------------------------------------# + # Storage: S3-compatible OSS that stores metadata in Postgres + #--------------------------------------------------------------# + # https://supabase.com/docs/guides/getting-started/architecture#storage-api-large-file-storage + storage: + container_name: supabase-storage + image: supabase/storage-api:v1.29.0 + depends_on: + rest: + condition: service_started + imgproxy: + condition: service_started + healthcheck: + test: [ "CMD","wget","--no-verbose","--tries=1","--spider","http://storage:5000/status" ] + timeout: 5s + interval: 5s + retries: 3 + restart: unless-stopped + environment: + ANON_KEY: ${ANON_KEY} + SERVICE_KEY: ${SERVICE_ROLE_KEY} + POSTGREST_URL: http://rest:3000 + PGRST_JWT_SECRET: ${JWT_SECRET} + DATABASE_URL: postgres://supabase_storage_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + REQUEST_ALLOW_X_FORWARDED_PATH: "true" + FILE_SIZE_LIMIT: 52428800 + TENANT_ID: stub + ENABLE_IMAGE_TRANSFORMATION: "true" + IMGPROXY_URL: http://imgproxy:5001 + + # local file storage (use minio instead) + #STORAGE_BACKEND: file + #FILE_STORAGE_BACKEND_PATH: /var/lib/storage + + STORAGE_BACKEND: s3 + REGION: ${S3_REGION} + GLOBAL_S3_BUCKET: ${S3_BUCKET} + GLOBAL_S3_ENDPOINT: ${S3_ENDPOINT} + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY} + AWS_DEFAULT_REGION: ${S3_REGION} + GLOBAL_S3_PROTOCOL: ${S3_PROTOCOL} + GLOBAL_S3_FORCE_PATH_STYLE: ${S3_FORCE_PATH_STYLE} + + # add pigsty ca to the container + NODE_EXTRA_CA_CERTS: /etc/pki/ca.crt + + volumes: + - /etc/pki/ca.crt:/etc/pki/ca.crt:ro + - ./volumes/storage:/var/lib/storage:z + extra_hosts: + - "${POSTGRES_DOMAIN}:${POSTGRES_HOST}" + - "sss.pigsty:${MINIO_DOMAIN_IP}" + + + #--------------------------------------------------------------# + # imgproxy: the proxy for image + #--------------------------------------------------------------# + imgproxy: + container_name: supabase-imgproxy + image: darthsim/imgproxy:v3.8.0 + healthcheck: + test: [ "CMD", "imgproxy", "health" ] + timeout: 5s + interval: 5s + retries: 3 + environment: + IMGPROXY_BIND: ":5001" + IMGPROXY_LOCAL_FILESYSTEM_ROOT: / + IMGPROXY_USE_ETAG: "true" + IMGPROXY_ENABLE_WEBP_DETECTION: ${IMGPROXY_ENABLE_WEBP_DETECTION} + volumes: + - ./volumes/storage:/var/lib/storage:z + + + #--------------------------------------------------------------# + # postgres-meta: RESTful API Service to managing your Postgres + #--------------------------------------------------------------# + meta: + container_name: supabase-meta + image: supabase/postgres-meta:v0.93.1 + restart: unless-stopped + depends_on: + analytics: + condition: service_healthy + environment: + PG_META_PORT: 8080 + PG_META_DB_HOST: ${POSTGRES_HOST} + PG_META_DB_PORT: ${POSTGRES_PORT} + PG_META_DB_NAME: ${POSTGRES_DB} + PG_META_DB_USER: supabase_admin + PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} + CRYPTO_KEY: ${PG_META_CRYPTO_KEY} + + + #--------------------------------------------------------------# + # Edge Functions: A modern runtime for JavaScript and TypeScript + #--------------------------------------------------------------# + # https://supabase.com/docs/guides/getting-started/architecture#deno-edge-functions + functions: + container_name: supabase-edge-functions + image: supabase/edge-runtime:v1.69.23 + restart: unless-stopped + volumes: + - ./volumes/functions:/home/deno/functions:Z + depends_on: + analytics: + condition: service_healthy + environment: + JWT_SECRET: ${JWT_SECRET} + SUPABASE_URL: http://kong:8000 + SUPABASE_ANON_KEY: ${ANON_KEY} + SUPABASE_SERVICE_ROLE_KEY: ${SERVICE_ROLE_KEY} + SUPABASE_DB_URL: postgresql://postgres:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + # Allow configuring VERIFY_JWT per function. This PR might help: https://github.com/supabase/cli/pull/786 + VERIFY_JWT: "${FUNCTIONS_VERIFY_JWT}" + command: [ "start", "--main-service", "/home/deno/functions/main" ] + extra_hosts: + - "${POSTGRES_DOMAIN}:${POSTGRES_HOST}" + + #--------------------------------------------------------------# + # Analytics: analysis log events + #--------------------------------------------------------------# + # https://docs.logflare.app/self-hosting/ + analytics: + container_name: supabase-analytics + image: supabase/logflare:1.22.6 + healthcheck: + test: [ "CMD", "curl", "http://localhost:4000/health" ] + timeout: 5s + interval: 5s + retries: 10 + restart: unless-stopped + environment: + LOGFLARE_NODE_HOST: 127.0.0.1 + DB_USERNAME: supabase_admin + DB_DATABASE: supabase + DB_HOSTNAME: ${POSTGRES_HOST} + DB_PORT: ${POSTGRES_PORT} + DB_PASSWORD: ${POSTGRES_PASSWORD} + DB_SCHEMA: _analytics + LOGFLARE_LOG_LEVEL: ${LOGFLARE_LOG_LEVEL} + LOGFLARE_PUBLIC_ACCESS_TOKEN: ${LOGFLARE_PUBLIC_ACCESS_TOKEN} + LOGFLARE_PRIVATE_ACCESS_TOKEN: ${LOGFLARE_PRIVATE_ACCESS_TOKEN} + LOGFLARE_SINGLE_TENANT: true + LOGFLARE_SUPABASE_MODE: true + LOGFLARE_MIN_CLUSTER_SIZE: 1 + # Comment variables to use Big Query backend for analytics + POSTGRES_BACKEND_URL: postgresql://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/supabase + POSTGRES_BACKEND_SCHEMA: _analytics + LOGFLARE_FEATURE_FLAG_OVERRIDE: multibackend=false + ports: + - 4000:4000 + extra_hosts: + - "${POSTGRES_DOMAIN}:${POSTGRES_HOST}" + + #--------------------------------------------------------------# + # Vector: collecting logs + #--------------------------------------------------------------# + vector: + container_name: supabase-vector + image: timberio/vector:0.28.1-alpine + healthcheck: + test: [ "CMD", "wget", "--no-verbose", "--tries=1", "--spider","http://vector:9001/health" ] + timeout: 5s + interval: 5s + retries: 3 + environment: + LOGFLARE_PUBLIC_ACCESS_TOKEN: ${LOGFLARE_PUBLIC_ACCESS_TOKEN} + volumes: + - ./volumes/logs/vector.yml:/etc/vector/vector.yml:ro + - ${DOCKER_SOCKET_LOCATION}:/var/run/docker.sock:ro + command: ["--config", "/etc/vector/vector.yml"] +... diff --git a/app/supabase/volumes/api/kong.yml b/app/supabase/volumes/api/kong.yml new file mode 100644 index 0000000..673aa6d --- /dev/null +++ b/app/supabase/volumes/api/kong.yml @@ -0,0 +1,283 @@ +_format_version: '2.1' +_transform: true + +### +### Consumers / Users +### +consumers: + - username: DASHBOARD + - username: anon + keyauth_credentials: + - key: $SUPABASE_ANON_KEY + - username: service_role + keyauth_credentials: + - key: $SUPABASE_SERVICE_KEY + +### +### Access Control List +### +acls: + - consumer: anon + group: anon + - consumer: service_role + group: admin + +### +### Dashboard credentials +### +basicauth_credentials: + - consumer: DASHBOARD + username: $DASHBOARD_USERNAME + password: $DASHBOARD_PASSWORD + +### +### API Routes +### +services: + ## Open Auth routes + - name: auth-v1-open + url: http://auth:9999/verify + routes: + - name: auth-v1-open + strip_path: true + paths: + - /auth/v1/verify + plugins: + - name: cors + - name: auth-v1-open-callback + url: http://auth:9999/callback + routes: + - name: auth-v1-open-callback + strip_path: true + paths: + - /auth/v1/callback + plugins: + - name: cors + - name: auth-v1-open-authorize + url: http://auth:9999/authorize + routes: + - name: auth-v1-open-authorize + strip_path: true + paths: + - /auth/v1/authorize + plugins: + - name: cors + + ## Secure Auth routes + - name: auth-v1 + _comment: 'GoTrue: /auth/v1/* -> http://auth:9999/*' + url: http://auth:9999/ + routes: + - name: auth-v1-all + strip_path: true + paths: + - /auth/v1/ + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + + ## Secure REST routes + - name: rest-v1 + _comment: 'PostgREST: /rest/v1/* -> http://rest:3000/*' + url: http://rest:3000/ + routes: + - name: rest-v1-all + strip_path: true + paths: + - /rest/v1/ + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: true + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + + ## Secure GraphQL routes + - name: graphql-v1 + _comment: 'PostgREST: /graphql/v1/* -> http://rest:3000/rpc/graphql' + url: http://rest:3000/rpc/graphql + routes: + - name: graphql-v1-all + strip_path: true + paths: + - /graphql/v1 + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: true + - name: request-transformer + config: + add: + headers: + - Content-Profile:graphql_public + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + + ## Secure Realtime routes + - name: realtime-v1-ws + _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*' + url: http://realtime-dev.supabase-realtime:4000/socket + protocol: ws + routes: + - name: realtime-v1-ws + strip_path: true + paths: + - /realtime/v1/ + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + - name: realtime-v1-rest + _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*' + url: http://realtime-dev.supabase-realtime:4000/api + protocol: http + routes: + - name: realtime-v1-rest + strip_path: true + paths: + - /realtime/v1/api + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + ## Storage routes: the storage server manages its own auth + - name: storage-v1 + _comment: 'Storage: /storage/v1/* -> http://storage:5000/*' + url: http://storage:5000/ + routes: + - name: storage-v1-all + strip_path: true + paths: + - /storage/v1/ + plugins: + - name: cors + + ## Edge Functions routes + - name: functions-v1 + _comment: 'Edge Functions: /functions/v1/* -> http://functions:9000/*' + url: http://functions:9000/ + routes: + - name: functions-v1-all + strip_path: true + paths: + - /functions/v1/ + plugins: + - name: cors + + ## Analytics routes + - name: analytics-v1 + _comment: 'Analytics: /analytics/v1/* -> http://logflare:4000/*' + url: http://analytics:4000/ + routes: + - name: analytics-v1-all + strip_path: true + paths: + - /analytics/v1/ + + ## Secure Database routes + - name: meta + _comment: 'pg-meta: /pg/* -> http://pg-meta:8080/*' + url: http://meta:8080/ + routes: + - name: meta-all + strip_path: true + paths: + - /pg/ + plugins: + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + + ## Block access to /api/mcp + - name: mcp-blocker + _comment: 'Block direct access to /api/mcp' + url: http://studio:3000/api/mcp + routes: + - name: mcp-blocker-route + strip_path: true + paths: + - /api/mcp + plugins: + - name: request-termination + config: + status_code: 403 + message: "Access is forbidden." + + ## MCP endpoint - local access + - name: mcp + _comment: 'MCP: /mcp -> http://studio:3000/api/mcp (local access)' + url: http://studio:3000/api/mcp + routes: + - name: mcp + strip_path: true + paths: + - /mcp + plugins: + # Block access to /mcp by default + - name: request-termination + config: + status_code: 403 + message: "Access is forbidden." + # Enable local access (danger zone!) + # 1. Comment out the 'request-termination' section above + # 2. Uncomment the entire section below, including 'deny' + # 3. Add your local IPs to the 'allow' list + #- name: cors + #- name: ip-restriction + # config: + # allow: + # - 127.0.0.1 + # - ::1 + # deny: [] + + ## Protected Dashboard - catch all remaining routes + - name: dashboard + _comment: 'Studio: /* -> http://studio:3000/*' + url: http://studio:3000/ + routes: + - name: dashboard-all + strip_path: true + paths: + - / + plugins: + - name: cors + - name: basic-auth + config: + hide_credentials: true \ No newline at end of file diff --git a/app/supabase/volumes/functions/hello/index.ts b/app/supabase/volumes/functions/hello/index.ts new file mode 100644 index 0000000..f1e20b9 --- /dev/null +++ b/app/supabase/volumes/functions/hello/index.ts @@ -0,0 +1,16 @@ +// Follow this setup guide to integrate the Deno language server with your editor: +// https://deno.land/manual/getting_started/setup_your_environment +// This enables autocomplete, go to definition, etc. + +import { serve } from "https://deno.land/std@0.177.1/http/server.ts" + +serve(async () => { + return new Response( + `"Hello from Edge Functions!"`, + { headers: { "Content-Type": "application/json" } }, + ) +}) + +// To invoke: +// curl 'http://localhost:/functions/v1/hello' \ +// --header 'Authorization: Bearer ' diff --git a/app/supabase/volumes/functions/main/index.ts b/app/supabase/volumes/functions/main/index.ts new file mode 100644 index 0000000..a094010 --- /dev/null +++ b/app/supabase/volumes/functions/main/index.ts @@ -0,0 +1,94 @@ +import { serve } from 'https://deno.land/std@0.131.0/http/server.ts' +import * as jose from 'https://deno.land/x/jose@v4.14.4/index.ts' + +console.log('main function started') + +const JWT_SECRET = Deno.env.get('JWT_SECRET') +const VERIFY_JWT = Deno.env.get('VERIFY_JWT') === 'true' + +function getAuthToken(req: Request) { + const authHeader = req.headers.get('authorization') + if (!authHeader) { + throw new Error('Missing authorization header') + } + const [bearer, token] = authHeader.split(' ') + if (bearer !== 'Bearer') { + throw new Error(`Auth header is not 'Bearer {token}'`) + } + return token +} + +async function verifyJWT(jwt: string): Promise { + const encoder = new TextEncoder() + const secretKey = encoder.encode(JWT_SECRET) + try { + await jose.jwtVerify(jwt, secretKey) + } catch (err) { + console.error(err) + return false + } + return true +} + +serve(async (req: Request) => { + if (req.method !== 'OPTIONS' && VERIFY_JWT) { + try { + const token = getAuthToken(req) + const isValidJWT = await verifyJWT(token) + + if (!isValidJWT) { + return new Response(JSON.stringify({ msg: 'Invalid JWT' }), { + status: 401, + headers: { 'Content-Type': 'application/json' }, + }) + } + } catch (e) { + console.error(e) + return new Response(JSON.stringify({ msg: e.toString() }), { + status: 401, + headers: { 'Content-Type': 'application/json' }, + }) + } + } + + const url = new URL(req.url) + const { pathname } = url + const path_parts = pathname.split('/') + const service_name = path_parts[1] + + if (!service_name || service_name === '') { + const error = { msg: 'missing function name in request' } + return new Response(JSON.stringify(error), { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }) + } + + const servicePath = `/home/deno/functions/${service_name}` + console.error(`serving the request with ${servicePath}`) + + const memoryLimitMb = 150 + const workerTimeoutMs = 1 * 60 * 1000 + const noModuleCache = false + const importMapPath = null + const envVarsObj = Deno.env.toObject() + const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]) + + try { + const worker = await EdgeRuntime.userWorkers.create({ + servicePath, + memoryLimitMb, + workerTimeoutMs, + noModuleCache, + importMapPath, + envVars, + }) + return await worker.fetch(req) + } catch (e) { + const error = { msg: e.toString() } + return new Response(JSON.stringify(error), { + status: 500, + headers: { 'Content-Type': 'application/json' }, + }) + } +}) diff --git a/app/supabase/volumes/logs/vector.yml b/app/supabase/volumes/logs/vector.yml new file mode 100644 index 0000000..65d5310 --- /dev/null +++ b/app/supabase/volumes/logs/vector.yml @@ -0,0 +1,253 @@ +api: + enabled: true + address: 0.0.0.0:9001 + +sources: + docker_host: + type: docker_logs + exclude_containers: + - supabase-vector + +transforms: + project_logs: + type: remap + inputs: + - docker_host + source: |- + .project = "default" + .event_message = del(.message) + .appname = del(.container_name) + del(.container_created_at) + del(.container_id) + del(.source_type) + del(.stream) + del(.label) + del(.image) + del(.host) + del(.stream) + router: + type: route + inputs: + - project_logs + route: + kong: '.appname == "supabase-kong"' + auth: '.appname == "supabase-auth"' + rest: '.appname == "supabase-rest"' + realtime: '.appname == "realtime-dev.supabase-realtime"' + storage: '.appname == "supabase-storage"' + functions: '.appname == "supabase-edge-functions"' + db: '.appname == "supabase-db"' + # Ignores non nginx errors since they are related with kong booting up + kong_logs: + type: remap + inputs: + - router.kong + source: |- + req, err = parse_nginx_log(.event_message, "combined") + if err == null { + .timestamp = req.timestamp + .metadata.request.headers.referer = req.referer + .metadata.request.headers.user_agent = req.agent + .metadata.request.headers.cf_connecting_ip = req.client + .metadata.request.method = req.method + .metadata.request.path = req.path + .metadata.request.protocol = req.protocol + .metadata.response.status_code = req.status + } + if err != null { + abort + } + # Ignores non nginx errors since they are related with kong booting up + kong_err: + type: remap + inputs: + - router.kong + source: |- + .metadata.request.method = "GET" + .metadata.response.status_code = 200 + parsed, err = parse_nginx_log(.event_message, "error") + if err == null { + .timestamp = parsed.timestamp + .severity = parsed.severity + .metadata.request.host = parsed.host + .metadata.request.headers.cf_connecting_ip = parsed.client + url, err = split(parsed.request, " ") + if err == null { + .metadata.request.method = url[0] + .metadata.request.path = url[1] + .metadata.request.protocol = url[2] + } + } + if err != null { + abort + } + # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency. + auth_logs: + type: remap + inputs: + - router.auth + source: |- + parsed, err = parse_json(.event_message) + if err == null { + .metadata.timestamp = parsed.time + .metadata = merge!(.metadata, parsed) + } + # PostgREST logs are structured so we separate timestamp from message using regex + rest_logs: + type: remap + inputs: + - router.rest + source: |- + parsed, err = parse_regex(.event_message, r'^(?P