diff --git a/app/dify/.env b/app/dify/.env new file mode 100644 index 0000000..7bcea75 --- /dev/null +++ b/app/dify/.env @@ -0,0 +1,800 @@ +# ------------------------------ +# Environment Variables for API service & worker +# ------------------------------ +# https://docs.dify.ai/zh-hans/getting-started/install-self-hosted/environments +DIFY_DATA=./volumes + +# ------------------------------ +# Common Variables +# ------------------------------ + +# The backend URL of the console API, +# used to concatenate the authorization callback. +# If empty, it is the same domain. +# Example: https://api.console.dify.ai +CONSOLE_API_URL= + +# The front-end URL of the console web, +# used to concatenate some front-end addresses and for CORS configuration use. +# If empty, it is the same domain. +# Example: https://console.dify.ai +CONSOLE_WEB_URL= + +# Service API Url, +# used to display Service API Base Url to the front-end. +# If empty, it is the same domain. +# Example: https://api.dify.ai +SERVICE_API_URL= + +# WebApp API backend Url, +# used to declare the back-end URL for the front-end API. +# If empty, it is the same domain. +# Example: https://api.app.dify.ai +APP_API_URL= + +# WebApp Url, +# used to display WebAPP API Base Url to the front-end. +# If empty, it is the same domain. +# Example: https://app.dify.ai +APP_WEB_URL= + +# File preview or download Url prefix. +# used to display File preview or download Url to the front-end or as Multi-model inputs; +# Url is signed and has expiration time. +FILES_URL= + +# ------------------------------ +# Server Configuration +# ------------------------------ + +# The log level for the application. +# Supported values are `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` +LOG_LEVEL=INFO +# Log file path +LOG_FILE=/app/logs/server.log +# Log file max size, the unit is MB +LOG_FILE_MAX_SIZE=20 +# Log file max backup count +LOG_FILE_BACKUP_COUNT=5 +# Log dateformat +LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S +# Log Timezone +LOG_TZ=UTC + +# Debug mode, default is false. +# It is recommended to turn on this configuration for local development +# to prevent some problems caused by monkey patch. +DEBUG=false + +# Flask debug mode, it can output trace information at the interface when turned on, +# which is convenient for debugging. +FLASK_DEBUG=false + +# A secret key that is used for securely signing the session cookie +# and encrypting sensitive information on the database. +# You can generate a strong key using `openssl rand -base64 42`. +SECRET_KEY=your-secret-key-placeholder + +# Password for admin user initialization. +# If left unset, admin user will not be prompted for a password +# when creating the initial admin account. +# The length of the password cannot exceed 30 characters. +INIT_PASSWORD= + +# Deployment environment. +# Supported values are `PRODUCTION`, `TESTING`. Default is `PRODUCTION`. +# Testing environment. There will be a distinct color label on the front-end page, +# indicating that this environment is a testing environment. +DEPLOY_ENV=PRODUCTION + +# Whether to enable the version check policy. +# If set to empty, https://updates.dify.ai will be called for version check. +CHECK_UPDATE_URL=https://updates.dify.ai + +# Used to change the OpenAI base address, default is https://api.openai.com/v1. +# When OpenAI cannot be accessed in China, replace it with a domestic mirror address, +# or when a local model provides OpenAI compatible API, it can be replaced. +OPENAI_API_BASE=https://api.openai.com/v1 + +# When enabled, migrations will be executed prior to application startup +# and the application will start after the migrations have completed. +MIGRATION_ENABLED=true + +# File Access Time specifies a time interval in seconds for the file to be accessed. +# The default value is 300 seconds. +FILES_ACCESS_TIMEOUT=300 + +# Access token expiration time in minutes +ACCESS_TOKEN_EXPIRE_MINUTES=60 + +# Refresh token expiration time in days +REFRESH_TOKEN_EXPIRE_DAYS=30 + +# The maximum number of active requests for the application, where 0 means unlimited, should be a non-negative integer. +APP_MAX_ACTIVE_REQUESTS=0 +APP_MAX_EXECUTION_TIME=1200 + +# ------------------------------ +# Container Startup Related Configuration +# Only effective when starting with docker image or docker-compose. +# ------------------------------ + +# API service binding address, default: 0.0.0.0, i.e., all addresses can be accessed. +DIFY_BIND_ADDRESS=0.0.0.0 + +# API service binding port number, default 5001. +DIFY_PORT=5001 + +# The number of API server workers, i.e., the number of workers. +# Formula: number of cpu cores x 2 + 1 for sync, 1 for Gevent +# Reference: https://docs.gunicorn.org/en/stable/design.html#how-many-workers +SERVER_WORKER_AMOUNT=1 + +# Defaults to gevent. If using windows, it can be switched to sync or solo. +SERVER_WORKER_CLASS=gevent + +# Default number of worker connections, the default is 10. +SERVER_WORKER_CONNECTIONS=10 + +# Similar to SERVER_WORKER_CLASS. +# If using windows, it can be switched to sync or solo. +CELERY_WORKER_CLASS= + +# Request handling timeout. The default is 200, +# it is recommended to set it to 360 to support a longer sse connection time. +GUNICORN_TIMEOUT=360 + +# The number of Celery workers. The default is 1, and can be set as needed. +CELERY_WORKER_AMOUNT= + +# Flag indicating whether to enable autoscaling of Celery workers. +# +# Autoscaling is useful when tasks are CPU intensive and can be dynamically +# allocated and deallocated based on the workload. +# +# When autoscaling is enabled, the maximum and minimum number of workers can +# be specified. The autoscaling algorithm will dynamically adjust the number +# of workers within the specified range. +# +# Default is false (i.e., autoscaling is disabled). +# +# Example: +# CELERY_AUTO_SCALE=true +CELERY_AUTO_SCALE=false + +# The maximum number of Celery workers that can be autoscaled. +# This is optional and only used when autoscaling is enabled. +# Default is not set. +CELERY_MAX_WORKERS= + +# The minimum number of Celery workers that can be autoscaled. +# This is optional and only used when autoscaling is enabled. +# Default is not set. +CELERY_MIN_WORKERS= + +# API Tool configuration +API_TOOL_DEFAULT_CONNECT_TIMEOUT=10 +API_TOOL_DEFAULT_READ_TIMEOUT=60 + + +# ------------------------------ +# Database Configuration +# The database uses PostgreSQL. Please use the public schema. +# It is consistent with the configuration in the 'db' service below. +# ------------------------------ + +DB_USERNAME=postgres +DB_PASSWORD=your-db-password +DB_HOST=db +DB_PORT=5432 +DB_DATABASE=dify +# The size of the database connection pool. +# The default is 30 connections, which can be appropriately increased. +SQLALCHEMY_POOL_SIZE=30 +# Database connection pool recycling time, the default is 3600 seconds. +SQLALCHEMY_POOL_RECYCLE=3600 +# Whether to print SQL, default is false. +SQLALCHEMY_ECHO=false + +# Maximum number of connections to the database +# Default is 100 +# +# Reference: https://www.postgresql.org/docs/current/runtime-config-connection.html#GUC-MAX-CONNECTIONS +POSTGRES_MAX_CONNECTIONS=100 + +# Sets the amount of shared memory used for postgres's shared buffers. +# Default is 128MB +# Recommended value: 25% of available memory +# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-SHARED-BUFFERS +POSTGRES_SHARED_BUFFERS=128MB + +# Sets the amount of memory used by each database worker for working space. +# Default is 4MB +# +# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-WORK-MEM +POSTGRES_WORK_MEM=4MB + +# Sets the amount of memory reserved for maintenance activities. +# Default is 64MB +# +# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-MAINTENANCE-WORK-MEM +POSTGRES_MAINTENANCE_WORK_MEM=64MB + +# Sets the planner's assumption about the effective cache size. +# Default is 4096MB +# +# Reference: https://www.postgresql.org/docs/current/runtime-config-query.html#GUC-EFFECTIVE-CACHE-SIZE +POSTGRES_EFFECTIVE_CACHE_SIZE=4096MB + +# ------------------------------ +# Redis Configuration +# This Redis configuration is used for caching and for pub/sub during conversation. +# ------------------------------ + +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_USERNAME= +REDIS_PASSWORD=your-redis-password +REDIS_USE_SSL=false +REDIS_DB=0 + +# Whether to use Redis Sentinel mode. +# If set to true, the application will automatically discover and connect to the master node through Sentinel. +REDIS_USE_SENTINEL=false + +# List of Redis Sentinel nodes. If Sentinel mode is enabled, provide at least one Sentinel IP and port. +# Format: `:,:,:` +REDIS_SENTINELS= +REDIS_SENTINEL_SERVICE_NAME= +REDIS_SENTINEL_USERNAME= +REDIS_SENTINEL_PASSWORD= +REDIS_SENTINEL_SOCKET_TIMEOUT=0.1 + +# List of Redis Cluster nodes. If Cluster mode is enabled, provide at least one Cluster IP and port. +# Format: `:,:,:` +REDIS_USE_CLUSTERS=false +REDIS_CLUSTERS= +REDIS_CLUSTERS_PASSWORD= + +# ------------------------------ +# Celery Configuration +# ------------------------------ + +# Use redis as the broker, and redis db 1 for celery broker. +# Format as follows: `redis://:@:/` +# Example: redis://:difyai123456@redis:6379/1 +# If use Redis Sentinel, format as follows: `sentinel://:@:/` +# Example: sentinel://localhost:26379/1;sentinel://localhost:26380/1;sentinel://localhost:26381/1 +CELERY_BROKER_URL=redis://:difyai123456@redis:6379/1 +BROKER_USE_SSL=false + +# If you are using Redis Sentinel for high availability, configure the following settings. +CELERY_USE_SENTINEL=false +CELERY_SENTINEL_MASTER_NAME= +CELERY_SENTINEL_SOCKET_TIMEOUT=0.1 + +# ------------------------------ +# CORS Configuration +# Used to set the front-end cross-domain access policy. +# ------------------------------ + +# Specifies the allowed origins for cross-origin requests to the Web API, +# e.g. https://dify.app or * for all origins. +WEB_API_CORS_ALLOW_ORIGINS=* + +# Specifies the allowed origins for cross-origin requests to the console API, +# e.g. https://cloud.dify.ai or * for all origins. +CONSOLE_CORS_ALLOW_ORIGINS=* + +# ------------------------------ +# File Storage Configuration +# ------------------------------ + +# The type of storage to use for storing user files. +STORAGE_TYPE=opendal + +# Apache OpenDAL Configuration +# The configuration for OpenDAL consists of the following format: OPENDAL__. +# You can find all the service configurations (CONFIG_NAME) in the repository at: https://github.com/apache/opendal/tree/main/core/src/services. +# Dify will scan configurations starting with OPENDAL_ and automatically apply them. +# The scheme name for the OpenDAL storage. +OPENDAL_SCHEME=fs +# Configurations for OpenDAL Local File System. +OPENDAL_FS_ROOT=storage + +# S3 Configuration +# +S3_ENDPOINT= +S3_REGION=us-east-1 +S3_BUCKET_NAME=difyai +S3_ACCESS_KEY= +S3_SECRET_KEY= +# Whether to use AWS managed IAM roles for authenticating with the S3 service. +# If set to false, the access key and secret key must be provided. +S3_USE_AWS_MANAGED_IAM=false + +# Azure Blob Configuration +# +AZURE_BLOB_ACCOUNT_NAME=difyai +AZURE_BLOB_ACCOUNT_KEY=difyai +AZURE_BLOB_CONTAINER_NAME=difyai-container +AZURE_BLOB_ACCOUNT_URL=https://.blob.core.windows.net + +# Google Storage Configuration +# +GOOGLE_STORAGE_BUCKET_NAME=your-bucket-name +GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64= + +# The Alibaba Cloud OSS configurations, +# +ALIYUN_OSS_BUCKET_NAME=your-bucket-name +ALIYUN_OSS_ACCESS_KEY=your-access-key +ALIYUN_OSS_SECRET_KEY=your-secret-key +ALIYUN_OSS_ENDPOINT=https://oss-ap-southeast-1-internal.aliyuncs.com +ALIYUN_OSS_REGION=ap-southeast-1 +ALIYUN_OSS_AUTH_VERSION=v4 +# Don't start with '/'. OSS doesn't support leading slash in object names. +ALIYUN_OSS_PATH=your-path + +# Tencent COS Configuration +# +TENCENT_COS_BUCKET_NAME=your-bucket-name +TENCENT_COS_SECRET_KEY=your-secret-key +TENCENT_COS_SECRET_ID=your-secret-id +TENCENT_COS_REGION=your-region +TENCENT_COS_SCHEME=your-scheme + +# Oracle Storage Configuration +# +OCI_ENDPOINT=https://your-object-storage-namespace.compat.objectstorage.us-ashburn-1.oraclecloud.com +OCI_BUCKET_NAME=your-bucket-name +OCI_ACCESS_KEY=your-access-key +OCI_SECRET_KEY=your-secret-key +OCI_REGION=us-ashburn-1 + +# Huawei OBS Configuration +# +HUAWEI_OBS_BUCKET_NAME=your-bucket-name +HUAWEI_OBS_SECRET_KEY=your-secret-key +HUAWEI_OBS_ACCESS_KEY=your-access-key +HUAWEI_OBS_SERVER=your-server-url + +# Volcengine TOS Configuration +# +VOLCENGINE_TOS_BUCKET_NAME=your-bucket-name +VOLCENGINE_TOS_SECRET_KEY=your-secret-key +VOLCENGINE_TOS_ACCESS_KEY=your-access-key +VOLCENGINE_TOS_ENDPOINT=your-server-url +VOLCENGINE_TOS_REGION=your-region + +# Baidu OBS Storage Configuration +# +BAIDU_OBS_BUCKET_NAME=your-bucket-name +BAIDU_OBS_SECRET_KEY=your-secret-key +BAIDU_OBS_ACCESS_KEY=your-access-key +BAIDU_OBS_ENDPOINT=your-server-url + +# Supabase Storage Configuration +# +SUPABASE_BUCKET_NAME=your-bucket-name +SUPABASE_API_KEY=your-access-key +SUPABASE_URL=your-server-url + +# ------------------------------ +# Vector Database Configuration +# ------------------------------ + +# The type of vector store to use. +# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `tidb_vector`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`, `opengauss`, `tablestore`. +VECTOR_STORE=weaviate + +# The Weaviate endpoint URL. Only available when VECTOR_STORE is `weaviate`. +WEAVIATE_ENDPOINT=http://weaviate:8080 +WEAVIATE_API_KEY=your-weaviate-api-key + +# The Qdrant endpoint URL. Only available when VECTOR_STORE is `qdrant`. +QDRANT_URL=http://qdrant:6333 +QDRANT_API_KEY=your-qdrant-api-key +QDRANT_CLIENT_TIMEOUT=20 +QDRANT_GRPC_ENABLED=false +QDRANT_GRPC_PORT=6334 + +# Milvus configuration. Only available when VECTOR_STORE is `milvus`. +# The milvus uri. +MILVUS_URI=http://host.docker.internal:19530 +MILVUS_TOKEN= +MILVUS_USER= +MILVUS_PASSWORD= +MILVUS_ENABLE_HYBRID_SEARCH=False + +# MyScale configuration, only available when VECTOR_STORE is `myscale` +# For multi-language support, please set MYSCALE_FTS_PARAMS with referring to: +# https://myscale.com/docs/en/text-search/#understanding-fts-index-parameters +MYSCALE_HOST=myscale +MYSCALE_PORT=8123 +MYSCALE_USER=default +MYSCALE_PASSWORD= +MYSCALE_DATABASE=dify +MYSCALE_FTS_PARAMS= + +# Couchbase configurations, only available when VECTOR_STORE is `couchbase` +# The connection string must include hostname defined in the docker-compose file (couchbase-server in this case) +COUCHBASE_CONNECTION_STRING=couchbase://couchbase-server +COUCHBASE_USER=Administrator +COUCHBASE_PASSWORD=password +COUCHBASE_BUCKET_NAME=Embeddings +COUCHBASE_SCOPE_NAME=_default + +# pgvector configurations, only available when VECTOR_STORE is `pgvector` +PGVECTOR_HOST=pgvector +PGVECTOR_PORT=5432 +PGVECTOR_USER=postgres +PGVECTOR_PASSWORD=your-pgvector-password +PGVECTOR_DATABASE=dify +PGVECTOR_MIN_CONNECTION=1 +PGVECTOR_MAX_CONNECTION=5 +PGVECTOR_PG_BIGM=false +PGVECTOR_PG_BIGM_VERSION=1.2-20240606 + +# pgvecto-rs configurations, only available when VECTOR_STORE is `pgvecto-rs` +PGVECTO_RS_HOST=pgvecto-rs +PGVECTO_RS_PORT=5432 +PGVECTO_RS_USER=postgres +PGVECTO_RS_PASSWORD=your-pgvecto-rs-password +PGVECTO_RS_DATABASE=dify + +# analyticdb configurations, only available when VECTOR_STORE is `analyticdb` +ANALYTICDB_KEY_ID=your-ak +ANALYTICDB_KEY_SECRET=your-sk +ANALYTICDB_REGION_ID=cn-hangzhou +ANALYTICDB_INSTANCE_ID=gp-ab123456 +ANALYTICDB_ACCOUNT=testaccount +ANALYTICDB_PASSWORD=testpassword +ANALYTICDB_NAMESPACE=dify +ANALYTICDB_NAMESPACE_PASSWORD=difypassword +ANALYTICDB_HOST=gp-test.aliyuncs.com +ANALYTICDB_PORT=5432 +ANALYTICDB_MIN_CONNECTION=1 +ANALYTICDB_MAX_CONNECTION=5 + +# TiDB vector configurations, only available when VECTOR_STORE is `tidb` +TIDB_VECTOR_HOST=tidb +TIDB_VECTOR_PORT=4000 +TIDB_VECTOR_USER= +TIDB_VECTOR_PASSWORD= +TIDB_VECTOR_DATABASE=dify + +# Tidb on qdrant configuration, only available when VECTOR_STORE is `tidb_on_qdrant` +TIDB_ON_QDRANT_URL=http://127.0.0.1 +TIDB_ON_QDRANT_API_KEY=dify +TIDB_ON_QDRANT_CLIENT_TIMEOUT=20 +TIDB_ON_QDRANT_GRPC_ENABLED=false +TIDB_ON_QDRANT_GRPC_PORT=6334 +TIDB_PUBLIC_KEY=dify +TIDB_PRIVATE_KEY=dify +TIDB_API_URL=http://127.0.0.1 +TIDB_IAM_API_URL=http://127.0.0.1 +TIDB_REGION=regions/aws-us-east-1 +TIDB_PROJECT_ID=dify +TIDB_SPEND_LIMIT=100 + +# Chroma configuration, only available when VECTOR_STORE is `chroma` +CHROMA_HOST=127.0.0.1 +CHROMA_PORT=8000 +CHROMA_TENANT=default_tenant +CHROMA_DATABASE=default_database +CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthClientProvider +CHROMA_AUTH_CREDENTIALS= + +# Oracle configuration, only available when VECTOR_STORE is `oracle` +ORACLE_USER=dify +ORACLE_PASSWORD=dify +ORACLE_DSN=oracle:1521/FREEPDB1 +ORACLE_CONFIG_DIR=/app/api/storage/wallet +ORACLE_WALLET_LOCATION=/app/api/storage/wallet +ORACLE_WALLET_PASSWORD=dify +ORACLE_IS_AUTONOMOUS=false + +# relyt configurations, only available when VECTOR_STORE is `relyt` +RELYT_HOST=db +RELYT_PORT=5432 +RELYT_USER=postgres +RELYT_PASSWORD=your-relyt-password +RELYT_DATABASE=postgres + +# open search configuration, only available when VECTOR_STORE is `opensearch` +OPENSEARCH_HOST=opensearch +OPENSEARCH_PORT=9200 +OPENSEARCH_USER=admin +OPENSEARCH_PASSWORD=admin +OPENSEARCH_SECURE=true + +# tencent vector configurations, only available when VECTOR_STORE is `tencent` +TENCENT_VECTOR_DB_URL=http://127.0.0.1 +TENCENT_VECTOR_DB_API_KEY=dify +TENCENT_VECTOR_DB_TIMEOUT=30 +TENCENT_VECTOR_DB_USERNAME=dify +TENCENT_VECTOR_DB_DATABASE=dify +TENCENT_VECTOR_DB_SHARD=1 +TENCENT_VECTOR_DB_REPLICAS=2 + +# ElasticSearch configuration, only available when VECTOR_STORE is `elasticsearch` +ELASTICSEARCH_HOST=0.0.0.0 +ELASTICSEARCH_PORT=9200 +ELASTICSEARCH_USERNAME=elastic +ELASTICSEARCH_PASSWORD=elastic +KIBANA_PORT=5601 + +# baidu vector configurations, only available when VECTOR_STORE is `baidu` +BAIDU_VECTOR_DB_ENDPOINT=http://127.0.0.1:5287 +BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS=30000 +BAIDU_VECTOR_DB_ACCOUNT=root +BAIDU_VECTOR_DB_API_KEY=dify +BAIDU_VECTOR_DB_DATABASE=dify +BAIDU_VECTOR_DB_SHARD=1 +BAIDU_VECTOR_DB_REPLICAS=3 + +# VikingDB configurations, only available when VECTOR_STORE is `vikingdb` +VIKINGDB_ACCESS_KEY=your-ak +VIKINGDB_SECRET_KEY=your-sk +VIKINGDB_REGION=cn-shanghai +VIKINGDB_HOST=api-vikingdb.xxx.volces.com +VIKINGDB_SCHEMA=http +VIKINGDB_CONNECTION_TIMEOUT=30 +VIKINGDB_SOCKET_TIMEOUT=30 + +# Lindorm configuration, only available when VECTOR_STORE is `lindorm` +LINDORM_URL=http://lindorm:30070 +LINDORM_USERNAME=lindorm +LINDORM_PASSWORD=lindorm + +# OceanBase Vector configuration, only available when VECTOR_STORE is `oceanbase` +OCEANBASE_VECTOR_HOST=oceanbase +OCEANBASE_VECTOR_PORT=2881 +OCEANBASE_VECTOR_USER=root@test +OCEANBASE_VECTOR_PASSWORD=your-oceanbase-password +OCEANBASE_VECTOR_DATABASE=test +OCEANBASE_CLUSTER_NAME=difyai +OCEANBASE_MEMORY_LIMIT=6G +OCEANBASE_ENABLE_HYBRID_SEARCH=false + +# opengauss configurations, only available when VECTOR_STORE is `opengauss` +OPENGAUSS_HOST=opengauss +OPENGAUSS_PORT=6600 +OPENGAUSS_USER=postgres +OPENGAUSS_PASSWORD=Dify@123 +OPENGAUSS_DATABASE=dify +OPENGAUSS_MIN_CONNECTION=1 +OPENGAUSS_MAX_CONNECTION=5 +OPENGAUSS_ENABLE_PQ=false + +# Upstash Vector configuration, only available when VECTOR_STORE is `upstash` +UPSTASH_VECTOR_URL=https://xxx-vector.upstash.io +UPSTASH_VECTOR_TOKEN=dify + +# TableStore Vector configuration +# (only used when VECTOR_STORE is tablestore) +TABLESTORE_ENDPOINT=https://instance-name.cn-hangzhou.ots.aliyuncs.com +TABLESTORE_INSTANCE_NAME=instance-name +TABLESTORE_ACCESS_KEY_ID=xxx +TABLESTORE_ACCESS_KEY_SECRET=xxx + +# ------------------------------ +# Knowledge Configuration +# ------------------------------ + +# Upload file size limit, default 15M. +UPLOAD_FILE_SIZE_LIMIT=15 + +# The maximum number of files that can be uploaded at a time, default 5. +UPLOAD_FILE_BATCH_LIMIT=5 + +# ETL type, support: `dify`, `Unstructured` +# `dify` Dify's proprietary file extraction scheme +# `Unstructured` Unstructured.io file extraction scheme +ETL_TYPE=dify + +# Unstructured API path and API key, needs to be configured when ETL_TYPE is Unstructured +# Or using Unstructured for document extractor node for pptx. +# For example: http://unstructured:8000/general/v0/general +UNSTRUCTURED_API_URL= +UNSTRUCTURED_API_KEY= +SCARF_NO_ANALYTICS=true + +# ------------------------------ +# Model Configuration +# ------------------------------ + +# The maximum number of tokens allowed for prompt generation. +# This setting controls the upper limit of tokens that can be used by the LLM +# when generating a prompt in the prompt generation tool. +# Default: 512 tokens. +PROMPT_GENERATION_MAX_TOKENS=512 + +# The maximum number of tokens allowed for code generation. +# This setting controls the upper limit of tokens that can be used by the LLM +# when generating code in the code generation tool. +# Default: 1024 tokens. +CODE_GENERATION_MAX_TOKENS=1024 + +# ------------------------------ +# Multi-modal Configuration +# ------------------------------ + +# The format of the image/video/audio/document sent when the multi-modal model is input, +# the default is base64, optional url. +# The delay of the call in url mode will be lower than that in base64 mode. +# It is generally recommended to use the more compatible base64 mode. +# If configured as url, you need to configure FILES_URL as an externally accessible address so that the multi-modal model can access the image/video/audio/document. +MULTIMODAL_SEND_FORMAT=base64 +# Upload image file size limit, default 10M. +UPLOAD_IMAGE_FILE_SIZE_LIMIT=10 +# Upload video file size limit, default 100M. +UPLOAD_VIDEO_FILE_SIZE_LIMIT=100 +# Upload audio file size limit, default 50M. +UPLOAD_AUDIO_FILE_SIZE_LIMIT=50 + +# ------------------------------ +# Sentry Configuration +# Used for application monitoring and error log tracking. +# ------------------------------ +SENTRY_DSN= + +# API Service Sentry DSN address, default is empty, when empty, +# all monitoring information is not reported to Sentry. +# If not set, Sentry error reporting will be disabled. +API_SENTRY_DSN= +# API Service The reporting ratio of Sentry events, if it is 0.01, it is 1%. +API_SENTRY_TRACES_SAMPLE_RATE=1.0 +# API Service The reporting ratio of Sentry profiles, if it is 0.01, it is 1%. +API_SENTRY_PROFILES_SAMPLE_RATE=1.0 + +# Web Service Sentry DSN address, default is empty, when empty, +# all monitoring information is not reported to Sentry. +# If not set, Sentry error reporting will be disabled. +WEB_SENTRY_DSN= + +# ------------------------------ +# Notion Integration Configuration +# Variables can be obtained by applying for Notion integration: https://www.notion.so/my-integrations +# ------------------------------ + +# Configure as "public" or "internal". +# Since Notion's OAuth redirect URL only supports HTTPS, +# if deploying locally, please use Notion's internal integration. +NOTION_INTEGRATION_TYPE=public +# Notion OAuth client secret (used for public integration type) +NOTION_CLIENT_SECRET= +# Notion OAuth client id (used for public integration type) +NOTION_CLIENT_ID= +# Notion internal integration secret. +# If the value of NOTION_INTEGRATION_TYPE is "internal", +# you need to configure this variable. +NOTION_INTERNAL_SECRET= + +# ------------------------------ +# Mail related configuration +# ------------------------------ + +# Mail type, support: resend, smtp +MAIL_TYPE=resend + +# Default send from email address, if not specified +MAIL_DEFAULT_SEND_FROM= + +# API-Key for the Resend email provider, used when MAIL_TYPE is `resend`. +RESEND_API_URL=https://api.resend.com +RESEND_API_KEY=your-resend-api-key + + +# SMTP server configuration, used when MAIL_TYPE is `smtp` +SMTP_SERVER= +SMTP_PORT=465 +SMTP_USERNAME= +SMTP_PASSWORD= +SMTP_USE_TLS=true +SMTP_OPPORTUNISTIC_TLS=false + +# ------------------------------ +# Others Configuration +# ------------------------------ + +# Maximum length of segmentation tokens for indexing +INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 + +# Member invitation link valid time (hours), +# Default: 72. +INVITE_EXPIRY_HOURS=72 + +# Reset password token valid time (minutes), +RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5 + +# The sandbox service endpoint. +CODE_EXECUTION_ENDPOINT=http://sandbox:8194 +CODE_EXECUTION_API_KEY=dify-sandbox +CODE_MAX_NUMBER=9223372036854775807 +CODE_MIN_NUMBER=-9223372036854775808 +CODE_MAX_DEPTH=5 +CODE_MAX_PRECISION=20 +CODE_MAX_STRING_LENGTH=80000 +CODE_MAX_STRING_ARRAY_LENGTH=30 +CODE_MAX_OBJECT_ARRAY_LENGTH=30 +CODE_MAX_NUMBER_ARRAY_LENGTH=1000 +CODE_EXECUTION_CONNECT_TIMEOUT=10 +CODE_EXECUTION_READ_TIMEOUT=60 +CODE_EXECUTION_WRITE_TIMEOUT=10 +TEMPLATE_TRANSFORM_MAX_LENGTH=80000 + +# Workflow runtime configuration +WORKFLOW_MAX_EXECUTION_STEPS=500 +WORKFLOW_MAX_EXECUTION_TIME=1200 +WORKFLOW_CALL_MAX_DEPTH=5 +MAX_VARIABLE_SIZE=204800 +WORKFLOW_PARALLEL_DEPTH_LIMIT=3 +WORKFLOW_FILE_UPLOAD_LIMIT=10 + +# HTTP request node in workflow configuration +HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 +HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576 +HTTP_REQUEST_NODE_SSL_VERIFY=True + +# SSRF Proxy server HTTP URL +SSRF_PROXY_HTTP_URL=http://ssrf_proxy:3128 +# SSRF Proxy server HTTPS URL +SSRF_PROXY_HTTPS_URL=http://ssrf_proxy:3128 + +# Maximum loop count in the workflow +LOOP_NODE_MAX_COUNT=100 + +# The maximum number of tools that can be used in the agent. +MAX_TOOLS_NUM=10 + +# Maximum number of Parallelism branches in the workflow +MAX_PARALLEL_LIMIT=10 + +# The maximum number of iterations for agent setting +MAX_ITERATIONS_NUM=5 + +# ------------------------------ +# Environment Variables for web Service +# ------------------------------ + +# The timeout for the text generation in millisecond +TEXT_GENERATION_TIMEOUT_MS=60000 + +# ------------------------------ +# Environment Variables for db Service +# ------------------------------ + +PGUSER=${DB_USERNAME} +# The password for the default postgres user. +POSTGRES_PASSWORD=${DB_PASSWORD} +# The name of the default postgres database. +POSTGRES_DB=${DB_DATABASE} +# postgres data directory +PGDATA=/var/lib/postgresql/data/pgdata + +# ------------------------------ +# Environment Variables for sandbox Service +# ------------------------------ + +# The API key for the sandbox service +SANDBOX_API_KEY=dify-sandbox +# The mode in which the Gin framework runs +SANDBOX_GIN_MODE=release +# The timeout for the worker in seconds +SANDBOX_WORKER_TIMEOUT=15 +# Enable network for the sandbox service +SANDBOX_ENABLE_NETWORK=true +# HTTP proxy URL for SSRF protection +SANDBOX_HTTP_PROXY=http://ssrf_proxy:3128 +# HTTPS proxy URL for SSRF protection +SANDBOX_HTTPS_PROXY=http://ssrf_proxy:3128 +# The port on which the sandbox service runs +SANDBOX_PORT=8194 + +# ------------------------------ +# Environment Variables for weaviate Service +# (only used when VECTOR_STORE is weaviate) +# ------------------------------ +WEAVIATE_PERSISTENCE_DATA_PATH=/var/lib/weaviate \ No newline at end of file diff --git a/app/dify/README.md b/app/dify/README.md new file mode 100644 index 0000000..f1ddd11 --- /dev/null +++ b/app/dify/README.md @@ -0,0 +1,122 @@ +# Dify + +Dify: https://dify.ai/ + +The Innovation Engine for GenAI Applications, Dify is an open-source LLM app development platform. Orchestrate LLM apps from agents to complex AI workflows, with an RAG engine. + +- [Self-Hosting Dify](https://pigsty.io/docs/app/dify) +- [GitHub: langgenius/Dify](https://github.com/langgenius/dify/) +- [Pigsty: Dify Docker Compose Template](https://github.com/pgsty/pigsty/tree/master/app/dify) + + +```bash +curl -fsSL https://repo.pigsty.io/get | bash; cd ~/pigsty +cd ~/pigsty +./bootstrap # prepare local repo & ansible +./configure -c app/dify # IMPORTANT: CHANGE CREDENTIALS!! +./deploy.yml # install pigsty & pgsql & minio +./redis.yml # install extra redis instances +./docker.yml # install docker & docker-compose +./app.yml # install dify with docker compose +``` + +------ + +## Get Started + +Define & Create required PostgreSQL and Docker resources with Pigsty: + +```yaml +all: + children: + + # the dify application (default username & password: admin/admin) + dify: + hosts: { 10.10.10.10: {} } + vars: + app: dify # specify app name to be installed (in the apps) + apps: # define all applications + dify: # app name, should have corresponding ~/app/dify folder + conf: # override /opt/dify/.env config file + # A secret key for signing and encryption, gen with `openssl rand -base64 42` (CHANGE PASSWORD!) + SECRET_KEY: your-secret-key-placeholder + DB_USERNAME: dify + DB_PASSWORD: your-db-password + DB_HOST: 10.10.10.10 + DB_PORT: 5432 + DB_DATABASE: dify + VECTOR_STORE: pgvector + PGVECTOR_HOST: 10.10.10.10 + PGVECTOR_PORT: 5432 + PGVECTOR_USER: dify + PGVECTOR_PASSWORD: your-pgvector-password + PGVECTOR_DATABASE: dify + PGVECTOR_MIN_CONNECTION: 2 + PGVECTOR_MAX_CONNECTION: 10 + NGINX_SERVER_NAME: localhost + DIFY_PORT: 5001 # expose DIFY nginx service with port 5001 by default + #STORAGE_TYPE: s3 + #S3_ENDPOINT: 'https://sss.pigsty' + #S3_BUCKET_NAME: 'dify' + #S3_ACCESS_KEY: 'dify' + #S3_SECRET_KEY: 'S3User.Dify' + #S3_REGION: 'us-east-1' + + pg-meta: + hosts: { 10.10.10.10: { pg_seq: 1, pg_role: primary } } + vars: + pg_cluster: pg-meta + pg_users: + - { name: dify ,password: your-pg-password ,pgbouncer: true ,roles: [ dbrole_admin ] ,superuser: true ,comment: dify superuser } + pg_databases: + - { name: dify ,owner: dify ,revokeconn: true ,comment: dify main database } + pg_hba_rules: + - { user: dify ,db: all ,addr: 172.17.0.0/16 ,auth: pwd ,title: 'allow dify access from local docker network' } + - { user: dbuser_view , db: all ,addr: infra ,auth: pwd ,title: 'allow grafana dashboard access cmdb from infra nodes' } + + infra: { hosts: { 10.10.10.10: { infra_seq: 1 } } } + etcd: { hosts: { 10.10.10.10: { etcd_seq: 1 } }, vars: { etcd_cluster: etcd } } + minio: { hosts: { 10.10.10.10: { minio_seq: 1 } }, vars: { minio_cluster: minio } } +``` + + +------ + +## Expose Dify Web Service + +Change `infra_portal` in `pigsty.yml`, with the new `dify` line: + +```yaml +infra_portal: # infra services exposed via portal + home : { domain: i.pigsty } # default domain name + + dify : { domain: dify.pigsty ,endpoint: "10.10.10.10:8001", websocket: true } +``` + +Then expose dify web service via Pigsty's Nginx server: + +```bash +./infra.yml -t nginx +``` + +Don't forget to add `dify.pigsty` to your DNS or local `/etc/hosts` / `C:\Windows\System32\drivers\etc\hosts` to access via domain name. + +If you are using a public domain, consider using [Certbot](https://pigsty.io/docs/infra/admin/cert) to get a free SSL certificate. + +```bash +certbot --nginx --agree-tos --email your@email.com -n -d dify.your.domain # replace with your email & dify domain +``` + +Then add `certbot` field to the `dify` entry: + +```yaml +infra_portal: + #... + dify : { domain: dify.pigsty.cc ,endpoint: "10.10.10.10:8001", websocket: true , certbot: 'dify.pigsty.cc' } +``` + +To take over nginx config back to pigsty: + +```bash +./infra.yml -t nginx_config # regenerate nginx config align with certbot modification +``` \ No newline at end of file diff --git a/app/electric/.env b/app/electric/.env new file mode 100644 index 0000000..6e94593 --- /dev/null +++ b/app/electric/.env @@ -0,0 +1,74 @@ +# https://electric-sql.com/docs/api/config + +# A user with REPLICATION privileges is required +DATABASE_URL: 'postgresql://replicator:DBUser.Replicator@10.10.10.10:5432/meta?sslmode=require' + +# Port that the HTTP API is exposed on. +ELECTRIC_PORT: 8002 + +# Postgres connection string. Used to connect to the Postgres database for anything but the replication, will default to the same as DATABASE_URL if not provided. +#ELECTRIC_QUERY_DATABASE_URL + +# How many connections Electric opens as a pool for handling shape queries. 20 by default +#ELECTRIC_DB_POOL_SIZE: 20 + +# Suffix for the logical replication publication and slot name. +#ELECTRIC_REPLICATION_STREAM_ID: default + +# When set to true, runs Electric in insecure mode and does not require an ELECTRIC_SECRET. Use with caution. +# API requests are unprotected and may risk exposing your database. Good for development environments. +ELECTRIC_INSECURE: true + +# Secret for shape requests to the HTTP API. This is required unless ELECTRIC_INSECURE is set to true. +# By default, the Electric API is public and authorises all shape requests against this secret. +#ELECTRIC_SECRET: your_electric_secret_here + +# A unique identifier for the Electric instance. Defaults to a randomly generated UUID. +#ELECTRIC_INSTANCE_ID: Electric.Utils.uuid4() + +#Name of the electric service. Used as a resource identifier and namespace. +#ELECTRIC_SERVICE_NAME: electric + +#Expose some unsafe operations that faciliate integration testing. Do not enable this in production. +#ELECTRIC_ENABLE_INTEGRATION_TESTING: false + +#ELECTRIC_LISTEN_ON_IPV6: false + +# Limit the maximum size of a shape log response, to ensure they are cached by upstream caches. Defaults to 10MB (10 * 1024 * 1024). +#ELECTRIC_SHAPE_CHUNK_BYTES_THRESHOLD: 10485760 + +# Where to store shape metadata. Defaults to storing on the filesystem. If provided must be one of MEMORY or FILE. +# ELECTRIC_PERSISTENT_STATE: FILE + +# Where to store shape logs. Defaults to storing on the filesystem. If provided must be one of MEMORY or FILE. +#ELECTRIC_STORAGE: ./persistent + +# Path to root folder for storing data on the filesystem. +#ELECTRIC_STORAGE_DIR: ./persistent + +# Set an OpenTelemetry endpoint URL to enable telemetry. +#ELECTRIC_OTLP_ENDPOINT + +# Debug tracing by printing spans to stdout, without batching. +#ELECTRIC_OTEL_DEBUG: false + +# Honeycomb.io api key. Specify along with HNY_DATASET to export traces directly to Honeycomb, without the need to run an OpenTelemetry Collector. +#ELECTRIC_HNY_API_KEY + +# Name of your Honeycomb Dataset. +#ELECTRIC_HNY_DATASET + +# Expose a prometheus reporter for telemetry data on the specified port. +ELECTRIC_PROMETHEUS_PORT: 8003 + +# Verbosity of Electric's log output. Available levels, in the order of increasing verbosity: debug info warning error +# ELECTRIC_LOG_LEVEL: info + +# Enable or disable ANSI coloring of Electric's log output. +#ELECTRIC_LOG_COLORS: false + +# Enable OTP SASL reporting at runtime. +# ELECTRIC_LOG_OTP_REPORTS false + +# Configure anonymous usage data about the instance being sent to a central checkpoint service. Collected information is anonymised and doesn't contain any information from the replicated data. +ELECTRIC_USAGE_REPORTING: false diff --git a/app/supabase/.env b/app/supabase/.env new file mode 100644 index 0000000..cfd7932 --- /dev/null +++ b/app/supabase/.env @@ -0,0 +1,140 @@ +#==============================================================# +# File : .env +# Desc : supabase docker configuration entries +# Ctime : 2023-09-19 +# Mtime : 2025-07-01 +# Path : app/supabase/.env +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +#==============================================================# +# https://github.com/supabase/supabase/blob/master/docker/.env.example + +############ +# Secrets +# YOU MUST CHANGE THESE BEFORE GOING INTO PRODUCTION +############ +# IMPORTANT: https://supabase.com/docs/guides/self-hosting/docker#securing-your-services + +POSTGRES_PASSWORD=DBUser.Supa # supabase dbsu password (shared by multiple supabase biz users) +JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long +ANON_KEY=your-anon-key-here +SERVICE_ROLE_KEY=your-service-role-key-here +SECRET_KEY_BASE=your-secret-key-base +PG_META_CRYPTO_KEY=your-encryption-key-32-chars-min + +DASHBOARD_USERNAME=supabase # change to your own username +DASHBOARD_PASSWORD=pigsty # change to your own password + + +############ +# Database - You can change these to any PostgreSQL database that has logical replication enabled. +############ +POSTGRES_HOST=10.10.10.10 # change to Pigsty managed PostgreSQL cluster/instance VIP/IP +POSTGRES_PORT=5432 # you can use other service port such as 5433, 5436, 6432, etc... +POSTGRES_DB=postgres # change to supabase database name, `supa` by default in pigsty +POSTGRES_DOMAIN=pg-meta # in case you want to use domain name in database URL + +############ +# Domain +# YOU MUST CHANGE THESE WHEN ACCESS VIA DOMAIN NAME +############ +# replace if you intend to use Studio outside of localhost +SUPABASE_PUBLIC_URL=http://supa.pigsty +API_EXTERNAL_URL=http://supa.pigsty +SITE_URL=http://supa.pigsty + + +############ +# API Proxy - Configuration for the Kong Reverse proxy. +############ +KONG_HTTP_PORT=8000 +KONG_HTTPS_PORT=8443 + + +############ +# API - Configuration for PostgREST. +############ +PGRST_DB_SCHEMAS=public,storage,graphql_public + + +############ +# Auth - Configuration for the GoTrue authentication server. +############ + +## General +ADDITIONAL_REDIRECT_URLS= +JWT_EXPIRY=3600 +DISABLE_SIGNUP=false + +## Mailer Config +MAILER_URLPATHS_CONFIRMATION="/auth/v1/verify" +MAILER_URLPATHS_INVITE="/auth/v1/verify" +MAILER_URLPATHS_RECOVERY="/auth/v1/verify" +MAILER_URLPATHS_EMAIL_CHANGE="/auth/v1/verify" + +## Email auth +ENABLE_EMAIL_SIGNUP=true +ENABLE_EMAIL_AUTOCONFIRM=true + +SMTP_ADMIN_EMAIL=admin@example.com +SMTP_HOST=supabase-mail +SMTP_PORT=2500 +SMTP_USER=fake_mail_user +SMTP_PASS=fake_mail_password +SMTP_SENDER_NAME=fake_sender +ENABLE_ANONYMOUS_USERS=false + +## Phone auth +ENABLE_PHONE_SIGNUP=true +ENABLE_PHONE_AUTOCONFIRM=true + + +############ +# Studio - Configuration for the Dashboard +############ + +STUDIO_PORT=3000 +STUDIO_DEFAULT_PROJECT=Pigsty +STUDIO_DEFAULT_ORGANIZATION=Pigsty + +# Enable webp support +IMGPROXY_ENABLE_WEBP_DETECTION=true + +# Add your OpenAI API key to enable SQL Editor Assistant +OPENAI_API_KEY= + +############ +# Storage - Use external s3 or minio +############ +S3_BUCKET=supa +S3_ENDPOINT=https://sss.pigsty:9000 +S3_ACCESS_KEY=supabase +S3_SECRET_KEY=S3User.Supabase +S3_FORCE_PATH_STYLE=true +S3_PROTOCOL=https +S3_REGION=stub +MINIO_DOMAIN_IP=10.10.10.10 + +############ +# Realtime - Configuration for Realtime +############ +SECRET_KEY_BASE=your-secret-key-base + +############ +# Functions - Configuration for Functions +############ +# NOTE: VERIFY_JWT applies to all functions. Per-function VERIFY_JWT is not supported yet. +FUNCTIONS_VERIFY_JWT=false + +############ +# Logs - Configuration for Analytics +# Please refer to https://supabase.com/docs/reference/self-hosting-analytics/introduction +############ + +# use 32~64 character long random string for each key, the keys must be different +LOGFLARE_PUBLIC_ACCESS_TOKEN=your-logflare-public-token-here +LOGFLARE_PRIVATE_ACCESS_TOKEN=your-logflare-private-token-here +LOGFLARE_LOG_LEVEL=error + +# Docker socket location - this value will differ depending on your OS +DOCKER_SOCKET_LOCATION=/var/run/docker.sock diff --git a/app/supabase/client.html b/app/supabase/client.html new file mode 100644 index 0000000..4d386a9 --- /dev/null +++ b/app/supabase/client.html @@ -0,0 +1,43 @@ + + + + + Supabase Demo + + + +

Supabase: the Hello World Demo

+

Javascript Snippet Demo

+ +

const supabaseUrl = 'http://supa.pigsty';

+

const supabaseKey = 'your-anon-key-here';

+

const client = supabase.createClient(supabaseUrl, supabaseKey);

+
+

Create sample table in supabase console

+

create table countries (id int8 primary key, name text); insert into countries (id, name) values (1, 'USA'); -- do this in supabase console


+ +

Async Fetch Results

+
+ + + diff --git a/conf/app/electric.yml b/conf/app/electric.yml new file mode 100644 index 0000000..94c6b1f --- /dev/null +++ b/conf/app/electric.yml @@ -0,0 +1,115 @@ +--- +#==============================================================# +# File : electric.yml +# Desc : pigsty config for running 1-node electric app +# Ctime : 2025-03-29 +# Mtime : 2025-12-12 +# Docs : https://pigsty.io/docs/app/odoo +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +#==============================================================# + +# tutorial: https://pigsty.io/docs/app/electric +# quick start: https://electric-sql.com/docs/quickstart +# how to use this template: +# +# curl -fsSL https://repo.pigsty.io/get | bash; cd ~/pigsty +# ./bootstrap # prepare local repo & ansible +# ./configure -c app/electric # use this dify config template +# vi pigsty.yml # IMPORTANT: CHANGE CREDENTIALS!! +# ./deploy.yml # install pigsty & pgsql & minio +# ./docker.yml # install docker & docker-compose +# ./app.yml # install dify with docker-compose + +all: + children: + # infra cluster for proxy, monitor, alert, etc.. + infra: + hosts: { 10.10.10.10: { infra_seq: 1 } } + vars: + + app: electric + apps: # define all applications + electric: # app name, should have corresponding ~/pigsty/app/electric folder + conf: # override /opt/electric/.env config file : https://electric-sql.com/docs/api/config + DATABASE_URL: 'postgresql://electric:DBUser.Electric@10.10.10.10:5432/electric?sslmode=require' + ELECTRIC_PORT: 8002 + ELECTRIC_PROMETHEUS_PORT: 8003 + ELECTRIC_INSECURE: true + #ELECTRIC_SECRET: your_electric_secret_here + + # etcd cluster for ha postgres + etcd: { hosts: { 10.10.10.10: { etcd_seq: 1 } }, vars: { etcd_cluster: etcd } } + + # minio cluster, s3 compatible object storage + #minio: { hosts: { 10.10.10.10: { minio_seq: 1 } }, vars: { minio_cluster: minio } } + + # postgres example cluster: pg-meta + pg-meta: + hosts: { 10.10.10.10: { pg_seq: 1, pg_role: primary } } + vars: + pg_cluster: pg-meta + pg_users: + - {name: electric ,password: DBUser.Electric ,pgbouncer: true , replication: true ,roles: [dbrole_admin] ,comment: electric main user } + pg_databases: [{ name: electric , owner: electric }] + pg_hba_rules: + - { user: electric , db: replication ,addr: infra ,auth: ssl ,title: 'allow electric intranet/docker ssl access' } + + #==============================================================# + # Global Parameters + #==============================================================# + vars: + + #----------------------------------# + # Meta Data + #----------------------------------# + version: v4.0.0 # pigsty version string + admin_ip: 10.10.10.10 # admin node ip address + region: default # upstream mirror region: default|china|europe + node_tune: oltp # node tuning specs: oltp,olap,tiny,crit + pg_conf: oltp.yml # pgsql tuning specs: {oltp,olap,tiny,crit}.yml + + docker_enabled: true # enable docker on app group + #docker_registry_mirrors: ["https://docker.1panel.live","https://docker.1ms.run","https://docker.xuanyuan.me","https://registry-1.docker.io"] + + proxy_env: # global proxy env when downloading packages + no_proxy: "localhost,127.0.0.1,10.0.0.0/8,192.168.0.0/16,*.pigsty,*.aliyun.com,mirrors.*,*.myqcloud.com,*.tsinghua.edu.cn" + # http_proxy: # set your proxy here: e.g http://user:pass@proxy.xxx.com + # https_proxy: # set your proxy here: e.g http://user:pass@proxy.xxx.com + # all_proxy: # set your proxy here: e.g http://user:pass@proxy.xxx.com + infra_portal: # domain names and upstream servers + home : { domain: i.pigsty } + electric: + domain: elec.pigsty + endpoint: "${admin_ip}:8002" + websocket: true # apply free ssl cert with certbot: make cert + certbot: odoo.pigsty # <----- replace with your own domain name! + + #----------------------------------# + # Safe Guard + #----------------------------------# + # you can enable these flags after bootstrap, to prevent purging running etcd / pgsql instances + etcd_safeguard: false # prevent purging running etcd instance? + pg_safeguard: false # prevent purging running postgres instance? false by default + + #----------------------------------# + # Repo, Node, Packages + #----------------------------------# + repo_enabled: false + node_repo_modules: node,infra,pgsql + pg_version: 18 # default postgres version + #pg_extensions: [ pg18-time ,pg18-gis ,pg18-rag ,pg18-fts ,pg18-olap ,pg18-feat ,pg18-lang ,pg18-type ,pg18-util ,pg18-func ,pg18-admin ,pg18-stat ,pg18-sec ,pg18-fdw ,pg18-sim ,pg18-etl] + + #----------------------------------------------# + # PASSWORD : https://pigsty.io/docs/setup/security/ + #----------------------------------------------# + grafana_admin_password: pigsty + grafana_view_password: DBUser.Viewer + pg_admin_password: DBUser.DBA + pg_monitor_password: DBUser.Monitor + pg_replication_password: DBUser.Replicator + patroni_password: Patroni.API + haproxy_admin_password: pigsty + minio_secret_key: S3User.MinIO + etcd_root_password: Etcd.Root +... diff --git a/conf/app/supa.yml b/conf/app/supa.yml new file mode 100644 index 0000000..461023e --- /dev/null +++ b/conf/app/supa.yml @@ -0,0 +1,305 @@ +--- +#==============================================================# +# File : supabase.yml +# Desc : Pigsty configuration for self-hosting supabase +# Ctime : 2023-09-19 +# Mtime : 2026-01-20 +# Docs : https://pigsty.io/docs/conf/supabase +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +#==============================================================# + +# supabase is available on el8/el9/u22/u24/d12 with pg15,16,17,18 +# tutorial: https://pigsty.io/docs/app/supabase +# Usage: +# curl https://repo.pigsty.io/get | bash # install pigsty +# ./configure -c supabase # use this supabase conf template +# ./deploy.yml # install pigsty & pgsql & minio +# ./docker.yml # install docker & docker compose +# ./app.yml # launch supabase with docker compose + +all: + children: + + + #----------------------------------------------# + # INFRA : https://pigsty.io/docs/infra + #----------------------------------------------# + infra: + hosts: + 10.10.10.10: { infra_seq: 1 } + vars: + repo_enabled: false # disable local repo + + #----------------------------------------------# + # ETCD : https://pigsty.io/docs/etcd + #----------------------------------------------# + etcd: + hosts: + 10.10.10.10: { etcd_seq: 1 } + vars: + etcd_cluster: etcd + etcd_safeguard: false # enable to prevent purging running etcd instance + + #----------------------------------------------# + # MINIO : https://pigsty.io/docs/minio + #----------------------------------------------# + minio: + hosts: + 10.10.10.10: { minio_seq: 1 } + vars: + minio_cluster: minio + minio_users: # list of minio user to be created + - { access_key: pgbackrest ,secret_key: S3User.Backup ,policy: pgsql } + - { access_key: s3user_meta ,secret_key: S3User.Meta ,policy: meta } + - { access_key: s3user_data ,secret_key: S3User.Data ,policy: data } + + #----------------------------------------------# + # PostgreSQL cluster for Supabase self-hosting + #----------------------------------------------# + pg-meta: + hosts: + 10.10.10.10: { pg_seq: 1, pg_role: primary } + vars: + pg_cluster: pg-meta + pg_users: + # supabase roles: anon, authenticated, dashboard_user + - { name: anon ,login: false } + - { name: authenticated ,login: false } + - { name: dashboard_user ,login: false ,replication: true ,createdb: true ,createrole: true } + - { name: service_role ,login: false ,bypassrls: true } + # supabase users: please use the same password + - { name: supabase_admin ,password: 'DBUser.Supa' ,pgbouncer: true ,inherit: true ,roles: [ dbrole_admin ] ,superuser: true ,replication: true ,createdb: true ,createrole: true ,bypassrls: true } + - { name: authenticator ,password: 'DBUser.Supa' ,pgbouncer: true ,inherit: false ,roles: [ dbrole_admin, authenticated ,anon ,service_role ] } + - { name: supabase_auth_admin ,password: 'DBUser.Supa' ,pgbouncer: true ,inherit: false ,roles: [ dbrole_admin ] ,createrole: true } + - { name: supabase_storage_admin ,password: 'DBUser.Supa' ,pgbouncer: true ,inherit: false ,roles: [ dbrole_admin, authenticated ,anon ,service_role ] ,createrole: true } + - { name: supabase_functions_admin ,password: 'DBUser.Supa' ,pgbouncer: true ,inherit: false ,roles: [ dbrole_admin ] ,createrole: true } + - { name: supabase_replication_admin ,password: 'DBUser.Supa' ,replication: true ,roles: [ dbrole_admin ]} + - { name: supabase_etl_admin ,password: 'DBUser.Supa' ,replication: true ,roles: [ pg_read_all_data, dbrole_readonly ]} + - { name: supabase_read_only_user ,password: 'DBUser.Supa' ,bypassrls: true ,roles: [ pg_read_all_data, dbrole_readonly ]} + pg_databases: + - name: postgres + baseline: supabase.sql + owner: supabase_admin + comment: supabase postgres database + schemas: [ extensions ,auth ,realtime ,storage ,graphql_public ,supabase_functions ,_analytics ,_realtime ] + extensions: + - { name: pgcrypto ,schema: extensions } # cryptographic functions + - { name: pg_net ,schema: extensions } # async HTTP + - { name: pgjwt ,schema: extensions } # json web token API for postgres + - { name: uuid-ossp ,schema: extensions } # generate universally unique identifiers (UUIDs) + - { name: pgsodium ,schema: extensions } # pgsodium is a modern cryptography library for Postgres. + - { name: supabase_vault ,schema: extensions } # Supabase Vault Extension + - { name: pg_graphql ,schema: extensions } # pg_graphql: GraphQL support + - { name: pg_jsonschema ,schema: extensions } # pg_jsonschema: Validate json schema + - { name: wrappers ,schema: extensions } # wrappers: FDW collections + - { name: http ,schema: extensions } # http: allows web page retrieval inside the database. + - { name: pg_cron ,schema: extensions } # pg_cron: Job scheduler for PostgreSQL + - { name: timescaledb ,schema: extensions } # timescaledb: Enables scalable inserts and complex queries for time-series data + - { name: pg_tle ,schema: extensions } # pg_tle: Trusted Language Extensions for PostgreSQL + - { name: vector ,schema: extensions } # pgvector: the vector similarity search + - { name: pgmq ,schema: extensions } # pgmq: A lightweight message queue like AWS SQS and RSMQ + - { name: supabase ,owner: supabase_admin ,comment: supabase analytics database ,schemas: [ extensions, _analytics ] } + + # supabase required extensions + pg_libs: 'timescaledb, pgsodium, plpgsql, plpgsql_check, pg_cron, pg_net, pg_stat_statements, auto_explain, pg_wait_sampling, pg_tle, plan_filter' + pg_extensions: [ pg18-main ,pg18-time ,pg18-gis ,pg18-rag ,pg18-fts ,pg18-olap ,pg18-feat ,pg18-lang ,pg18-type ,pg18-util ,pg18-func ,pg18-admin ,pg18-stat ,pg18-sec ,pg18-fdw ,pg18-sim ,pg18-etl] + pg_parameters: { cron.database_name: postgres } + pg_hba_rules: # supabase hba rules, require access from docker network + - { user: all ,db: postgres ,addr: intra ,auth: pwd ,title: 'allow supabase access from intranet' ,order: 50 } + - { user: all ,db: postgres ,addr: 172.17.0.0/16 ,auth: pwd ,title: 'allow access from local docker network' ,order: 50 } + pg_crontab: + - '00 01 * * * /pg/bin/pg-backup full' # make a full backup every 1am + - '* * * * * /pg/bin/supa-kick' # kick supabase _analytics lag per minute: https://github.com/pgsty/pigsty/issues/581 + + #----------------------------------------------# + # Supabase + #----------------------------------------------# + # ./docker.yml + # ./app.yml + + # the supabase stateless containers (default username & password: supabase/pigsty) + supabase: + hosts: + 10.10.10.10: {} + vars: + docker_enabled: true # enable docker on this group + #docker_registry_mirrors: ["https://docker.1panel.live","https://docker.1ms.run","https://docker.xuanyuan.me","https://registry-1.docker.io"] + app: supabase # specify app name (supa) to be installed (in the apps) + apps: # define all applications + supabase: # the definition of supabase app + conf: # override /opt/supabase/.env + + # IMPORTANT: CHANGE JWT_SECRET AND REGENERATE CREDENTIAL ACCORDING!!!!!!!!!!! + # https://supabase.com/docs/guides/self-hosting/docker#securing-your-services + JWT_SECRET: your-super-secret-jwt-token-with-at-least-32-characters-long + ANON_KEY: your-anon-key-here + SERVICE_ROLE_KEY: your-service-role-key-here + PG_META_CRYPTO_KEY: your-encryption-key-32-chars-min + + DASHBOARD_USERNAME: supabase + DASHBOARD_PASSWORD: pigsty + + # 32~64 random characters string for logflare + LOGFLARE_PUBLIC_ACCESS_TOKEN: your-logflare-public-token-here + LOGFLARE_PRIVATE_ACCESS_TOKEN: your-logflare-private-token-here + + # postgres connection string (use the correct ip and port) + POSTGRES_HOST: 10.10.10.10 # point to the local postgres node + POSTGRES_PORT: 5436 # access via the 'default' service, which always route to the primary postgres + POSTGRES_DB: postgres # the supabase underlying database + POSTGRES_PASSWORD: DBUser.Supa # password for supabase_admin and multiple supabase users + + # expose supabase via domain name + SITE_URL: https://supa.pigsty # <------- Change This to your external domain name + API_EXTERNAL_URL: https://supa.pigsty # <------- Otherwise the storage api may not work! + SUPABASE_PUBLIC_URL: https://supa.pigsty # <------- DO NOT FORGET TO PUT IT IN infra_portal! + + # if using s3/minio as file storage + S3_BUCKET: data + S3_ENDPOINT: https://sss.pigsty:9000 + S3_ACCESS_KEY: s3user_data + S3_SECRET_KEY: S3User.Data + S3_FORCE_PATH_STYLE: true + S3_PROTOCOL: https + S3_REGION: stub + MINIO_DOMAIN_IP: 10.10.10.10 # sss.pigsty domain name will resolve to this ip statically + + # if using SMTP (optional) + #SMTP_ADMIN_EMAIL: admin@example.com + #SMTP_HOST: supabase-mail + #SMTP_PORT: 2500 + #SMTP_USER: fake_mail_user + #SMTP_PASS: fake_mail_password + #SMTP_SENDER_NAME: fake_sender + #ENABLE_ANONYMOUS_USERS: false + + + #==============================================================# + # Global Parameters + #==============================================================# + vars: + + #----------------------------------------------# + # INFRA : https://pigsty.io/docs/infra + #----------------------------------------------# + version: v4.0.0 # pigsty version string + admin_ip: 10.10.10.10 # admin node ip address + region: default # upstream mirror region: default|china|europe + proxy_env: # global proxy env when downloading packages + no_proxy: "localhost,127.0.0.1,10.0.0.0/8,192.168.0.0/16,*.pigsty,*.aliyun.com,mirrors.*,*.myqcloud.com,*.tsinghua.edu.cn" + # http_proxy: # set your proxy here: e.g http://user:pass@proxy.xxx.com + # https_proxy: # set your proxy here: e.g http://user:pass@proxy.xxx.com + # all_proxy: # set your proxy here: e.g http://user:pass@proxy.xxx.com + certbot_sign: false # enable certbot to sign https certificate for infra portal + certbot_email: your@email.com # replace your email address to receive expiration notice + infra_portal: # infra services exposed via portal + home : { domain: i.pigsty } # default domain name + pgadmin : { domain: adm.pigsty ,endpoint: "${admin_ip}:8885" } + bytebase : { domain: ddl.pigsty ,endpoint: "${admin_ip}:8887" } + #minio : { domain: m.pigsty ,endpoint: "${admin_ip}:9001" ,scheme: https ,websocket: true } + + # Nginx / Domain / HTTPS : https://pigsty.io/docs/infra/admin/portal + supa : # nginx server config for supabase + domain: supa.pigsty # REPLACE IT WITH YOUR OWN DOMAIN! + endpoint: "10.10.10.10:8000" # supabase service endpoint: IP:PORT + websocket: true # add websocket support + certbot: supa.pigsty # certbot cert name, apply with `make cert` + + #----------------------------------------------# + # NODE : https://pigsty.io/docs/node/param + #----------------------------------------------# + nodename_overwrite: false # do not overwrite node hostname on single node mode + node_tune: oltp # node tuning specs: oltp,olap,tiny,crit + node_etc_hosts: # add static domains to all nodes /etc/hosts + - 10.10.10.10 i.pigsty sss.pigsty supa.pigsty + node_repo_modules: node,pgsql,infra # use pre-made local repo rather than install from upstream + node_repo_remove: true # remove existing node repo for node managed by pigsty + #node_packages: [openssh-server] # packages to be installed current nodes with latest version + #node_timezone: Asia/Hong_Kong # overwrite node timezone + + #----------------------------------------------# + # PGSQL : https://pigsty.io/docs/pgsql/param + #----------------------------------------------# + pg_version: 18 # default postgres version + pg_conf: oltp.yml # pgsql tuning specs: {oltp,olap,tiny,crit}.yml + pg_safeguard: false # prevent purging running postgres instance? + pg_default_schemas: [ monitor, extensions ] # add new schema: exxtensions + pg_default_extensions: # default extensions to be created + - { name: pg_stat_statements ,schema: monitor } + - { name: pgstattuple ,schema: monitor } + - { name: pg_buffercache ,schema: monitor } + - { name: pageinspect ,schema: monitor } + - { name: pg_prewarm ,schema: monitor } + - { name: pg_visibility ,schema: monitor } + - { name: pg_freespacemap ,schema: monitor } + - { name: pg_wait_sampling ,schema: monitor } + # move default extensions to `extensions` schema for supabase + - { name: postgres_fdw ,schema: extensions } + - { name: file_fdw ,schema: extensions } + - { name: btree_gist ,schema: extensions } + - { name: btree_gin ,schema: extensions } + - { name: pg_trgm ,schema: extensions } + - { name: intagg ,schema: extensions } + - { name: intarray ,schema: extensions } + - { name: pg_repack ,schema: extensions } + + #----------------------------------------------# + # BACKUP : https://pigsty.io/docs/pgsql/backup + #----------------------------------------------# + minio_endpoint: https://sss.pigsty:9000 # explicit overwrite minio endpoint with haproxy port + pgbackrest_method: minio # pgbackrest repo method: local,minio,[user-defined...] + pgbackrest_repo: # pgbackrest repo: https://pgbackrest.org/configuration.html#section-repository + local: # default pgbackrest repo with local posix fs + path: /pg/backup # local backup directory, `/pg/backup` by default + retention_full_type: count # retention full backups by count + retention_full: 2 # keep 2, at most 3 full backups when using local fs repo + minio: # optional minio repo for pgbackrest + type: s3 # minio is s3-compatible, so s3 is used + s3_endpoint: sss.pigsty # minio endpoint domain name, `sss.pigsty` by default + s3_region: us-east-1 # minio region, us-east-1 by default, useless for minio + s3_bucket: pgsql # minio bucket name, `pgsql` by default + s3_key: pgbackrest # minio user access key for pgbackrest + s3_key_secret: S3User.Backup # minio user secret key for pgbackrest <------------------ HEY, DID YOU CHANGE THIS? + s3_uri_style: path # use path style uri for minio rather than host style + path: /pgbackrest # minio backup path, default is `/pgbackrest` + storage_port: 9000 # minio port, 9000 by default + storage_ca_file: /etc/pki/ca.crt # minio ca file path, `/etc/pki/ca.crt` by default + block: y # Enable block incremental backup + bundle: y # bundle small files into a single file + bundle_limit: 20MiB # Limit for file bundles, 20MiB for object storage + bundle_size: 128MiB # Target size for file bundles, 128MiB for object storage + cipher_type: aes-256-cbc # enable AES encryption for remote backup repo + cipher_pass: pgBackRest # AES encryption password, default is 'pgBackRest' <----- HEY, DID YOU CHANGE THIS? + retention_full_type: time # retention full backup by time on minio repo + retention_full: 14 # keep full backup for the last 14 days + s3: # you can use cloud object storage as backup repo + type: s3 # Add your object storage credentials here! + s3_endpoint: oss-cn-beijing-internal.aliyuncs.com + s3_region: oss-cn-beijing + s3_bucket: + s3_key: + s3_key_secret: + s3_uri_style: host + path: /pgbackrest + bundle: y # bundle small files into a single file + bundle_limit: 20MiB # Limit for file bundles, 20MiB for object storage + bundle_size: 128MiB # Target size for file bundles, 128MiB for object storage + cipher_type: aes-256-cbc # enable AES encryption for remote backup repo + cipher_pass: pgBackRest # AES encryption password, default is 'pgBackRest' + retention_full_type: time # retention full backup by time on minio repo + retention_full: 14 # keep full backup for the last 14 days + + #----------------------------------------------# + # PASSWORD : https://pigsty.io/docs/setup/security/ + #----------------------------------------------# + grafana_admin_password: pigsty + grafana_view_password: DBUser.Viewer + pg_admin_password: DBUser.DBA + pg_monitor_password: DBUser.Monitor + pg_replication_password: DBUser.Replicator + patroni_password: Patroni.API + haproxy_admin_password: pigsty + minio_secret_key: S3User.MinIO + etcd_root_password: Etcd.Root +...