diff --git a/appwrite/code/.env.example b/appwrite/code/.env.example index 18e247c..110cf4d 100644 --- a/appwrite/code/.env.example +++ b/appwrite/code/.env.example @@ -6,6 +6,7 @@ _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled _APP_OPTIONS_ROUTER_PROTECTION=disabled _APP_OPENSSL_KEY_V1=your-secret-key _APP_DOMAIN=localhost +_APP_CUSTOM_DOMAIN_DENY_LIST=example.com,test.com,app.example.com _APP_DOMAIN_FUNCTIONS=functions.localhost _APP_DOMAIN_TARGET=localhost _APP_CONSOLE_WHITELIST_ROOT=enabled @@ -27,6 +28,8 @@ _APP_USAGE_TIMESERIES_INTERVAL=30 _APP_USAGE_DATABASE_INTERVAL=900 _APP_WORKER_PER_CORE=6 _APP_CONSOLE_SESSION_ALERTS=disabled +_APP_COMPRESSION_ENABLED=enabled +_APP_COMPRESSION_MIN_SIZE_BYTES=1024 _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_USER= @@ -58,6 +61,7 @@ _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= _APP_STORAGE_S3_REGION=us-east-1 _APP_STORAGE_S3_BUCKET= +_APP_STORAGE_S3_ENDPOINT= _APP_STORAGE_DO_SPACES_ACCESS_KEY= _APP_STORAGE_DO_SPACES_SECRET= _APP_STORAGE_DO_SPACES_REGION=us-east-1 @@ -107,6 +111,7 @@ _APP_MAINTENANCE_DELAY=0 _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 +_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE=15778800 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 diff --git a/appwrite/code/docker-compose.yml b/appwrite/code/docker-compose.yml index cca2c8d..f3d746a 100644 --- a/appwrite/code/docker-compose.yml +++ b/appwrite/code/docker-compose.yml @@ -28,7 +28,7 @@ services: - appwrite appwrite: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 <<: *x-logging restart: unless-stopped networks: @@ -61,6 +61,7 @@ services: - _APP_ENV - _APP_WORKER_PER_CORE - _APP_LOCALE + - _APP_COMPRESSION_MIN_SIZE_BYTES - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS - _APP_CONSOLE_SESSION_ALERTS @@ -103,6 +104,7 @@ services: - _APP_STORAGE_S3_SECRET - _APP_STORAGE_S3_REGION - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_S3_ENDPOINT - _APP_STORAGE_DO_SPACES_ACCESS_KEY - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION @@ -134,6 +136,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - _APP_SMS_PROVIDER @@ -150,10 +153,9 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_ASSISTANT_OPENAI_API_KEY - appwrite-console: <<: *x-logging - image: appwrite/console:5.0.12 + image: appwrite/console:5.2.58 restart: unless-stopped networks: - appwrite @@ -173,7 +175,7 @@ services: - traefik.http.routers.appwrite_console_https.tls=true appwrite-realtime: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: realtime <<: *x-logging restart: unless-stopped @@ -215,7 +217,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-audits: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: worker-audits <<: *x-logging restart: unless-stopped @@ -240,7 +242,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-webhooks: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: worker-webhooks <<: *x-logging restart: unless-stopped @@ -267,7 +269,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-deletes: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: worker-deletes <<: *x-logging restart: unless-stopped @@ -300,6 +302,7 @@ services: - _APP_STORAGE_S3_SECRET - _APP_STORAGE_S3_REGION - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_S3_ENDPOINT - _APP_STORAGE_DO_SPACES_ACCESS_KEY - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION @@ -321,10 +324,13 @@ services: - _APP_EXECUTOR_HOST - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_EXECUTION + - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS + - _APP_EMAIL_CERTIFICATES appwrite-worker-databases: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: worker-databases <<: *x-logging restart: unless-stopped @@ -349,7 +355,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-builds: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: worker-builds <<: *x-logging restart: unless-stopped @@ -393,6 +399,7 @@ services: - _APP_STORAGE_S3_SECRET - _APP_STORAGE_S3_REGION - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_S3_ENDPOINT - _APP_STORAGE_DO_SPACES_ACCESS_KEY - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION @@ -411,7 +418,7 @@ services: - _APP_STORAGE_WASABI_BUCKET appwrite-worker-certificates: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: worker-certificates <<: *x-logging restart: unless-stopped @@ -443,7 +450,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-functions: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: worker-functions <<: *x-logging restart: unless-stopped @@ -480,7 +487,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-mails: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: worker-mails <<: *x-logging restart: unless-stopped @@ -509,9 +516,11 @@ services: - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD - _APP_LOGGING_CONFIG + - _APP_DOMAIN + - _APP_OPTIONS_FORCE_HTTPS appwrite-worker-messaging: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: worker-messaging <<: *x-logging restart: unless-stopped @@ -542,6 +551,7 @@ services: - _APP_STORAGE_S3_SECRET - _APP_STORAGE_S3_REGION - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_S3_ENDPOINT - _APP_STORAGE_DO_SPACES_ACCESS_KEY - _APP_STORAGE_DO_SPACES_SECRET - _APP_STORAGE_DO_SPACES_REGION @@ -560,7 +570,7 @@ services: - _APP_STORAGE_WASABI_BUCKET appwrite-worker-migrations: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: worker-migrations <<: *x-logging restart: unless-stopped @@ -589,7 +599,7 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET appwrite-task-maintenance: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: maintenance <<: *x-logging restart: unless-stopped @@ -618,12 +628,43 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - appwrite-worker-usage: - image: appwrite/appwrite:1.6.0 - entrypoint: worker-usage + appwrite-task-stats-resources: + image: appwrite/appwrite:1.6.2 + entrypoint: stats-resources + <<: *x-logging + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_CONFIG + - _APP_DATABASE_SHARED_TABLES + - _APP_STATS_RESOURCES_INTERVAL + + appwrite-worker-stats-resources: + image: appwrite/appwrite:1.6.2 + entrypoint: worker-stats-resources <<: *x-logging restart: unless-stopped networks: @@ -646,12 +687,13 @@ services: - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_STATS_RESOURCES_INTERVAL - appwrite-worker-usage-dump: - image: appwrite/appwrite:1.6.0 - entrypoint: worker-usage-dump + appwrite-worker-stats-usage: + image: appwrite/appwrite:1.6.2 + entrypoint: worker-stats-usage <<: *x-logging + restart: unless-stopped networks: - appwrite depends_on: @@ -675,7 +717,7 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL appwrite-task-scheduler-functions: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: schedule-functions <<: *x-logging restart: unless-stopped @@ -699,7 +741,7 @@ services: - _APP_DB_PASS appwrite-task-scheduler-executions: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: schedule-executions <<: *x-logging restart: unless-stopped @@ -723,7 +765,7 @@ services: - _APP_DB_PASS appwrite-task-scheduler-messages: - image: appwrite/appwrite:1.6.0 + image: appwrite/appwrite:1.6.2 entrypoint: schedule-messages <<: *x-logging restart: unless-stopped @@ -786,6 +828,7 @@ services: - OPR_EXECUTOR_STORAGE_S3_SECRET=$_APP_STORAGE_S3_SECRET - OPR_EXECUTOR_STORAGE_S3_REGION=$_APP_STORAGE_S3_REGION - OPR_EXECUTOR_STORAGE_S3_BUCKET=$_APP_STORAGE_S3_BUCKET + - OPR_EXECUTOR_STORAGE_S3_ENDPOINT=$_APP_STORAGE_S3_ENDPOINT - OPR_EXECUTOR_STORAGE_DO_SPACES_ACCESS_KEY=$_APP_STORAGE_DO_SPACES_ACCESS_KEY - OPR_EXECUTOR_STORAGE_DO_SPACES_SECRET=$_APP_STORAGE_DO_SPACES_SECRET - OPR_EXECUTOR_STORAGE_DO_SPACES_REGION=$_APP_STORAGE_DO_SPACES_REGION diff --git a/dify/code/.env.example b/dify/code/.env.example index afd2caf..aacef0e 100644 --- a/dify/code/.env.example +++ b/dify/code/.env.example @@ -39,6 +39,12 @@ APP_WEB_URL= # File preview or download Url prefix. # used to display File preview or download Url to the front-end or as Multi-model inputs; # Url is signed and has expiration time. +# Setting FILES_URL is required for file processing plugins. +# - For https://example.com, use FILES_URL=https://example.com +# - For http://example.com, use FILES_URL=http://example.com +# Recommendation: use a dedicated domain (e.g., https://upload.example.com). +# Alternatively, use http://:5001 or http://api:5001, +# ensuring port 5001 is externally accessible (see docker-compose.yaml). FILES_URL= # ------------------------------ @@ -68,15 +74,19 @@ DEBUG=false # which is convenient for debugging. FLASK_DEBUG=false -# A secretkey that is used for securely signing the session cookie +# Enable request logging, which will log the request and response information. +# And the log level is DEBUG +ENABLE_REQUEST_LOGGING=False + +# A secret key that is used for securely signing the session cookie # and encrypting sensitive information on the database. # You can generate a strong key using `openssl rand -base64 42`. SECRET_KEY=sk-9f73s3ljTXVcMT3Blb3ljTqtsKiGHXVcMT3BlbkFJLK7U # Password for admin user initialization. # If left unset, admin user will not be prompted for a password -# when creating the initial admin account. -# The length of the password cannot exceed 30 charactors. +# when creating the initial admin account. +# The length of the password cannot exceed 30 characters. INIT_PASSWORD= # Deployment environment. @@ -174,6 +184,12 @@ CELERY_MIN_WORKERS= API_TOOL_DEFAULT_CONNECT_TIMEOUT=10 API_TOOL_DEFAULT_READ_TIMEOUT=60 +# ------------------------------- +# Datasource Configuration +# -------------------------------- +ENABLE_WEBSITE_JINAREADER=true +ENABLE_WEBSITE_FIRECRAWL=true +ENABLE_WEBSITE_WATERCRAWL=true # ------------------------------ # Database Configuration @@ -344,7 +360,7 @@ TENCENT_COS_SCHEME=your-scheme # Oracle Storage Configuration # -OCI_ENDPOINT=https://objectstorage.us-ashburn-1.oraclecloud.com +OCI_ENDPOINT=https://your-object-storage-namespace.compat.objectstorage.us-ashburn-1.oraclecloud.com OCI_BUCKET_NAME=your-bucket-name OCI_ACCESS_KEY=your-access-key OCI_SECRET_KEY=your-secret-key @@ -383,7 +399,7 @@ SUPABASE_URL=your-server-url # ------------------------------ # The type of vector store to use. -# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `tidb_vector`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`. +# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`, `opengauss`, `tablestore`,`vastbase`,`tidb`,`tidb_on_qdrant`,`baidu`,`lindorm`,`huawei_cloud`,`upstash`. VECTOR_STORE=weaviate # The Weaviate endpoint URL. Only available when VECTOR_STORE is `weaviate`. @@ -397,13 +413,15 @@ QDRANT_CLIENT_TIMEOUT=20 QDRANT_GRPC_ENABLED=false QDRANT_GRPC_PORT=6334 -# Milvus configuration Only available when VECTOR_STORE is `milvus`. +# Milvus configuration. Only available when VECTOR_STORE is `milvus`. # The milvus uri. -MILVUS_URI=http://127.0.0.1:19530 +MILVUS_URI=http://host.docker.internal:19530 +MILVUS_DATABASE= MILVUS_TOKEN= -MILVUS_USER=root -MILVUS_PASSWORD=Milvus +MILVUS_USER= +MILVUS_PASSWORD= MILVUS_ENABLE_HYBRID_SEARCH=False +MILVUS_ANALYZER_PARAMS= # MyScale configuration, only available when VECTOR_STORE is `myscale` # For multi-language support, please set MYSCALE_FTS_PARAMS with referring to: @@ -431,6 +449,17 @@ PGVECTOR_PASSWORD=difyai123456 PGVECTOR_DATABASE=dify PGVECTOR_MIN_CONNECTION=1 PGVECTOR_MAX_CONNECTION=5 +PGVECTOR_PG_BIGM=false +PGVECTOR_PG_BIGM_VERSION=1.2-20240606 + +# vastbase configurations, only available when VECTOR_STORE is `vastbase` +VASTBASE_HOST=vastbase +VASTBASE_PORT=5432 +VASTBASE_USER=dify +VASTBASE_PASSWORD=Difyai123456 +VASTBASE_DATABASE=dify +VASTBASE_MIN_CONNECTION=1 +VASTBASE_MAX_CONNECTION=5 # pgvecto-rs configurations, only available when VECTOR_STORE is `pgvecto-rs` PGVECTO_RS_HOST=pgvecto-rs @@ -453,7 +482,7 @@ ANALYTICDB_PORT=5432 ANALYTICDB_MIN_CONNECTION=1 ANALYTICDB_MAX_CONNECTION=5 -# TiDB vector configurations, only available when VECTOR_STORE is `tidb` +# TiDB vector configurations, only available when VECTOR_STORE is `tidb_vector` TIDB_VECTOR_HOST=tidb TIDB_VECTOR_PORT=4000 TIDB_VECTOR_USER= @@ -483,11 +512,13 @@ CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthClientProvider CHROMA_AUTH_CREDENTIALS= # Oracle configuration, only available when VECTOR_STORE is `oracle` -ORACLE_HOST=oracle -ORACLE_PORT=1521 ORACLE_USER=dify ORACLE_PASSWORD=dify -ORACLE_DATABASE=FREEPDB1 +ORACLE_DSN=oracle:1521/FREEPDB1 +ORACLE_CONFIG_DIR=/app/api/storage/wallet +ORACLE_WALLET_LOCATION=/app/api/storage/wallet +ORACLE_WALLET_PASSWORD=dify +ORACLE_IS_AUTONOMOUS=false # relyt configurations, only available when VECTOR_STORE is `relyt` RELYT_HOST=db @@ -499,9 +530,13 @@ RELYT_DATABASE=postgres # open search configuration, only available when VECTOR_STORE is `opensearch` OPENSEARCH_HOST=opensearch OPENSEARCH_PORT=9200 +OPENSEARCH_SECURE=true +OPENSEARCH_AUTH_METHOD=basic OPENSEARCH_USER=admin OPENSEARCH_PASSWORD=admin -OPENSEARCH_SECURE=true +# If using AWS managed IAM, e.g. Managed Cluster or OpenSearch Serverless +OPENSEARCH_AWS_REGION=ap-southeast-1 +OPENSEARCH_AWS_SERVICE=aoss # tencent vector configurations, only available when VECTOR_STORE is `tencent` TENCENT_VECTOR_DB_URL=http://127.0.0.1 @@ -511,6 +546,7 @@ TENCENT_VECTOR_DB_USERNAME=dify TENCENT_VECTOR_DB_DATABASE=dify TENCENT_VECTOR_DB_SHARD=1 TENCENT_VECTOR_DB_REPLICAS=2 +TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH=false # ElasticSearch configuration, only available when VECTOR_STORE is `elasticsearch` ELASTICSEARCH_HOST=0.0.0.0 @@ -541,6 +577,7 @@ VIKINGDB_SOCKET_TIMEOUT=30 LINDORM_URL=http://lindorm:30070 LINDORM_USERNAME=lindorm LINDORM_PASSWORD=lindorm +LINDORM_QUERY_TIMEOUT=1 # OceanBase Vector configuration, only available when VECTOR_STORE is `oceanbase` OCEANBASE_VECTOR_HOST=oceanbase @@ -550,11 +587,34 @@ OCEANBASE_VECTOR_PASSWORD=difyai123456 OCEANBASE_VECTOR_DATABASE=test OCEANBASE_CLUSTER_NAME=difyai OCEANBASE_MEMORY_LIMIT=6G +OCEANBASE_ENABLE_HYBRID_SEARCH=false + +# opengauss configurations, only available when VECTOR_STORE is `opengauss` +OPENGAUSS_HOST=opengauss +OPENGAUSS_PORT=6600 +OPENGAUSS_USER=postgres +OPENGAUSS_PASSWORD=Dify@123 +OPENGAUSS_DATABASE=dify +OPENGAUSS_MIN_CONNECTION=1 +OPENGAUSS_MAX_CONNECTION=5 +OPENGAUSS_ENABLE_PQ=false + +# huawei cloud search service vector configurations, only available when VECTOR_STORE is `huawei_cloud` +HUAWEI_CLOUD_HOSTS=https://127.0.0.1:9200 +HUAWEI_CLOUD_USER=admin +HUAWEI_CLOUD_PASSWORD=admin # Upstash Vector configuration, only available when VECTOR_STORE is `upstash` UPSTASH_VECTOR_URL=https://xxx-vector.upstash.io UPSTASH_VECTOR_TOKEN=dify +# TableStore Vector configuration +# (only used when VECTOR_STORE is tablestore) +TABLESTORE_ENDPOINT=https://instance-name.cn-hangzhou.ots.aliyuncs.com +TABLESTORE_INSTANCE_NAME=instance-name +TABLESTORE_ACCESS_KEY_ID=xxx +TABLESTORE_ACCESS_KEY_SECRET=xxx + # ------------------------------ # Knowledge Configuration # ------------------------------ @@ -582,17 +642,22 @@ SCARF_NO_ANALYTICS=true # ------------------------------ # The maximum number of tokens allowed for prompt generation. -# This setting controls the upper limit of tokens that can be used by the LLM +# This setting controls the upper limit of tokens that can be used by the LLM # when generating a prompt in the prompt generation tool. # Default: 512 tokens. PROMPT_GENERATION_MAX_TOKENS=512 # The maximum number of tokens allowed for code generation. -# This setting controls the upper limit of tokens that can be used by the LLM +# This setting controls the upper limit of tokens that can be used by the LLM # when generating code in the code generation tool. # Default: 1024 tokens. CODE_GENERATION_MAX_TOKENS=1024 +# Enable or disable plugin based token counting. If disabled, token counting will return 0. +# This can improve performance by skipping token counting operations. +# Default: false (disabled). +PLUGIN_BASED_TOKEN_COUNTING_ENABLED=false + # ------------------------------ # Multi-modal Configuration # ------------------------------ @@ -709,15 +774,34 @@ MAX_VARIABLE_SIZE=204800 WORKFLOW_PARALLEL_DEPTH_LIMIT=3 WORKFLOW_FILE_UPLOAD_LIMIT=10 +# Workflow storage configuration +# Options: rdbms, hybrid +# rdbms: Use only the relational database (default) +# hybrid: Save new data to object storage, read from both object storage and RDBMS +WORKFLOW_NODE_EXECUTION_STORAGE=rdbms + # HTTP request node in workflow configuration HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576 +HTTP_REQUEST_NODE_SSL_VERIFY=True # SSRF Proxy server HTTP URL SSRF_PROXY_HTTP_URL=http://ssrf_proxy:3128 # SSRF Proxy server HTTPS URL SSRF_PROXY_HTTPS_URL=http://ssrf_proxy:3128 +# Maximum loop count in the workflow +LOOP_NODE_MAX_COUNT=100 + +# The maximum number of tools that can be used in the agent. +MAX_TOOLS_NUM=10 + +# Maximum number of Parallelism branches in the workflow +MAX_PARALLEL_LIMIT=10 + +# The maximum number of iterations for agent setting +MAX_ITERATIONS_NUM=5 + # ------------------------------ # Environment Variables for web Service # ------------------------------ @@ -785,7 +869,7 @@ CHROMA_IS_PERSISTENT=TRUE # ------------------------------ # Environment Variables for Oracle Service -# (only used when VECTOR_STORE is Oracle) +# (only used when VECTOR_STORE is oracle) # ------------------------------ ORACLE_PWD=Dify123456 ORACLE_CHARACTERSET=AL32UTF8 @@ -954,6 +1038,7 @@ PLUGIN_DEBUGGING_PORT=5003 EXPOSE_PLUGIN_DEBUGGING_HOST=localhost EXPOSE_PLUGIN_DEBUGGING_PORT=5003 +# If this key is changed, DIFY_INNER_API_KEY in plugin_daemon service must also be updated or agent node will fail. PLUGIN_DIFY_INNER_API_KEY=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1 PLUGIN_DIFY_INNER_API_URL=http://api:5001 @@ -963,3 +1048,59 @@ MARKETPLACE_ENABLED=true MARKETPLACE_API_URL=https://marketplace.dify.ai FORCE_VERIFYING_SIGNATURE=true + +PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120 +PLUGIN_MAX_EXECUTION_TIMEOUT=600 +# PIP_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple +PIP_MIRROR_URL= + +# https://github.com/langgenius/dify-plugin-daemon/blob/main/.env.example +# Plugin storage type, local aws_s3 tencent_cos azure_blob aliyun_oss +PLUGIN_STORAGE_TYPE=local +PLUGIN_STORAGE_LOCAL_ROOT=/app/storage +PLUGIN_WORKING_PATH=/app/storage/cwd +PLUGIN_INSTALLED_PATH=plugin +PLUGIN_PACKAGE_CACHE_PATH=plugin_packages +PLUGIN_MEDIA_CACHE_PATH=assets +# Plugin oss bucket +PLUGIN_STORAGE_OSS_BUCKET= +# Plugin oss s3 credentials +PLUGIN_S3_USE_AWS_MANAGED_IAM=false +PLUGIN_S3_ENDPOINT= +PLUGIN_S3_USE_PATH_STYLE=false +PLUGIN_AWS_ACCESS_KEY= +PLUGIN_AWS_SECRET_KEY= +PLUGIN_AWS_REGION= +# Plugin oss azure blob +PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME= +PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING= +# Plugin oss tencent cos +PLUGIN_TENCENT_COS_SECRET_KEY= +PLUGIN_TENCENT_COS_SECRET_ID= +PLUGIN_TENCENT_COS_REGION= +# Plugin oss aliyun oss +PLUGIN_ALIYUN_OSS_REGION= +PLUGIN_ALIYUN_OSS_ENDPOINT= +PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID= +PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET= +PLUGIN_ALIYUN_OSS_AUTH_VERSION=v4 +PLUGIN_ALIYUN_OSS_PATH= + +# ------------------------------ +# OTLP Collector Configuration +# ------------------------------ +ENABLE_OTEL=false +OTLP_BASE_ENDPOINT=http://localhost:4318 +OTLP_API_KEY= +OTEL_EXPORTER_OTLP_PROTOCOL= +OTEL_EXPORTER_TYPE=otlp +OTEL_SAMPLING_RATE=0.1 +OTEL_BATCH_EXPORT_SCHEDULE_DELAY=5000 +OTEL_MAX_QUEUE_SIZE=2048 +OTEL_MAX_EXPORT_BATCH_SIZE=512 +OTEL_METRIC_EXPORT_INTERVAL=60000 +OTEL_BATCH_EXPORT_TIMEOUT=10000 +OTEL_METRIC_EXPORT_TIMEOUT=30000 + +# Prevent Clickjacking +ALLOW_EMBED=false diff --git a/dify/code/README.md b/dify/code/README.md index c3cd1f9..22dfe2c 100644 --- a/dify/code/README.md +++ b/dify/code/README.md @@ -14,7 +14,6 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T - **Unified Vector Database Services**: All vector database services are now managed from a single Docker Compose file `docker-compose.yaml`. You can switch between different vector databases by setting the `VECTOR_STORE` environment variable in your `.env` file. - **Mandatory .env File**: A `.env` file is now required to run `docker compose up`. This file is crucial for configuring your deployment and for any custom settings to persist through upgrades. -- **Legacy Support**: Previous deployment files are now located in the `docker-legacy` directory and will no longer be maintained. ### How to Deploy Dify with `docker-compose.yaml` @@ -27,7 +26,10 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T - Execute `docker compose up` from the `docker` directory to start the services. - To specify a vector database, set the `VECTOR_STORE` variable in your `.env` file to your desired vector database service, such as `milvus`, `weaviate`, or `opensearch`. 4. **SSL Certificate Setup**: - - Rrefer `docker/certbot/README.md` to set up SSL certificates using Certbot. + - Refer `docker/certbot/README.md` to set up SSL certificates using Certbot. +5. **OpenTelemetry Collector Setup**: + - Change `ENABLE_OTEL` to `true` in `.env`. + - Configure `OTLP_BASE_ENDPOINT` properly. ### How to Deploy Middleware for Developing Dify @@ -36,7 +38,8 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T - Navigate to the `docker` directory. - Ensure the `middleware.env` file is created by running `cp middleware.env.example middleware.env` (refer to the `middleware.env.example` file). 2. **Running Middleware Services**: - - Execute `docker-compose -f docker-compose.middleware.yaml up --env-file middleware.env -d` to start the middleware services. + - Navigate to the `docker` directory. + - Execute `docker compose -f docker-compose.middleware.yaml --profile weaviate -p dify up -d` to start the middleware services. (Change the profile to other vector database if you are not using weaviate) ### Migration for Existing Users @@ -54,7 +57,7 @@ For users migrating from the `docker-legacy` setup: - **Vector Database Services**: Depending on the type of vector database used (`VECTOR_STORE`), users can set specific endpoints, ports, and authentication details. - **Storage Services**: Depending on the storage type (`STORAGE_TYPE`), users can configure specific settings for S3, Azure Blob, Google Storage, etc. -- **API and Web Services**: Users can define URLs and other settings that affect how the API and web frontends operate. +- **API and Web Services**: Users can define URLs and other settings that affect how the API and web frontend operate. #### Other notable variables @@ -88,7 +91,11 @@ The `.env.example` file provided in the Docker setup is extensive and covers a w 8. **CORS Configuration**: - `WEB_API_CORS_ALLOW_ORIGINS`, `CONSOLE_CORS_ALLOW_ORIGINS`: Settings for cross-origin resource sharing. -9. **Other Service-Specific Environment Variables**: +9. **OpenTelemetry Configuration**: + - `ENABLE_OTEL`: Enable OpenTelemetry collector in api. + - `OTLP_BASE_ENDPOINT`: Endpoint for your OTLP exporter. + +10. **Other Service-Specific Environment Variables**: - Each service like `nginx`, `redis`, `db`, and vector databases have specific environment variables that are directly referenced in the `docker-compose.yaml`. ### Additional Information diff --git a/dify/code/couchbase-server/Dockerfile b/dify/code/couchbase-server/Dockerfile index bd8af64..23e487e 100644 --- a/dify/code/couchbase-server/Dockerfile +++ b/dify/code/couchbase-server/Dockerfile @@ -1,4 +1,4 @@ FROM couchbase/server:latest AS stage_base -# FROM couchbase:latest AS stage_base +# FROM couchbase:latest AS stage_base COPY init-cbserver.sh /opt/couchbase/init/ -RUN chmod +x /opt/couchbase/init/init-cbserver.sh \ No newline at end of file +RUN chmod +x /opt/couchbase/init/init-cbserver.sh diff --git a/dify/code/couchbase-server/init-cbserver.sh b/dify/code/couchbase-server/init-cbserver.sh index e66bc18..e19a650 100755 --- a/dify/code/couchbase-server/init-cbserver.sh +++ b/dify/code/couchbase-server/init-cbserver.sh @@ -1,8 +1,8 @@ #!/bin/bash -# used to start couchbase server - can't get around this as docker compose only allows you to start one command - so we have to start couchbase like the standard couchbase Dockerfile would +# used to start couchbase server - can't get around this as docker compose only allows you to start one command - so we have to start couchbase like the standard couchbase Dockerfile would # https://github.com/couchbase/docker/blob/master/enterprise/couchbase-server/7.2.0/Dockerfile#L88 -/entrypoint.sh couchbase-server & +/entrypoint.sh couchbase-server & # track if setup is complete so we don't try to setup again FILE=/opt/couchbase/init/setupComplete.txt @@ -36,9 +36,9 @@ if ! [ -f "$FILE" ]; then --bucket-ramsize $COUCHBASE_BUCKET_RAMSIZE \ --bucket-type couchbase - # create file so we know that the cluster is setup and don't run the setup again + # create file so we know that the cluster is setup and don't run the setup again touch $FILE -fi +fi # docker compose will stop the container from running unless we do this # known issue and workaround tail -f /dev/null diff --git a/dify/code/docker-compose-template.yaml b/dify/code/docker-compose-template.yaml index f284449..ceb32e4 100644 --- a/dify/code/docker-compose-template.yaml +++ b/dify/code/docker-compose-template.yaml @@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:0.15.3 + image: langgenius/dify-api:1.4.0 restart: always environment: # Use the shared environment variables. @@ -12,11 +12,15 @@ services: SENTRY_DSN: ${API_SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} + PLUGIN_REMOTE_INSTALL_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost} + PLUGIN_REMOTE_INSTALL_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003} PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} depends_on: - - db - - redis + db: + condition: service_healthy + redis: + condition: service_started volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage @@ -27,7 +31,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.15.3 + image: langgenius/dify-api:1.4.0 restart: always environment: # Use the shared environment variables. @@ -40,8 +44,10 @@ services: PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} depends_on: - - db - - redis + db: + condition: service_healthy + redis: + condition: service_started volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage @@ -51,7 +57,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.15.3 + image: langgenius/dify-web:1.4.0 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -60,12 +66,19 @@ services: NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-0} TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} CSP_WHITELIST: ${CSP_WHITELIST:-} + ALLOW_EMBED: ${ALLOW_EMBED:-false} MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai} TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-} INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-} PM2_INSTANCES: ${PM2_INSTANCES:-2} - + LOOP_NODE_MAX_COUNT: ${LOOP_NODE_MAX_COUNT:-100} + MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} + MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} + MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-5} + ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} + ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} + ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} # The postgres database. db: image: postgres:15-alpine @@ -84,12 +97,10 @@ services: volumes: - ./volumes/db/data:/var/lib/postgresql/data healthcheck: - test: [ 'CMD', 'pg_isready' ] + test: [ 'CMD', 'pg_isready', '-h', 'db', '-U', '${PGUSER:-postgres}', '-d', '${POSTGRES_DB:-dify}' ] interval: 1s timeout: 3s - retries: 30 - ports: - - '${EXPOSE_DB_PORT:-5432}:5432' + retries: 60 # The redis cache. redis: @@ -107,7 +118,7 @@ services: # The DifySandbox sandbox: - image: langgenius/dify-sandbox:0.2.10 + image: langgenius/dify-sandbox:0.2.12 restart: always environment: # The DifySandbox configurations @@ -120,8 +131,10 @@ services: HTTP_PROXY: ${SANDBOX_HTTP_PROXY:-http://ssrf_proxy:3128} HTTPS_PROXY: ${SANDBOX_HTTPS_PROXY:-http://ssrf_proxy:3128} SANDBOX_PORT: ${SANDBOX_PORT:-8194} + PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} volumes: - ./volumes/sandbox/dependencies:/dependencies + - ./volumes/sandbox/conf:/conf healthcheck: test: [ 'CMD', 'curl', '-f', 'http://localhost:8194/health' ] networks: @@ -129,7 +142,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.0.2-local + image: langgenius/dify-plugin-daemon:0.0.10-local restart: always environment: # Use the shared environment variables. @@ -140,20 +153,48 @@ services: MAX_PLUGIN_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false} DIFY_INNER_API_URL: ${PLUGIN_DIFY_INNER_API_URL:-http://api:5001} - DIFY_INNER_API_KEY: ${INNER_API_KEY_FOR_PLUGIN:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} - PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_REMOTE_INSTALL_HOST:-0.0.0.0} - PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_REMOTE_INSTALL_PORT:-5003} + DIFY_INNER_API_KEY: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_DEBUGGING_HOST:-0.0.0.0} + PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003} PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} + PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} + PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} + PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} + PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} + PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} + PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} + PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} + S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} + AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} + AWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} + AWS_REGION: ${PLUGIN_AWS_REGION:-} + AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-} + AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-} + TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-} + TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-} + TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-} + ALIYUN_OSS_REGION: ${PLUGIN_ALIYUN_OSS_REGION:-} + ALIYUN_OSS_ENDPOINT: ${PLUGIN_ALIYUN_OSS_ENDPOINT:-} + ALIYUN_OSS_ACCESS_KEY_ID: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID:-} + ALIYUN_OSS_ACCESS_KEY_SECRET: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET:-} + ALIYUN_OSS_AUTH_VERSION: ${PLUGIN_ALIYUN_OSS_AUTH_VERSION:-v4} + ALIYUN_OSS_PATH: ${PLUGIN_ALIYUN_OSS_PATH:-} ports: - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}" volumes: - ./volumes/plugin_daemon:/app/storage - + depends_on: + db: + condition: service_healthy # ssrf_proxy server # for more information, please refer to - # https://docs.dify.ai/learn-more/faq/install-faq#id-18.-why-is-ssrf_proxy-needed + # https://docs.dify.ai/learn-more/faq/install-faq#18-why-is-ssrf-proxy-needed%3F ssrf_proxy: image: ubuntu/squid:latest restart: always @@ -315,8 +356,37 @@ services: POSTGRES_DB: ${PGVECTOR_POSTGRES_DB:-dify} # postgres data directory PGDATA: ${PGVECTOR_PGDATA:-/var/lib/postgresql/data/pgdata} + # pg_bigm module for full text search + PG_BIGM: ${PGVECTOR_PG_BIGM:-false} + PG_BIGM_VERSION: ${PGVECTOR_PG_BIGM_VERSION:-1.2-20240606} volumes: - ./volumes/pgvector/data:/var/lib/postgresql/data + - ./pgvector/docker-entrypoint.sh:/docker-entrypoint.sh + entrypoint: [ '/docker-entrypoint.sh' ] + healthcheck: + test: [ 'CMD', 'pg_isready' ] + interval: 1s + timeout: 3s + retries: 30 + + # get image from https://www.vastdata.com.cn/ + vastbase: + image: vastdata/vastbase-vector + profiles: + - vastbase + restart: always + environment: + - VB_DBCOMPATIBILITY=PG + - VB_DB=dify + - VB_USERNAME=dify + - VB_PASSWORD=Difyai123456 + ports: + - '5434:5432' + volumes: + - ./vastbase/lic:/home/vastbase/vastbase/lic + - ./vastbase/data:/home/vastbase/data + - ./vastbase/backup:/home/vastbase/backup + - ./vastbase/backup_log:/home/vastbase/backup_log healthcheck: test: [ 'CMD', 'pg_isready' ] interval: 1s @@ -360,7 +430,8 @@ services: # OceanBase vector database oceanbase: - image: quay.io/oceanbase/oceanbase-ce:4.3.3.0-100000142024101215 + image: oceanbase/oceanbase-ce:4.3.5.1-101000042025031818 + container_name: oceanbase profiles: - oceanbase restart: always @@ -373,7 +444,9 @@ services: OB_SYS_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} OB_TENANT_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} - OB_SERVER_IP: '127.0.0.1' + MODE: MINI + ports: + - "${OCEANBASE_VECTOR_PORT:-2881}:2881" # Oracle vector database oracle: @@ -495,6 +568,28 @@ services: depends_on: - opensearch + # opengauss vector database. + opengauss: + image: opengauss/opengauss:7.0.0-RC1 + profiles: + - opengauss + privileged: true + restart: always + environment: + GS_USERNAME: ${OPENGAUSS_USER:-postgres} + GS_PASSWORD: ${OPENGAUSS_PASSWORD:-Dify@123} + GS_PORT: ${OPENGAUSS_PORT:-6600} + GS_DB: ${OPENGAUSS_DATABASE:-dify} + volumes: + - ./volumes/opengauss/data:/var/lib/opengauss/data + healthcheck: + test: [ "CMD-SHELL", "netstat -lntp | grep tcp6 > /dev/null 2>&1" ] + interval: 10s + timeout: 10s + retries: 10 + ports: + - ${OPENGAUSS_PORT:-6600}:${OPENGAUSS_PORT:-6600} + # MyScale vector database myscale: container_name: myscale diff --git a/dify/code/docker-compose.middleware.yaml b/dify/code/docker-compose.middleware.yaml index 961f953..8690a50 100644 --- a/dify/code/docker-compose.middleware.yaml +++ b/dify/code/docker-compose.middleware.yaml @@ -29,6 +29,8 @@ services: redis: image: redis:6-alpine restart: always + env_file: + - ./middleware.env environment: REDISCLI_AUTH: ${REDIS_PASSWORD:-difyai123456} volumes: @@ -43,8 +45,10 @@ services: # The DifySandbox sandbox: - image: langgenius/dify-sandbox:0.2.10 + image: langgenius/dify-sandbox:0.2.12 restart: always + env_file: + - ./middleware.env environment: # The DifySandbox configurations # Make sure you are changing this key for your deployment with a strong key. @@ -56,6 +60,7 @@ services: HTTP_PROXY: ${SANDBOX_HTTP_PROXY:-http://ssrf_proxy:3128} HTTPS_PROXY: ${SANDBOX_HTTPS_PROXY:-http://ssrf_proxy:3128} SANDBOX_PORT: ${SANDBOX_PORT:-8194} + PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} volumes: - ./volumes/sandbox/dependencies:/dependencies - ./volumes/sandbox/conf:/conf @@ -66,8 +71,10 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.0.2-local + image: langgenius/dify-plugin-daemon:0.0.10-local restart: always + env_file: + - ./middleware.env environment: # Use the shared environment variables. DB_HOST: ${DB_HOST:-db} @@ -88,6 +95,32 @@ services: PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003} PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} + PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} + PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} + PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} + PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} + PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} + PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} + PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} + S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} + AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} + AWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} + AWS_REGION: ${PLUGIN_AWS_REGION:-} + AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-} + AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-} + TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-} + TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-} + TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-} + ALIYUN_OSS_REGION: ${PLUGIN_ALIYUN_OSS_REGION:-} + ALIYUN_OSS_ENDPOINT: ${PLUGIN_ALIYUN_OSS_ENDPOINT:-} + ALIYUN_OSS_ACCESS_KEY_ID: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID:-} + ALIYUN_OSS_ACCESS_KEY_SECRET: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET:-} + ALIYUN_OSS_AUTH_VERSION: ${PLUGIN_ALIYUN_OSS_AUTH_VERSION:-v4} + ALIYUN_OSS_PATH: ${PLUGIN_ALIYUN_OSS_PATH:-} ports: - "${EXPOSE_PLUGIN_DAEMON_PORT:-5002}:${PLUGIN_DAEMON_PORT:-5002}" - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}" @@ -96,7 +129,7 @@ services: # ssrf_proxy server # for more information, please refer to - # https://docs.dify.ai/learn-more/faq/install-faq#id-18.-why-is-ssrf_proxy-needed + # https://docs.dify.ai/learn-more/faq/install-faq#18-why-is-ssrf-proxy-needed%3F ssrf_proxy: image: ubuntu/squid:latest restart: always @@ -104,6 +137,8 @@ services: - ./ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template - ./ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint-mount.sh entrypoint: [ "sh", "-c", "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" ] + env_file: + - ./middleware.env environment: # pls clearly modify the squid env vars to fit your network environment. HTTP_PORT: ${SSRF_HTTP_PORT:-3128} diff --git a/dify/code/docker-compose.png b/dify/code/docker-compose.png index bdac113..015d450 100644 Binary files a/dify/code/docker-compose.png and b/dify/code/docker-compose.png differ diff --git a/dify/code/docker-compose.yaml b/dify/code/docker-compose.yaml index 5ee9cb3..90dd29c 100644 --- a/dify/code/docker-compose.yaml +++ b/dify/code/docker-compose.yaml @@ -19,6 +19,7 @@ x-shared-env: &shared-api-worker-env LOG_TZ: ${LOG_TZ:-UTC} DEBUG: ${DEBUG:-false} FLASK_DEBUG: ${FLASK_DEBUG:-false} + ENABLE_REQUEST_LOGGING: ${ENABLE_REQUEST_LOGGING:-False} SECRET_KEY: ${SECRET_KEY:-sk-9f73s3ljTXVcMT3Blb3ljTqtsKiGHXVcMT3BlbkFJLK7U} INIT_PASSWORD: ${INIT_PASSWORD:-} DEPLOY_ENV: ${DEPLOY_ENV:-PRODUCTION} @@ -43,6 +44,9 @@ x-shared-env: &shared-api-worker-env CELERY_MIN_WORKERS: ${CELERY_MIN_WORKERS:-} API_TOOL_DEFAULT_CONNECT_TIMEOUT: ${API_TOOL_DEFAULT_CONNECT_TIMEOUT:-10} API_TOOL_DEFAULT_READ_TIMEOUT: ${API_TOOL_DEFAULT_READ_TIMEOUT:-60} + ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} + ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} + ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} DB_USERNAME: ${DB_USERNAME:-postgres} DB_PASSWORD: ${DB_PASSWORD:-difyai123456} DB_HOST: ${DB_HOST:-db} @@ -105,7 +109,7 @@ x-shared-env: &shared-api-worker-env TENCENT_COS_SECRET_ID: ${TENCENT_COS_SECRET_ID:-your-secret-id} TENCENT_COS_REGION: ${TENCENT_COS_REGION:-your-region} TENCENT_COS_SCHEME: ${TENCENT_COS_SCHEME:-your-scheme} - OCI_ENDPOINT: ${OCI_ENDPOINT:-https://objectstorage.us-ashburn-1.oraclecloud.com} + OCI_ENDPOINT: ${OCI_ENDPOINT:-https://your-object-storage-namespace.compat.objectstorage.us-ashburn-1.oraclecloud.com} OCI_BUCKET_NAME: ${OCI_BUCKET_NAME:-your-bucket-name} OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-your-access-key} OCI_SECRET_KEY: ${OCI_SECRET_KEY:-your-secret-key} @@ -134,11 +138,13 @@ x-shared-env: &shared-api-worker-env QDRANT_CLIENT_TIMEOUT: ${QDRANT_CLIENT_TIMEOUT:-20} QDRANT_GRPC_ENABLED: ${QDRANT_GRPC_ENABLED:-false} QDRANT_GRPC_PORT: ${QDRANT_GRPC_PORT:-6334} - MILVUS_URI: ${MILVUS_URI:-http://127.0.0.1:19530} + MILVUS_URI: ${MILVUS_URI:-http://host.docker.internal:19530} + MILVUS_DATABASE: ${MILVUS_DATABASE:-} MILVUS_TOKEN: ${MILVUS_TOKEN:-} - MILVUS_USER: ${MILVUS_USER:-root} - MILVUS_PASSWORD: ${MILVUS_PASSWORD:-Milvus} + MILVUS_USER: ${MILVUS_USER:-} + MILVUS_PASSWORD: ${MILVUS_PASSWORD:-} MILVUS_ENABLE_HYBRID_SEARCH: ${MILVUS_ENABLE_HYBRID_SEARCH:-False} + MILVUS_ANALYZER_PARAMS: ${MILVUS_ANALYZER_PARAMS:-} MYSCALE_HOST: ${MYSCALE_HOST:-myscale} MYSCALE_PORT: ${MYSCALE_PORT:-8123} MYSCALE_USER: ${MYSCALE_USER:-default} @@ -157,6 +163,15 @@ x-shared-env: &shared-api-worker-env PGVECTOR_DATABASE: ${PGVECTOR_DATABASE:-dify} PGVECTOR_MIN_CONNECTION: ${PGVECTOR_MIN_CONNECTION:-1} PGVECTOR_MAX_CONNECTION: ${PGVECTOR_MAX_CONNECTION:-5} + PGVECTOR_PG_BIGM: ${PGVECTOR_PG_BIGM:-false} + PGVECTOR_PG_BIGM_VERSION: ${PGVECTOR_PG_BIGM_VERSION:-1.2-20240606} + VASTBASE_HOST: ${VASTBASE_HOST:-vastbase} + VASTBASE_PORT: ${VASTBASE_PORT:-5432} + VASTBASE_USER: ${VASTBASE_USER:-dify} + VASTBASE_PASSWORD: ${VASTBASE_PASSWORD:-Difyai123456} + VASTBASE_DATABASE: ${VASTBASE_DATABASE:-dify} + VASTBASE_MIN_CONNECTION: ${VASTBASE_MIN_CONNECTION:-1} + VASTBASE_MAX_CONNECTION: ${VASTBASE_MAX_CONNECTION:-5} PGVECTO_RS_HOST: ${PGVECTO_RS_HOST:-pgvecto-rs} PGVECTO_RS_PORT: ${PGVECTO_RS_PORT:-5432} PGVECTO_RS_USER: ${PGVECTO_RS_USER:-postgres} @@ -197,11 +212,13 @@ x-shared-env: &shared-api-worker-env CHROMA_DATABASE: ${CHROMA_DATABASE:-default_database} CHROMA_AUTH_PROVIDER: ${CHROMA_AUTH_PROVIDER:-chromadb.auth.token_authn.TokenAuthClientProvider} CHROMA_AUTH_CREDENTIALS: ${CHROMA_AUTH_CREDENTIALS:-} - ORACLE_HOST: ${ORACLE_HOST:-oracle} - ORACLE_PORT: ${ORACLE_PORT:-1521} ORACLE_USER: ${ORACLE_USER:-dify} ORACLE_PASSWORD: ${ORACLE_PASSWORD:-dify} - ORACLE_DATABASE: ${ORACLE_DATABASE:-FREEPDB1} + ORACLE_DSN: ${ORACLE_DSN:-oracle:1521/FREEPDB1} + ORACLE_CONFIG_DIR: ${ORACLE_CONFIG_DIR:-/app/api/storage/wallet} + ORACLE_WALLET_LOCATION: ${ORACLE_WALLET_LOCATION:-/app/api/storage/wallet} + ORACLE_WALLET_PASSWORD: ${ORACLE_WALLET_PASSWORD:-dify} + ORACLE_IS_AUTONOMOUS: ${ORACLE_IS_AUTONOMOUS:-false} RELYT_HOST: ${RELYT_HOST:-db} RELYT_PORT: ${RELYT_PORT:-5432} RELYT_USER: ${RELYT_USER:-postgres} @@ -209,9 +226,12 @@ x-shared-env: &shared-api-worker-env RELYT_DATABASE: ${RELYT_DATABASE:-postgres} OPENSEARCH_HOST: ${OPENSEARCH_HOST:-opensearch} OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200} + OPENSEARCH_SECURE: ${OPENSEARCH_SECURE:-true} + OPENSEARCH_AUTH_METHOD: ${OPENSEARCH_AUTH_METHOD:-basic} OPENSEARCH_USER: ${OPENSEARCH_USER:-admin} OPENSEARCH_PASSWORD: ${OPENSEARCH_PASSWORD:-admin} - OPENSEARCH_SECURE: ${OPENSEARCH_SECURE:-true} + OPENSEARCH_AWS_REGION: ${OPENSEARCH_AWS_REGION:-ap-southeast-1} + OPENSEARCH_AWS_SERVICE: ${OPENSEARCH_AWS_SERVICE:-aoss} TENCENT_VECTOR_DB_URL: ${TENCENT_VECTOR_DB_URL:-http://127.0.0.1} TENCENT_VECTOR_DB_API_KEY: ${TENCENT_VECTOR_DB_API_KEY:-dify} TENCENT_VECTOR_DB_TIMEOUT: ${TENCENT_VECTOR_DB_TIMEOUT:-30} @@ -219,6 +239,7 @@ x-shared-env: &shared-api-worker-env TENCENT_VECTOR_DB_DATABASE: ${TENCENT_VECTOR_DB_DATABASE:-dify} TENCENT_VECTOR_DB_SHARD: ${TENCENT_VECTOR_DB_SHARD:-1} TENCENT_VECTOR_DB_REPLICAS: ${TENCENT_VECTOR_DB_REPLICAS:-2} + TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH: ${TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH:-false} ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-0.0.0.0} ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_USERNAME:-elastic} @@ -241,6 +262,7 @@ x-shared-env: &shared-api-worker-env LINDORM_URL: ${LINDORM_URL:-http://lindorm:30070} LINDORM_USERNAME: ${LINDORM_USERNAME:-lindorm} LINDORM_PASSWORD: ${LINDORM_PASSWORD:-lindorm} + LINDORM_QUERY_TIMEOUT: ${LINDORM_QUERY_TIMEOUT:-1} OCEANBASE_VECTOR_HOST: ${OCEANBASE_VECTOR_HOST:-oceanbase} OCEANBASE_VECTOR_PORT: ${OCEANBASE_VECTOR_PORT:-2881} OCEANBASE_VECTOR_USER: ${OCEANBASE_VECTOR_USER:-root@test} @@ -248,8 +270,24 @@ x-shared-env: &shared-api-worker-env OCEANBASE_VECTOR_DATABASE: ${OCEANBASE_VECTOR_DATABASE:-test} OCEANBASE_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} OCEANBASE_MEMORY_LIMIT: ${OCEANBASE_MEMORY_LIMIT:-6G} + OCEANBASE_ENABLE_HYBRID_SEARCH: ${OCEANBASE_ENABLE_HYBRID_SEARCH:-false} + OPENGAUSS_HOST: ${OPENGAUSS_HOST:-opengauss} + OPENGAUSS_PORT: ${OPENGAUSS_PORT:-6600} + OPENGAUSS_USER: ${OPENGAUSS_USER:-postgres} + OPENGAUSS_PASSWORD: ${OPENGAUSS_PASSWORD:-Dify@123} + OPENGAUSS_DATABASE: ${OPENGAUSS_DATABASE:-dify} + OPENGAUSS_MIN_CONNECTION: ${OPENGAUSS_MIN_CONNECTION:-1} + OPENGAUSS_MAX_CONNECTION: ${OPENGAUSS_MAX_CONNECTION:-5} + OPENGAUSS_ENABLE_PQ: ${OPENGAUSS_ENABLE_PQ:-false} + HUAWEI_CLOUD_HOSTS: ${HUAWEI_CLOUD_HOSTS:-https://127.0.0.1:9200} + HUAWEI_CLOUD_USER: ${HUAWEI_CLOUD_USER:-admin} + HUAWEI_CLOUD_PASSWORD: ${HUAWEI_CLOUD_PASSWORD:-admin} UPSTASH_VECTOR_URL: ${UPSTASH_VECTOR_URL:-https://xxx-vector.upstash.io} UPSTASH_VECTOR_TOKEN: ${UPSTASH_VECTOR_TOKEN:-dify} + TABLESTORE_ENDPOINT: ${TABLESTORE_ENDPOINT:-https://instance-name.cn-hangzhou.ots.aliyuncs.com} + TABLESTORE_INSTANCE_NAME: ${TABLESTORE_INSTANCE_NAME:-instance-name} + TABLESTORE_ACCESS_KEY_ID: ${TABLESTORE_ACCESS_KEY_ID:-xxx} + TABLESTORE_ACCESS_KEY_SECRET: ${TABLESTORE_ACCESS_KEY_SECRET:-xxx} UPLOAD_FILE_SIZE_LIMIT: ${UPLOAD_FILE_SIZE_LIMIT:-15} UPLOAD_FILE_BATCH_LIMIT: ${UPLOAD_FILE_BATCH_LIMIT:-5} ETL_TYPE: ${ETL_TYPE:-dify} @@ -258,6 +296,7 @@ x-shared-env: &shared-api-worker-env SCARF_NO_ANALYTICS: ${SCARF_NO_ANALYTICS:-true} PROMPT_GENERATION_MAX_TOKENS: ${PROMPT_GENERATION_MAX_TOKENS:-512} CODE_GENERATION_MAX_TOKENS: ${CODE_GENERATION_MAX_TOKENS:-1024} + PLUGIN_BASED_TOKEN_COUNTING_ENABLED: ${PLUGIN_BASED_TOKEN_COUNTING_ENABLED:-false} MULTIMODAL_SEND_FORMAT: ${MULTIMODAL_SEND_FORMAT:-base64} UPLOAD_IMAGE_FILE_SIZE_LIMIT: ${UPLOAD_IMAGE_FILE_SIZE_LIMIT:-10} UPLOAD_VIDEO_FILE_SIZE_LIMIT: ${UPLOAD_VIDEO_FILE_SIZE_LIMIT:-100} @@ -304,10 +343,16 @@ x-shared-env: &shared-api-worker-env MAX_VARIABLE_SIZE: ${MAX_VARIABLE_SIZE:-204800} WORKFLOW_PARALLEL_DEPTH_LIMIT: ${WORKFLOW_PARALLEL_DEPTH_LIMIT:-3} WORKFLOW_FILE_UPLOAD_LIMIT: ${WORKFLOW_FILE_UPLOAD_LIMIT:-10} + WORKFLOW_NODE_EXECUTION_STORAGE: ${WORKFLOW_NODE_EXECUTION_STORAGE:-rdbms} HTTP_REQUEST_NODE_MAX_BINARY_SIZE: ${HTTP_REQUEST_NODE_MAX_BINARY_SIZE:-10485760} HTTP_REQUEST_NODE_MAX_TEXT_SIZE: ${HTTP_REQUEST_NODE_MAX_TEXT_SIZE:-1048576} + HTTP_REQUEST_NODE_SSL_VERIFY: ${HTTP_REQUEST_NODE_SSL_VERIFY:-True} SSRF_PROXY_HTTP_URL: ${SSRF_PROXY_HTTP_URL:-http://ssrf_proxy:3128} SSRF_PROXY_HTTPS_URL: ${SSRF_PROXY_HTTPS_URL:-http://ssrf_proxy:3128} + LOOP_NODE_MAX_COUNT: ${LOOP_NODE_MAX_COUNT:-100} + MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} + MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} + MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-5} TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} PGUSER: ${PGUSER:-${DB_USERNAME}} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-${DB_PASSWORD}} @@ -410,11 +455,51 @@ x-shared-env: &shared-api-worker-env MARKETPLACE_ENABLED: ${MARKETPLACE_ENABLED:-true} MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + PLUGIN_PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} + PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} + PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} + PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} + PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} + PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} + PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} + PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} + PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} + PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} + PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} + PLUGIN_AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} + PLUGIN_AWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} + PLUGIN_AWS_REGION: ${PLUGIN_AWS_REGION:-} + PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-} + PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-} + PLUGIN_TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-} + PLUGIN_TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-} + PLUGIN_TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-} + PLUGIN_ALIYUN_OSS_REGION: ${PLUGIN_ALIYUN_OSS_REGION:-} + PLUGIN_ALIYUN_OSS_ENDPOINT: ${PLUGIN_ALIYUN_OSS_ENDPOINT:-} + PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID:-} + PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET:-} + PLUGIN_ALIYUN_OSS_AUTH_VERSION: ${PLUGIN_ALIYUN_OSS_AUTH_VERSION:-v4} + PLUGIN_ALIYUN_OSS_PATH: ${PLUGIN_ALIYUN_OSS_PATH:-} + ENABLE_OTEL: ${ENABLE_OTEL:-false} + OTLP_BASE_ENDPOINT: ${OTLP_BASE_ENDPOINT:-http://localhost:4318} + OTLP_API_KEY: ${OTLP_API_KEY:-} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-} + OTEL_EXPORTER_TYPE: ${OTEL_EXPORTER_TYPE:-otlp} + OTEL_SAMPLING_RATE: ${OTEL_SAMPLING_RATE:-0.1} + OTEL_BATCH_EXPORT_SCHEDULE_DELAY: ${OTEL_BATCH_EXPORT_SCHEDULE_DELAY:-5000} + OTEL_MAX_QUEUE_SIZE: ${OTEL_MAX_QUEUE_SIZE:-2048} + OTEL_MAX_EXPORT_BATCH_SIZE: ${OTEL_MAX_EXPORT_BATCH_SIZE:-512} + OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-60000} + OTEL_BATCH_EXPORT_TIMEOUT: ${OTEL_BATCH_EXPORT_TIMEOUT:-10000} + OTEL_METRIC_EXPORT_TIMEOUT: ${OTEL_METRIC_EXPORT_TIMEOUT:-30000} + ALLOW_EMBED: ${ALLOW_EMBED:-false} services: # API service api: - image: langgenius/dify-api:0.15.3 + image: langgenius/dify-api:1.4.0 restart: always environment: # Use the shared environment variables. @@ -424,11 +509,15 @@ services: SENTRY_DSN: ${API_SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} + PLUGIN_REMOTE_INSTALL_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost} + PLUGIN_REMOTE_INSTALL_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003} PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} depends_on: - - db - - redis + db: + condition: service_healthy + redis: + condition: service_started volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage @@ -439,7 +528,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.15.3 + image: langgenius/dify-api:1.4.0 restart: always environment: # Use the shared environment variables. @@ -452,8 +541,10 @@ services: PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} depends_on: - - db - - redis + db: + condition: service_healthy + redis: + condition: service_started volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage @@ -463,7 +554,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.15.3 + image: langgenius/dify-web:1.4.0 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -472,12 +563,19 @@ services: NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-0} TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} CSP_WHITELIST: ${CSP_WHITELIST:-} + ALLOW_EMBED: ${ALLOW_EMBED:-false} MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai} TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-} INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-} PM2_INSTANCES: ${PM2_INSTANCES:-2} - + LOOP_NODE_MAX_COUNT: ${LOOP_NODE_MAX_COUNT:-100} + MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} + MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} + MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-5} + ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} + ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} + ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} # The postgres database. db: image: postgres:15-alpine @@ -496,10 +594,20 @@ services: volumes: - ./volumes/db/data:/var/lib/postgresql/data healthcheck: - test: [ 'CMD', 'pg_isready' ] + test: + [ + 'CMD', + 'pg_isready', + '-h', + 'db', + '-U', + '${PGUSER:-postgres}', + '-d', + '${POSTGRES_DB:-dify}' + ] interval: 1s timeout: 3s - retries: 30 + retries: 60 # The redis cache. redis: @@ -517,7 +625,7 @@ services: # The DifySandbox sandbox: - image: langgenius/dify-sandbox:0.2.10 + image: langgenius/dify-sandbox:0.2.12 restart: always environment: # The DifySandbox configurations @@ -530,8 +638,10 @@ services: HTTP_PROXY: ${SANDBOX_HTTP_PROXY:-http://ssrf_proxy:3128} HTTPS_PROXY: ${SANDBOX_HTTPS_PROXY:-http://ssrf_proxy:3128} SANDBOX_PORT: ${SANDBOX_PORT:-8194} + PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} volumes: - ./volumes/sandbox/dependencies:/dependencies + - ./volumes/sandbox/conf:/conf healthcheck: test: [ 'CMD', 'curl', '-f', 'http://localhost:8194/health' ] networks: @@ -539,7 +649,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.0.2-local + image: langgenius/dify-plugin-daemon:0.0.10-local restart: always environment: # Use the shared environment variables. @@ -550,17 +660,46 @@ services: MAX_PLUGIN_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false} DIFY_INNER_API_URL: ${PLUGIN_DIFY_INNER_API_URL:-http://api:5001} - DIFY_INNER_API_KEY: ${INNER_API_KEY_FOR_PLUGIN:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} - PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_REMOTE_INSTALL_HOST:-0.0.0.0} - PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_REMOTE_INSTALL_PORT:-5003} + DIFY_INNER_API_KEY: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_DEBUGGING_HOST:-0.0.0.0} + PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003} PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} + PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} + PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} + PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} + PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} + PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} + PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} + PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} + S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} + S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} + AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} + AWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} + AWS_REGION: ${PLUGIN_AWS_REGION:-} + AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-} + AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-} + TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-} + TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-} + TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-} + ALIYUN_OSS_REGION: ${PLUGIN_ALIYUN_OSS_REGION:-} + ALIYUN_OSS_ENDPOINT: ${PLUGIN_ALIYUN_OSS_ENDPOINT:-} + ALIYUN_OSS_ACCESS_KEY_ID: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID:-} + ALIYUN_OSS_ACCESS_KEY_SECRET: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET:-} + ALIYUN_OSS_AUTH_VERSION: ${PLUGIN_ALIYUN_OSS_AUTH_VERSION:-v4} + ALIYUN_OSS_PATH: ${PLUGIN_ALIYUN_OSS_PATH:-} volumes: - ./volumes/plugin_daemon:/app/storage + depends_on: + db: + condition: service_healthy # ssrf_proxy server # for more information, please refer to - # https://docs.dify.ai/learn-more/faq/install-faq#id-18.-why-is-ssrf_proxy-needed + # https://docs.dify.ai/learn-more/faq/install-faq#18-why-is-ssrf-proxy-needed%3F ssrf_proxy: image: ubuntu/squid:latest restart: always @@ -738,8 +877,35 @@ services: POSTGRES_DB: ${PGVECTOR_POSTGRES_DB:-dify} # postgres data directory PGDATA: ${PGVECTOR_PGDATA:-/var/lib/postgresql/data/pgdata} + # pg_bigm module for full text search + PG_BIGM: ${PGVECTOR_PG_BIGM:-false} + PG_BIGM_VERSION: ${PGVECTOR_PG_BIGM_VERSION:-1.2-20240606} volumes: - ./volumes/pgvector/data:/var/lib/postgresql/data + - ./pgvector/docker-entrypoint.sh:/docker-entrypoint.sh + entrypoint: [ '/docker-entrypoint.sh' ] + healthcheck: + test: [ 'CMD', 'pg_isready' ] + interval: 1s + timeout: 3s + retries: 30 + + # get image from https://www.vastdata.com.cn/ + vastbase: + image: vastdata/vastbase-vector + profiles: + - vastbase + restart: always + environment: + - VB_DBCOMPATIBILITY=PG + - VB_DB=dify + - VB_USERNAME=dify + - VB_PASSWORD=Difyai123456 + volumes: + - ./vastbase/lic:/home/vastbase/vastbase/lic + - ./vastbase/data:/home/vastbase/data + - ./vastbase/backup:/home/vastbase/backup + - ./vastbase/backup_log:/home/vastbase/backup_log healthcheck: test: [ 'CMD', 'pg_isready' ] interval: 1s @@ -783,7 +949,7 @@ services: # OceanBase vector database oceanbase: - image: quay.io/oceanbase/oceanbase-ce:4.3.3.0-100000142024101215 + image: oceanbase/oceanbase-ce:4.3.5.1-101000042025031818 profiles: - oceanbase restart: always @@ -796,7 +962,7 @@ services: OB_SYS_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} OB_TENANT_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} - OB_SERVER_IP: '127.0.0.1' + MODE: MINI # Oracle vector database oracle: @@ -912,6 +1078,26 @@ services: depends_on: - opensearch + # opengauss vector database. + opengauss: + image: opengauss/opengauss:7.0.0-RC1 + profiles: + - opengauss + privileged: true + restart: always + environment: + GS_USERNAME: ${OPENGAUSS_USER:-postgres} + GS_PASSWORD: ${OPENGAUSS_PASSWORD:-Dify@123} + GS_PORT: ${OPENGAUSS_PORT:-6600} + GS_DB: ${OPENGAUSS_DATABASE:-dify} + volumes: + - ./volumes/opengauss/data:/var/lib/opengauss/data + healthcheck: + test: [ "CMD-SHELL", "netstat -lntp | grep tcp6 > /dev/null 2>&1" ] + interval: 10s + timeout: 10s + retries: 10 + # MyScale vector database myscale: image: myscale/myscaledb:1.6.4 diff --git a/dify/code/middleware.env.example b/dify/code/middleware.env.example index aaecad7..2437026 100644 --- a/dify/code/middleware.env.example +++ b/dify/code/middleware.env.example @@ -114,4 +114,41 @@ PLUGIN_DIFY_INNER_API_URL=http://api:5001 MARKETPLACE_ENABLED=true MARKETPLACE_API_URL=https://marketplace.dify.ai -FORCE_VERIFYING_SIGNATURE=true \ No newline at end of file +FORCE_VERIFYING_SIGNATURE=true + +PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120 +PLUGIN_MAX_EXECUTION_TIMEOUT=600 +# PIP_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple +PIP_MIRROR_URL= + +# https://github.com/langgenius/dify-plugin-daemon/blob/main/.env.example +# Plugin storage type, local aws_s3 tencent_cos azure_blob +PLUGIN_STORAGE_TYPE=local +PLUGIN_STORAGE_LOCAL_ROOT=/app/storage +PLUGIN_WORKING_PATH=/app/storage/cwd +PLUGIN_INSTALLED_PATH=plugin +PLUGIN_PACKAGE_CACHE_PATH=plugin_packages +PLUGIN_MEDIA_CACHE_PATH=assets +# Plugin oss bucket +PLUGIN_STORAGE_OSS_BUCKET= +# Plugin oss s3 credentials +PLUGIN_S3_USE_AWS_MANAGED_IAM=false +PLUGIN_S3_ENDPOINT= +PLUGIN_S3_USE_PATH_STYLE=false +PLUGIN_AWS_ACCESS_KEY= +PLUGIN_AWS_SECRET_KEY= +PLUGIN_AWS_REGION= +# Plugin oss azure blob +PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME= +PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING= +# Plugin oss tencent cos +PLUGIN_TENCENT_COS_SECRET_KEY= +PLUGIN_TENCENT_COS_SECRET_ID= +PLUGIN_TENCENT_COS_REGION= +# Plugin oss aliyun oss +PLUGIN_ALIYUN_OSS_REGION= +PLUGIN_ALIYUN_OSS_ENDPOINT= +PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID= +PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET= +PLUGIN_ALIYUN_OSS_AUTH_VERSION=v4 +PLUGIN_ALIYUN_OSS_PATH= diff --git a/dify/code/nginx/conf.d/default.conf.template b/dify/code/nginx/conf.d/default.conf.template index a24219e..a458412 100644 --- a/dify/code/nginx/conf.d/default.conf.template +++ b/dify/code/nginx/conf.d/default.conf.template @@ -29,7 +29,7 @@ server { include proxy.conf; } - location /e { + location /e/ { proxy_pass http://plugin_daemon:5002; proxy_set_header Dify-Hook-Url $scheme://$host$request_uri; include proxy.conf; diff --git a/dify/code/nginx/docker-entrypoint.sh b/dify/code/nginx/docker-entrypoint.sh index d343cb3..763254e 100755 --- a/dify/code/nginx/docker-entrypoint.sh +++ b/dify/code/nginx/docker-entrypoint.sh @@ -1,5 +1,7 @@ #!/bin/bash +HTTPS_CONFIG='' + if [ "${NGINX_HTTPS_ENABLED}" = "true" ]; then # Check if the certificate and key files for the specified domain exist if [ -n "${CERTBOT_DOMAIN}" ] && \ @@ -20,6 +22,7 @@ if [ "${NGINX_HTTPS_ENABLED}" = "true" ]; then # Substitute the HTTPS_CONFIG in the default.conf.template with content from https.conf.template envsubst '${HTTPS_CONFIG}' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf fi +export HTTPS_CONFIG if [ "${NGINX_ENABLE_CERTBOT_CHALLENGE}" = "true" ]; then ACME_CHALLENGE_LOCATION='location /.well-known/acme-challenge/ { root /var/www/html; }' @@ -33,7 +36,7 @@ env_vars=$(printenv | cut -d= -f1 | sed 's/^/$/g' | paste -sd, -) envsubst "$env_vars" < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf envsubst "$env_vars" < /etc/nginx/proxy.conf.template > /etc/nginx/proxy.conf -envsubst < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf +envsubst "$env_vars" < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf # Start Nginx using the default entrypoint -exec nginx -g 'daemon off;' \ No newline at end of file +exec nginx -g 'daemon off;' diff --git a/dify/code/nginx/https.conf.template b/dify/code/nginx/https.conf.template index 95ea36f..296908d 100644 --- a/dify/code/nginx/https.conf.template +++ b/dify/code/nginx/https.conf.template @@ -6,4 +6,4 @@ ssl_certificate_key ${SSL_CERTIFICATE_KEY_PATH}; ssl_protocols ${NGINX_SSL_PROTOCOLS}; ssl_prefer_server_ciphers on; ssl_session_cache shared:SSL:10m; -ssl_session_timeout 10m; \ No newline at end of file +ssl_session_timeout 10m; diff --git a/dify/code/nginx/nginx.conf.template b/dify/code/nginx/nginx.conf.template index 32a5716..20446fa 100644 --- a/dify/code/nginx/nginx.conf.template +++ b/dify/code/nginx/nginx.conf.template @@ -31,4 +31,4 @@ http { client_max_body_size ${NGINX_CLIENT_MAX_BODY_SIZE}; include /etc/nginx/conf.d/*.conf; -} \ No newline at end of file +} diff --git a/dify/code/pgvector/docker-entrypoint.sh b/dify/code/pgvector/docker-entrypoint.sh new file mode 100755 index 0000000..262eacf --- /dev/null +++ b/dify/code/pgvector/docker-entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +PG_MAJOR=16 + +if [ "${PG_BIGM}" = "true" ]; then + # install pg_bigm + apt-get update + apt-get install -y curl make gcc postgresql-server-dev-${PG_MAJOR} + + curl -LO https://github.com/pgbigm/pg_bigm/archive/refs/tags/v${PG_BIGM_VERSION}.tar.gz + tar xf v${PG_BIGM_VERSION}.tar.gz + cd pg_bigm-${PG_BIGM_VERSION} || exit 1 + make USE_PGXS=1 PG_CONFIG=/usr/bin/pg_config + make USE_PGXS=1 PG_CONFIG=/usr/bin/pg_config install + + cd - || exit 1 + rm -rf v${PG_BIGM_VERSION}.tar.gz pg_bigm-${PG_BIGM_VERSION} + + # enable pg_bigm + sed -i -e 's/^#\s*shared_preload_libraries.*/shared_preload_libraries = '\''pg_bigm'\''/' /var/lib/postgresql/data/pgdata/postgresql.conf +fi + +# Run the original entrypoint script +exec /usr/local/bin/docker-entrypoint.sh postgres diff --git a/dify/code/ssrf_proxy/squid.conf.template b/dify/code/ssrf_proxy/squid.conf.template index 676fe73..1775a1f 100644 --- a/dify/code/ssrf_proxy/squid.conf.template +++ b/dify/code/ssrf_proxy/squid.conf.template @@ -19,6 +19,8 @@ acl Safe_ports port 488 # gss-http acl Safe_ports port 591 # filemaker acl Safe_ports port 777 # multiling http acl CONNECT method CONNECT +acl allowed_domains dstdomain .marketplace.dify.ai +http_access allow allowed_domains http_access deny !Safe_ports http_access deny CONNECT !SSL_ports http_access allow localhost manager @@ -42,10 +44,13 @@ refresh_pattern . 0 20% 4320 # cache_dir ufs /var/spool/squid 100 16 256 # upstream proxy, set to your own upstream proxy IP to avoid SSRF attacks -# cache_peer 172.1.1.1 parent 3128 0 no-query no-digest no-netdb-exchange default +# cache_peer 172.1.1.1 parent 3128 0 no-query no-digest no-netdb-exchange default ################################## Reverse Proxy To Sandbox ################################ http_port ${REVERSE_PROXY_PORT} accel vhost cache_peer ${SANDBOX_HOST} parent ${SANDBOX_PORT} 0 no-query originserver acl src_all src all http_access allow src_all + +# Unless the option's size is increased, an error will occur when uploading more than two files. +client_request_buffer_max_size 100 MB diff --git a/dify/code/startupscripts/init.sh b/dify/code/startupscripts/init.sh index c6e6e19..dcee1e1 100755 --- a/dify/code/startupscripts/init.sh +++ b/dify/code/startupscripts/init.sh @@ -8,6 +8,6 @@ if [ -f ${DB_INITIALIZED} ]; then exit else echo 'File does not exist. Standards for first time Start up this DB' - "$ORACLE_HOME"/bin/sqlplus -s "/ as sysdba" @"/opt/oracle/scripts/startup/init_user.script"; + "$ORACLE_HOME"/bin/sqlplus -s "/ as sysdba" @"/opt/oracle/scripts/startup/init_user.script"; touch ${DB_INITIALIZED} fi diff --git a/dify/code/startupscripts/init_user.script b/dify/code/startupscripts/init_user.script index 7aa7c28..e710d82 100755 --- a/dify/code/startupscripts/init_user.script +++ b/dify/code/startupscripts/init_user.script @@ -1,10 +1,10 @@ show pdbs; -ALTER SYSTEM SET PROCESSES=500 SCOPE=SPFILE; +ALTER SYSTEM SET PROCESSES=500 SCOPE=SPFILE; alter session set container= freepdb1; create user dify identified by dify DEFAULT TABLESPACE users quota unlimited on users; grant DB_DEVELOPER_ROLE to dify; BEGIN -CTX_DDL.CREATE_PREFERENCE('my_chinese_vgram_lexer','CHINESE_VGRAM_LEXER'); +CTX_DDL.CREATE_PREFERENCE('dify.world_lexer','WORLD_LEXER'); END; / diff --git a/dify/code/tidb/config/pd.toml b/dify/code/tidb/config/pd.toml index 042b251..01e352a 100644 --- a/dify/code/tidb/config/pd.toml +++ b/dify/code/tidb/config/pd.toml @@ -1,4 +1,4 @@ # PD Configuration File reference: # https://docs.pingcap.com/tidb/stable/pd-configuration-file#pd-configuration-file [replication] -max-replicas = 1 \ No newline at end of file +max-replicas = 1 diff --git a/dify/code/volumes/myscale/config/users.d/custom_users_config.xml b/dify/code/volumes/myscale/config/users.d/custom_users_config.xml index 67f24b6..b46e73a 100644 --- a/dify/code/volumes/myscale/config/users.d/custom_users_config.xml +++ b/dify/code/volumes/myscale/config/users.d/custom_users_config.xml @@ -14,4 +14,4 @@ 1 - \ No newline at end of file + diff --git a/dify/code/volumes/oceanbase/init.d/vec_memory.sql b/dify/code/volumes/oceanbase/init.d/vec_memory.sql index f4c283f..0d859e5 100644 --- a/dify/code/volumes/oceanbase/init.d/vec_memory.sql +++ b/dify/code/volumes/oceanbase/init.d/vec_memory.sql @@ -1 +1 @@ -ALTER SYSTEM SET ob_vector_memory_limit_percentage = 30; \ No newline at end of file +ALTER SYSTEM SET ob_vector_memory_limit_percentage = 30; diff --git a/plane/code/.env.example b/plane/code/.env.example index 78914c3..78031a4 100644 --- a/plane/code/.env.example +++ b/plane/code/.env.example @@ -5,12 +5,13 @@ WEB_REPLICAS=1 SPACE_REPLICAS=1 ADMIN_REPLICAS=1 API_REPLICAS=1 +WORKER_REPLICAS=1 +BEAT_WORKER_REPLICAS=1 +LIVE_REPLICAS=1 NGINX_PORT=80 WEB_URL=http://${APP_DOMAIN} DEBUG=0 -SENTRY_DSN= -SENTRY_ENVIRONMENT=production CORS_ALLOWED_ORIGINS=http://${APP_DOMAIN} API_BASE_URL=http://api:8000 @@ -55,3 +56,8 @@ GUNICORN_WORKERS=1 # UNCOMMENT `DOCKER_PLATFORM` IF YOU ARE ON `ARM64` AND DOCKER IMAGE IS NOT AVAILABLE FOR RESPECTIVE `APP_RELEASE` # DOCKER_PLATFORM=linux/amd64 +# Force HTTPS for handling SSL Termination +MINIO_ENDPOINT_SSL=0 + +# API key rate limit +API_KEY_RATE_LIMIT=60/minute diff --git a/plane/code/README.md b/plane/code/README.md index ccd8bf3..e5a8089 100644 --- a/plane/code/README.md +++ b/plane/code/README.md @@ -55,18 +55,30 @@ Installing plane is a very easy and minimal step process. - User context used must have access to docker services. In most cases, use sudo su to switch as root user - Use the terminal (or gitbash) window to run all the future steps -### Downloading Latest Stable Release +### Downloading Latest Release ``` mkdir plane-selfhost cd plane-selfhost +``` +#### For *Docker Compose* based setup + +``` curl -fsSL -o setup.sh https://github.com/makeplane/plane/releases/latest/download/setup.sh chmod +x setup.sh ``` +#### For *Docker Swarm* based setup + +``` +curl -fsSL -o setup.sh https://github.com/makeplane/plane/releases/latest/download/swarm.sh + +chmod +x setup.sh +``` + --- ### Proceed with setup @@ -77,8 +89,9 @@ Lets get started by running the `./setup.sh` command. This will prompt you with the below options. +#### Docker Compose ```bash -Select a Action you want to perform: +Select an Action you want to perform: 1) Install (x86_64) 2) Start 3) Stop @@ -87,17 +100,42 @@ Select a Action you want to perform: 6) View Logs 7) Backup Data 8) Exit + Action [2]: 1 ``` For the 1st time setup, type "1" as action input. -This will create a create a folder `plane-app` or `plane-app-preview` (in case of preview deployment) and will download 2 files inside that +This will create a folder `plane-app` and will download 2 files inside that - `docker-compose.yaml` - `plane.env` -Again the `options [1-8]` will be popped up and this time hit `8` to exit. +Again the `options [1-8]` will be popped up, and this time hit `8` to exit. + +#### Docker Swarm + +```bash +Select an Action you want to perform: + 1) Deploy Stack + 2) Remove Stack + 3) View Stack Status + 4) Redeploy Stack + 5) Upgrade + 6) View Logs + 7) Exit + +Action [3]: 1 +``` + +For the 1st time setup, type "1" as action input. + +This will create a create a folder `plane-app` and will download 2 files inside that + +- `docker-compose.yaml` +- `plane.env` + +Again the `options [1-7]` will be popped up, and this time hit `7` to exit. --- @@ -116,7 +154,7 @@ There are many other settings you can play with, but we suggest you configure `E --- -### Continue with setup - Start Server +### Continue with setup - Start Server (Docker Compose) Lets again run the `./setup.sh` command. You will again be prompted with the below options. This time select `2` to start the sevices @@ -147,9 +185,11 @@ You have successfully self hosted `Plane` instance. Access the application by go --- -### Stopping the Server +### Stopping the Server / Remove Stack -In case you want to make changes to `.env` variables, we suggest you to stop the services before doing that. +In case you want to make changes to `plane.env` variables, we suggest you to stop the services before doing that. + +#### Docker Compose Lets again run the `./setup.sh` command. You will again be prompted with the below options. This time select `3` to stop the sevices @@ -171,14 +211,34 @@ If all goes well, you must see something like this ![Stop Services](images/stopped.png) +#### Docker Swarm + +Lets again run the `./setup.sh` command. You will again be prompted with the below options. This time select `2` to stop the sevices + +```bash +Select an Action you want to perform: + 1) Deploy Stack + 2) Remove Stack + 3) View Stack Status + 4) Redeploy Stack + 5) Upgrade + 6) View Logs + 7) Exit + +Action [3]: 2 +``` + +If all goes well, you will see the confirmation from docker cli + --- -### Restarting the Server +### Restarting the Server / Redeploy Stack -In case you want to make changes to `.env` variables, without stopping the server or you noticed some abnormalies in services, you can restart the services with RESTART option. +In case you want to make changes to `plane.env` variables, without stopping the server or you noticed some abnormalies in services, you can restart the services with `RESTART` / `REDEPLOY` option. Lets again run the `./setup.sh` command. You will again be prompted with the below options. This time select `4` to restart the sevices +#### Docker Compose ```bash Select a Action you want to perform: 1) Install (x86_64) @@ -197,14 +257,32 @@ If all goes well, you must see something like this ![Restart Services](images/restart.png) +#### Docker Swarm + +```bash + 1) Deploy Stack + 2) Remove Stack + 3) View Stack Status + 4) Redeploy Stack + 5) Upgrade + 6) View Logs + 7) Exit + +Action [3]: 4 +``` + +If all goes well, you will see the confirmation from docker cli + --- -### Upgrading Plane Version +### Upgrading Plane Version It is always advised to keep Plane up to date with the latest release. Lets again run the `./setup.sh` command. You will again be prompted with the below options. This time select `5` to upgrade the release. +#### Docker Compose + ```bash Select a Action you want to perform: 1) Install (x86_64) @@ -231,13 +309,41 @@ Once done, choose `8` to exit from prompt. Once done with making changes in `plane.env` file, jump on to `Start Server` +#### Docker Swarm + +Lets again run the `./setup.sh` command. You will again be prompted with the below options. This time select `5` to upgrade the release. + +```bash + 1) Deploy Stack + 2) Remove Stack + 3) View Stack Status + 4) Redeploy Stack + 5) Upgrade + 6) View Logs + 7) Exit + +Action [3]: 5 +``` + +By choosing this, it will stop the services and then will download the latest `docker-compose.yaml` and `plane.env`. + +Once done, choose `7` to exit from prompt. + +> It is very important for you to validate the `plane.env` for the new changes. + +Once done with making changes in `plane.env` file, jump on to `Redeploy Stack` + --- ### View Logs There would a time when you might want to check what is happening inside the API, Worker or any other container. -Lets again run the `./setup.sh` command. You will again be prompted with the below options. This time select `6` to view logs. +Lets again run the `./setup.sh` command. You will again be prompted with the below options. + +This time select `6` to view logs. + +#### Docker Compose ```bash Select a Action you want to perform: @@ -253,7 +359,22 @@ Select a Action you want to perform: Action [2]: 6 ``` +#### Docker Swarm + +```bash + 1) Deploy Stack + 2) Remove Stack + 3) View Stack Status + 4) Redeploy Stack + 5) Upgrade + 6) View Logs + 7) Exit + +Action [3]: 6 +``` + +#### Service Menu Options for Logs This will further open sub-menu with list of services ```bash Select a Service you want to view the logs for: @@ -267,9 +388,10 @@ Select a Service you want to view the logs for: 8) Redis 9) Postgres 10) Minio + 11) RabbitMQ 0) Back to Main Menu -Service: +Service: 3 ``` Select any of the service to view the logs e.g. `3`. Expect something similar to this @@ -323,7 +445,7 @@ Similarly, you can view the logs of other services. --- -### Backup Data +### Backup Data (Docker Compose) There would a time when you might want to backup your data from docker volumes to external storage like S3 or drives. @@ -355,7 +477,7 @@ Backup completed successfully. Backup files are stored in /....../plane-app/back --- -### Restore Data +### Restore Data (Docker Compose) When you want to restore the previously backed-up data, follow the instructions below. diff --git a/plane/code/docker-compose.yml b/plane/code/docker-compose.yml index 1406aad..65fb84e 100644 --- a/plane/code/docker-compose.yml +++ b/plane/code/docker-compose.yml @@ -44,78 +44,70 @@ x-live-env: &live-env x-app-env: &app-env WEB_URL: ${WEB_URL:-http://localhost} DEBUG: ${DEBUG:-0} - SENTRY_DSN: ${SENTRY_DSN} - SENTRY_ENVIRONMENT: ${SENTRY_ENVIRONMENT:-production} CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS} GUNICORN_WORKERS: 1 USE_MINIO: ${USE_MINIO:-1} DATABASE_URL: ${DATABASE_URL:-postgresql://plane:plane@plane-db/plane} SECRET_KEY: ${SECRET_KEY:-60gp0byfz2dvffa45cxl20p1scy9xbpf6d8c5y0geejgkyp1b5} - ADMIN_BASE_URL: ${ADMIN_BASE_URL} - SPACE_BASE_URL: ${SPACE_BASE_URL} - APP_BASE_URL: ${APP_BASE_URL} AMQP_URL: ${AMQP_URL:-amqp://plane:plane@plane-mq:5672/plane} + API_KEY_RATE_LIMIT: ${API_KEY_RATE_LIMIT:-60/minute} + MINIO_ENDPOINT_SSL: ${MINIO_ENDPOINT_SSL:-0} services: web: - image: ${DOCKERHUB_USER:-makeplane}/plane-frontend:${APP_RELEASE:-stable} - platform: ${DOCKER_PLATFORM:-} - pull_policy: if_not_present - restart: unless-stopped + image: artifacts.plane.so/makeplane/plane-frontend:${APP_RELEASE:-stable} command: node web/server.js web deploy: replicas: ${WEB_REPLICAS:-1} + restart_policy: + condition: on-failure depends_on: - api - worker space: - image: ${DOCKERHUB_USER:-makeplane}/plane-space:${APP_RELEASE:-stable} - platform: ${DOCKER_PLATFORM:-} - pull_policy: if_not_present - restart: unless-stopped + image: artifacts.plane.so/makeplane/plane-space:${APP_RELEASE:-stable} command: node space/server.js space deploy: replicas: ${SPACE_REPLICAS:-1} + restart_policy: + condition: on-failure depends_on: - api - worker - web admin: - image: ${DOCKERHUB_USER:-makeplane}/plane-admin:${APP_RELEASE:-stable} - platform: ${DOCKER_PLATFORM:-} - pull_policy: if_not_present - restart: unless-stopped + image: artifacts.plane.so/makeplane/plane-admin:${APP_RELEASE:-stable} command: node admin/server.js admin deploy: replicas: ${ADMIN_REPLICAS:-1} + restart_policy: + condition: on-failure depends_on: - api - web live: - image: ${DOCKERHUB_USER:-makeplane}/plane-live:${APP_RELEASE:-stable} - platform: ${DOCKER_PLATFORM:-} - pull_policy: if_not_present - restart: unless-stopped + image: artifacts.plane.so/makeplane/plane-live:${APP_RELEASE:-stable} command: node live/dist/server.js live environment: <<: [ *live-env ] deploy: replicas: ${LIVE_REPLICAS:-1} + restart_policy: + condition: on-failure depends_on: - api - web api: - image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-stable} - platform: ${DOCKER_PLATFORM:-} - pull_policy: if_not_present - restart: unless-stopped + image: artifacts.plane.so/makeplane/plane-backend:${APP_RELEASE:-stable} command: ./bin/docker-entrypoint-api.sh deploy: replicas: ${API_REPLICAS:-1} + restart_policy: + condition: on-failure volumes: - logs_api:/code/plane/logs environment: @@ -126,11 +118,12 @@ services: - plane-mq worker: - image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-stable} - platform: ${DOCKER_PLATFORM:-} - pull_policy: if_not_present - restart: unless-stopped + image: artifacts.plane.so/makeplane/plane-backend:${APP_RELEASE:-stable} command: ./bin/docker-entrypoint-worker.sh + deploy: + replicas: ${WORKER_REPLICAS:-1} + restart_policy: + condition: on-failure volumes: - logs_worker:/code/plane/logs environment: @@ -142,11 +135,12 @@ services: - plane-mq beat-worker: - image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-stable} - platform: ${DOCKER_PLATFORM:-} - pull_policy: if_not_present - restart: unless-stopped + image: artifacts.plane.so/makeplane/plane-backend:${APP_RELEASE:-stable} command: ./bin/docker-entrypoint-beat.sh + deploy: + replicas: ${BEAT_WORKER_REPLICAS:-1} + restart_policy: + condition: on-failure volumes: - logs_beat-worker:/code/plane/logs environment: @@ -158,11 +152,12 @@ services: - plane-mq migrator: - image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-stable} - platform: ${DOCKER_PLATFORM:-} - pull_policy: if_not_present - restart: "no" + image: artifacts.plane.so/makeplane/plane-backend:${APP_RELEASE:-stable} command: ./bin/docker-entrypoint-migrator.sh + deploy: + replicas: 1 + restart_policy: + condition: on-failure volumes: - logs_migrator:/code/plane/logs environment: @@ -171,11 +166,14 @@ services: - plane-db - plane-redis + # Comment this if you already have a database running plane-db: image: postgres:15.7-alpine - pull_policy: if_not_present - restart: unless-stopped command: postgres -c 'max_connections=1000' + deploy: + replicas: 1 + restart_policy: + condition: on-failure environment: <<: *db-env volumes: @@ -183,24 +181,32 @@ services: plane-redis: image: valkey/valkey:7.2.5-alpine - pull_policy: if_not_present - restart: unless-stopped + deploy: + replicas: 1 + restart_policy: + condition: on-failure volumes: - redisdata:/data plane-mq: image: rabbitmq:3.13.6-management-alpine - restart: always + deploy: + replicas: 1 + restart_policy: + condition: on-failure environment: <<: *mq-env volumes: - rabbitmq_data:/var/lib/rabbitmq + # Comment this if you using any external s3 compatible storage plane-minio: image: minio/minio:latest - pull_policy: if_not_present - restart: unless-stopped command: server /export --console-address ":9090" + deploy: + replicas: 1 + restart_policy: + condition: on-failure environment: <<: *minio-env volumes: @@ -208,12 +214,13 @@ services: # Comment this if you already have a reverse proxy running proxy: - image: ${DOCKERHUB_USER:-makeplane}/plane-proxy:${APP_RELEASE:-stable} - platform: ${DOCKER_PLATFORM:-} - pull_policy: if_not_present - restart: unless-stopped + image: artifacts.plane.so/makeplane/plane-proxy:${APP_RELEASE:-stable} environment: <<: *proxy-env + deploy: + replicas: 1 + restart_policy: + condition: on-failure depends_on: - web - api @@ -222,7 +229,6 @@ services: volumes: pgdata: redisdata: - uploads: logs_api: logs_worker: diff --git a/plane/code/install.sh b/plane/code/install.sh index 1c2208c..9f0065f 100755 --- a/plane/code/install.sh +++ b/plane/code/install.sh @@ -5,7 +5,7 @@ SCRIPT_DIR=$PWD SERVICE_FOLDER=plane-app PLANE_INSTALL_DIR=$PWD/$SERVICE_FOLDER export APP_RELEASE=stable -export DOCKERHUB_USER=makeplane +export DOCKERHUB_USER=artifacts.plane.so/makeplane export PULL_POLICY=${PULL_POLICY:-if_not_present} export GH_REPO=makeplane/plane export RELEASE_DOWNLOAD_URL="https://github.com/$GH_REPO/releases/download" @@ -366,7 +366,7 @@ function startServices() { local api_container_id=$(docker container ls -q -f "name=$SERVICE_FOLDER-api") local idx2=0 - while ! docker logs $api_container_id 2>&1 | grep -m 1 -i "Application startup complete" | grep -q "."; + while ! docker exec $api_container_id python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/')" > /dev/null 2>&1; do local message=">> Waiting for API Service to Start" local dots=$(printf '%*s' $idx2 | tr ' ' '.') @@ -457,12 +457,13 @@ function viewLogs(){ echo " 8) Redis" echo " 9) Postgres" echo " 10) Minio" + echo " 11) RabbitMQ" echo " 0) Back to Main Menu" echo read -p "Service: " DOCKER_SERVICE_NAME - until (( DOCKER_SERVICE_NAME >= 0 && DOCKER_SERVICE_NAME <= 10 )); do - echo "Invalid selection. Please enter a number between 1 and 11." + until (( DOCKER_SERVICE_NAME >= 0 && DOCKER_SERVICE_NAME <= 11 )); do + echo "Invalid selection. Please enter a number between 0 and 11." read -p "Service: " DOCKER_SERVICE_NAME done @@ -481,6 +482,7 @@ function viewLogs(){ 8) viewSpecificLogs "plane-redis";; 9) viewSpecificLogs "plane-db";; 10) viewSpecificLogs "plane-minio";; + 11) viewSpecificLogs "plane-mq";; 0) askForAction;; *) echo "INVALID SERVICE NAME SUPPLIED";; esac @@ -499,49 +501,76 @@ function viewLogs(){ redis) viewSpecificLogs "plane-redis";; postgres) viewSpecificLogs "plane-db";; minio) viewSpecificLogs "plane-minio";; + rabbitmq) viewSpecificLogs "plane-mq";; *) echo "INVALID SERVICE NAME SUPPLIED";; esac else echo "INVALID SERVICE NAME SUPPLIED" fi } -function backupSingleVolume() { - backupFolder=$1 - selectedVolume=$2 - # Backup data from Docker volume to the backup folder - # docker run --rm -v "$selectedVolume":/source -v "$backupFolder":/backup busybox sh -c 'cp -r /source/* /backup/' - local tobereplaced="plane-app_" - local replacewith="" +function backup_container_dir() { + local BACKUP_FOLDER=$1 + local CONTAINER_NAME=$2 + local CONTAINER_DATA_DIR=$3 + local SERVICE_FOLDER=$4 - local svcName="${selectedVolume//$tobereplaced/$replacewith}" + echo "Backing up $CONTAINER_NAME data..." + local CONTAINER_ID=$(/bin/bash -c "$COMPOSE_CMD -f $DOCKER_FILE_PATH ps -q $CONTAINER_NAME") + if [ -z "$CONTAINER_ID" ]; then + echo "Error: $CONTAINER_NAME container not found. Make sure the services are running." + return 1 + fi - docker run --rm \ - -e TAR_NAME="$svcName" \ - -v "$selectedVolume":/"$svcName" \ - -v "$backupFolder":/backup \ - busybox sh -c 'tar -czf "/backup/${TAR_NAME}.tar.gz" /${TAR_NAME}' + # Create a temporary directory for the backup + mkdir -p "$BACKUP_FOLDER/$SERVICE_FOLDER" + + # Copy the data directory from the running container + echo "Copying $CONTAINER_NAME data directory..." + docker cp -q "$CONTAINER_ID:$CONTAINER_DATA_DIR/." "$BACKUP_FOLDER/$SERVICE_FOLDER/" + local cp_status=$? + + if [ $cp_status -ne 0 ]; then + echo "Error: Failed to copy $SERVICE_FOLDER data" + rm -rf $BACKUP_FOLDER/$SERVICE_FOLDER + return 1 + fi + + # Create tar.gz of the data + cd "$BACKUP_FOLDER" + tar -czf "${SERVICE_FOLDER}.tar.gz" "$SERVICE_FOLDER/" + local tar_status=$? + if [ $tar_status -eq 0 ]; then + rm -rf "$SERVICE_FOLDER/" + fi + cd - > /dev/null + + if [ $tar_status -ne 0 ]; then + echo "Error: Failed to create tar archive" + return 1 + fi + + echo "Successfully backed up $SERVICE_FOLDER data" } + function backupData() { local datetime=$(date +"%Y%m%d-%H%M") local BACKUP_FOLDER=$PLANE_INSTALL_DIR/backup/$datetime mkdir -p "$BACKUP_FOLDER" - volumes=$(docker volume ls -f "name=$SERVICE_FOLDER" --format "{{.Name}}" | grep -E "_pgdata|_redisdata|_uploads") - # Check if there are any matching volumes - if [ -z "$volumes" ]; then - echo "No volumes found starting with '$SERVICE_FOLDER'" + # Check if docker-compose.yml exists + if [ ! -f "$DOCKER_FILE_PATH" ]; then + echo "Error: docker-compose.yml not found at $DOCKER_FILE_PATH" exit 1 fi - for vol in $volumes; do - echo "Backing Up $vol" - backupSingleVolume "$BACKUP_FOLDER" "$vol" - done + backup_container_dir "$BACKUP_FOLDER" "plane-db" "/var/lib/postgresql/data" "pgdata" || exit 1 + backup_container_dir "$BACKUP_FOLDER" "plane-minio" "/export" "uploads" || exit 1 + backup_container_dir "$BACKUP_FOLDER" "plane-mq" "/var/lib/rabbitmq" "rabbitmq_data" || exit 1 + backup_container_dir "$BACKUP_FOLDER" "plane-redis" "/data" "redisdata" || exit 1 echo "" echo "Backup completed successfully. Backup files are stored in $BACKUP_FOLDER" echo "" - } function askForAction() { local DEFAULT_ACTION=$1 @@ -628,7 +657,7 @@ if [ -f "$DOCKER_ENV_PATH" ]; then CUSTOM_BUILD=$(getEnvValue "CUSTOM_BUILD" "$DOCKER_ENV_PATH") if [ -z "$DOCKERHUB_USER" ]; then - DOCKERHUB_USER=makeplane + DOCKERHUB_USER=artifacts.plane.so/makeplane updateEnvFile "DOCKERHUB_USER" "$DOCKERHUB_USER" "$DOCKER_ENV_PATH" fi diff --git a/plane/code/restore.sh b/plane/code/restore.sh index cd453b2..23b8de6 100755 --- a/plane/code/restore.sh +++ b/plane/code/restore.sh @@ -66,8 +66,10 @@ function restoreData() { exit 1 fi + local volume_suffix + volume_suffix="_pgdata|_redisdata|_uploads|_rabbitmq_data" local volumes - volumes=$(docker volume ls -f "name=plane-app" --format "{{.Name}}" | grep -E "_pgdata|_redisdata|_uploads") + volumes=$(docker volume ls -f "name=plane-app" --format "{{.Name}}" | grep -E "$volume_suffix") # Check if there are any matching volumes if [ -z "$volumes" ]; then echo ".....No volumes found starting with 'plane-app'" @@ -87,7 +89,7 @@ function restoreData() { echo "Found $BACKUP_FILE" local docVol - docVol=$(docker volume ls -f "name=$restoreVolName" --format "{{.Name}}" | grep -E "_pgdata|_redisdata|_uploads") + docVol=$(docker volume ls -f "name=$restoreVolName" --format "{{.Name}}" | grep -E "$volume_suffix") if [ -z "$docVol" ]; then echo "Skipping: No volume found with name $restoreVolName" diff --git a/plane/code/swarm.sh b/plane/code/swarm.sh new file mode 100755 index 0000000..c58f05e --- /dev/null +++ b/plane/code/swarm.sh @@ -0,0 +1,612 @@ +#!/bin/bash + +BRANCH=${BRANCH:-master} +SERVICE_FOLDER=plane-app +SCRIPT_DIR=$PWD +PLANE_INSTALL_DIR=$PWD/$SERVICE_FOLDER +export APP_RELEASE="stable" +export DOCKERHUB_USER=artifacts.plane.so/makeplane + +export GH_REPO=makeplane/plane +export RELEASE_DOWNLOAD_URL="https://github.com/$GH_REPO/releases/download" +export FALLBACK_DOWNLOAD_URL="https://raw.githubusercontent.com/$GH_REPO/$BRANCH/deploy/selfhost" + +OS_NAME=$(uname) + +# Create necessary directories +mkdir -p $PLANE_INSTALL_DIR/archive + +DOCKER_FILE_PATH=$PLANE_INSTALL_DIR/docker-compose.yml +DOCKER_ENV_PATH=$PLANE_INSTALL_DIR/plane.env + +function print_header() { +clear + +cat <<"EOF" +-------------------------------------------- + ____ _ ///////// +| _ \| | __ _ _ __ ___ ///////// +| |_) | |/ _` | '_ \ / _ \ ///// ///// +| __/| | (_| | | | | __/ ///// ///// +|_| |_|\__,_|_| |_|\___| //// + //// +-------------------------------------------- +Project management tool from the future +-------------------------------------------- +EOF +} + +function checkLatestRelease(){ + echo "Checking for the latest release..." >&2 + local latest_release=$(curl -s https://api.github.com/repos/$GH_REPO/releases/latest | grep -o '"tag_name": "[^"]*"' | sed 's/"tag_name": "//;s/"//g') + if [ -z "$latest_release" ]; then + echo "Failed to check for the latest release. Exiting..." >&2 + exit 1 + fi + + echo $latest_release +} + +# Function to read stack name from env file +function readStackName() { + if [ -f "$DOCKER_ENV_PATH" ]; then + local saved_stack_name=$(grep "^STACK_NAME=" "$DOCKER_ENV_PATH" | cut -d'=' -f2) + if [ -n "$saved_stack_name" ]; then + stack_name=$saved_stack_name + return 1 + fi + fi + return 0 +} + +# Function to get stack name (either from env or user input) +function getStackName() { + read -p "Enter stack name [plane]: " input_stack_name + if [ -z "$input_stack_name" ]; then + input_stack_name="plane" + fi + stack_name=$input_stack_name + updateEnvFile "STACK_NAME" "$stack_name" "$DOCKER_ENV_PATH" + echo "Using stack name: $stack_name" +} + +function syncEnvFile(){ + echo "Syncing environment variables..." >&2 + if [ -f "$PLANE_INSTALL_DIR/plane.env.bak" ]; then + # READ keys of plane.env and update the values from plane.env.bak + while IFS= read -r line + do + # ignore if the line is empty or starts with # + if [ -z "$line" ] || [[ $line == \#* ]]; then + continue + fi + key=$(echo "$line" | cut -d'=' -f1) + value=$(getEnvValue "$key" "$PLANE_INSTALL_DIR/plane.env.bak") + if [ -n "$value" ]; then + updateEnvFile "$key" "$value" "$DOCKER_ENV_PATH" + fi + done < "$DOCKER_ENV_PATH" + + value=$(getEnvValue "STACK_NAME" "$PLANE_INSTALL_DIR/plane.env.bak") + if [ -n "$value" ]; then + updateEnvFile "STACK_NAME" "$value" "$DOCKER_ENV_PATH" + fi + fi + echo "Environment variables synced successfully" >&2 + rm -f $PLANE_INSTALL_DIR/plane.env.bak +} + +function getEnvValue() { + local key=$1 + local file=$2 + + if [ -z "$key" ] || [ -z "$file" ]; then + echo "Invalid arguments supplied" + exit 1 + fi + + if [ -f "$file" ]; then + grep -q "^$key=" "$file" + if [ $? -eq 0 ]; then + local value + value=$(grep "^$key=" "$file" | cut -d'=' -f2) + echo "$value" + else + echo "" + fi + fi +} + +function updateEnvFile() { + local key=$1 + local value=$2 + local file=$3 + + if [ -z "$key" ] || [ -z "$value" ] || [ -z "$file" ]; then + echo "Invalid arguments supplied" + exit 1 + fi + + if [ -f "$file" ]; then + # check if key exists in the file + grep -q "^$key=" "$file" + if [ $? -ne 0 ]; then + echo "$key=$value" >> "$file" + return + else + if [ "$OS_NAME" == "Darwin" ]; then + value=$(echo "$value" | sed 's/|/\\|/g') + sed -i '' "s|^$key=.*|$key=$value|g" "$file" + else + sed -i "s/^$key=.*/$key=$value/g" "$file" + fi + fi + else + echo "File not found: $file" + exit 1 + fi +} + +function download() { + cd $SCRIPT_DIR || exit 1 + TS=$(date +%s) + if [ -f "$PLANE_INSTALL_DIR/docker-compose.yml" ] + then + mv $PLANE_INSTALL_DIR/docker-compose.yml $PLANE_INSTALL_DIR/archive/$TS.docker-compose.yml + fi + + echo $RELEASE_DOWNLOAD_URL + echo $FALLBACK_DOWNLOAD_URL + echo $APP_RELEASE + + RESPONSE=$(curl -H 'Cache-Control: no-cache, no-store' -s -w "HTTPSTATUS:%{http_code}" "$RELEASE_DOWNLOAD_URL/$APP_RELEASE/docker-compose.yml?$(date +%s)") + BODY=$(echo "$RESPONSE" | sed -e 's/HTTPSTATUS\:.*//g') + STATUS=$(echo "$RESPONSE" | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') + + if [ "$STATUS" -eq 200 ]; then + echo "$BODY" > $PLANE_INSTALL_DIR/docker-compose.yml + else + # Fallback to download from the raw github url + RESPONSE=$(curl -H 'Cache-Control: no-cache, no-store' -s -w "HTTPSTATUS:%{http_code}" "$FALLBACK_DOWNLOAD_URL/docker-compose.yml?$(date +%s)") + BODY=$(echo "$RESPONSE" | sed -e 's/HTTPSTATUS\:.*//g') + STATUS=$(echo "$RESPONSE" | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') + + if [ "$STATUS" -eq 200 ]; then + echo "$BODY" > $PLANE_INSTALL_DIR/docker-compose.yml + else + echo "Failed to download docker-compose.yml. HTTP Status: $STATUS" + echo "URL: $RELEASE_DOWNLOAD_URL/$APP_RELEASE/docker-compose.yml" + mv $PLANE_INSTALL_DIR/archive/$TS.docker-compose.yml $PLANE_INSTALL_DIR/docker-compose.yml + exit 1 + fi + fi + + RESPONSE=$(curl -H 'Cache-Control: no-cache, no-store' -s -w "HTTPSTATUS:%{http_code}" "$RELEASE_DOWNLOAD_URL/$APP_RELEASE/variables.env?$(date +%s)") + BODY=$(echo "$RESPONSE" | sed -e 's/HTTPSTATUS\:.*//g') + STATUS=$(echo "$RESPONSE" | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') + + if [ "$STATUS" -eq 200 ]; then + echo "$BODY" > $PLANE_INSTALL_DIR/variables-upgrade.env + else + # Fallback to download from the raw github url + RESPONSE=$(curl -H 'Cache-Control: no-cache, no-store' -s -w "HTTPSTATUS:%{http_code}" "$FALLBACK_DOWNLOAD_URL/variables.env?$(date +%s)") + BODY=$(echo "$RESPONSE" | sed -e 's/HTTPSTATUS\:.*//g') + STATUS=$(echo "$RESPONSE" | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') + + if [ "$STATUS" -eq 200 ]; then + echo "$BODY" > $PLANE_INSTALL_DIR/variables-upgrade.env + else + echo "Failed to download variables.env. HTTP Status: $STATUS" + echo "URL: $RELEASE_DOWNLOAD_URL/$APP_RELEASE/variables.env" + mv $PLANE_INSTALL_DIR/archive/$TS.docker-compose.yml $PLANE_INSTALL_DIR/docker-compose.yml + exit 1 + fi + fi + + if [ -f "$DOCKER_ENV_PATH" ]; + then + cp "$DOCKER_ENV_PATH" "$PLANE_INSTALL_DIR/archive/$TS.env" + cp "$DOCKER_ENV_PATH" "$PLANE_INSTALL_DIR/plane.env.bak" + fi + + mv $PLANE_INSTALL_DIR/variables-upgrade.env $DOCKER_ENV_PATH + + syncEnvFile + + updateEnvFile "APP_RELEASE" "$APP_RELEASE" "$DOCKER_ENV_PATH" + +} +function deployStack() { + # Check if docker compose file and env file exist + if [ ! -f "$DOCKER_FILE_PATH" ] || [ ! -f "$DOCKER_ENV_PATH" ]; then + echo "Configuration files not found" + echo "Downloading it now......" + APP_RELEASE=$(checkLatestRelease) + download + fi + if [ -z "$stack_name" ]; then + getStackName + fi + echo "Starting ${stack_name} stack..." + + # Pull envs + if [ -f "$DOCKER_ENV_PATH" ]; then + set -o allexport; source $DOCKER_ENV_PATH; set +o allexport; + else + echo "Environment file not found: $DOCKER_ENV_PATH" + exit 1 + fi + + # Deploy the stack + docker stack deploy -c $DOCKER_FILE_PATH $stack_name + + echo "Waiting for services to be deployed..." + sleep 10 + + # Check migrator service + local migrator_service=$(docker service ls --filter name=${stack_name}_migrator -q) + if [ -n "$migrator_service" ]; then + echo ">> Waiting for Data Migration to finish" + while docker service ls --filter name=${stack_name}_migrator | grep -q "running"; do + echo -n "." + sleep 1 + done + echo "" + + # Get the most recent container for the migrator service + local migrator_container=$(docker ps -a --filter name=${stack_name}_migrator --latest -q) + + if [ -n "$migrator_container" ]; then + # Get the exit code of the container + local exit_code=$(docker inspect --format='{{.State.ExitCode}}' $migrator_container) + + if [ "$exit_code" != "0" ]; then + echo "Server failed to start ❌" + echo "Migration failed with exit code: $exit_code" + echo "Please check the logs for the 'migrator' service and resolve the issue(s)." + echo "Stop the services by running the command: ./swarm.sh stop" + exit 1 + else + echo " Data Migration completed successfully ✅" + fi + else + echo "Warning: Could not find migrator container to check exit status" + fi + fi + + # Check API service + local api_service=$(docker service ls --filter name=${stack_name}_api -q) + while docker service ls --filter name=${stack_name}_api | grep -q "running"; do + local running_container=$(docker ps --filter "name=${stack_name}_api" --filter "status=running" -q) + if [ -n "$running_container" ]; then + if docker container logs $running_container 2>/dev/null | grep -q "Application Startup Complete"; then + break + fi + fi + sleep 2 + done + + if [ -z "$api_service" ]; then + echo "Plane Server failed to start ❌" + echo "Please check the logs for the 'api' service and resolve the issue(s)." + echo "Stop the services by running the command: ./swarm.sh stop" + exit 1 + fi + echo " Plane Server started successfully ✅" + echo "" + echo " You can access the application at $WEB_URL" + echo "" +} + +function removeStack() { + if [ -z "$stack_name" ]; then + echo "Stack name not found" + exit 1 + fi + echo "Removing ${stack_name} stack..." + docker stack rm "$stack_name" + echo "Waiting for services to be removed..." + while docker stack ls | grep -q "$stack_name"; do + sleep 1 + done + sleep 20 + echo "Services stopped successfully ✅" +} + +function viewStatus() { + echo "Checking status of ${stack_name} stack..." + if [ -z "$stack_name" ]; then + echo "Stack name not found" + exit 1 + fi + docker stack ps "$stack_name" +} + +function redeployStack() { + removeStack + echo "ReDeploying ${stack_name} stack..." + deployStack +} + +function upgrade() { + + echo "Checking status of ${stack_name} stack..." + if [ -z "$stack_name" ]; then + echo "Stack name not found" + exit 1 + fi + + local latest_release=$(checkLatestRelease) + + echo "" + echo "Current release: $APP_RELEASE" + + if [ "$latest_release" == "$APP_RELEASE" ]; then + echo "" + echo "You are already using the latest release" + exit 0 + fi + + echo "Latest release: $latest_release" + echo "" + + # Check for confirmation to upgrade + echo "Do you want to upgrade to the latest release ($latest_release)?" + read -p "Continue? [y/N]: " confirm + + if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + echo "Exiting..." + exit 0 + fi + + export APP_RELEASE=$latest_release + + # check if stack exists + echo "Upgrading ${stack_name} stack..." + + # check env file and take backup + if [ -f "$DOCKER_ENV_PATH" ]; then + cp "$DOCKER_ENV_PATH" "${DOCKER_ENV_PATH}.bak" + fi + + download + redeployStack +} + +function viewSpecificLogs() { + local service=$1 + + # Input validation + if [ -z "$service" ]; then + echo "Error: Please specify a service name" + return 1 + fi + + # Main loop for service logs + while true; do + # Get all running containers for the service + local running_containers=$(docker ps --filter "name=${stack_name}_${service}" --filter "status=running" -q) + + # If no running containers found, try service logs + if [ -z "$running_containers" ]; then + echo "No running containers found for ${stack_name}_${service}, checking service logs..." + if docker service inspect ${stack_name}_${service} >/dev/null 2>&1; then + echo "Press Ctrl+C or 'q' to exit logs" + docker service logs ${stack_name}_${service} -f + break + else + echo "Error: No running containers or services found for ${stack_name}_${service}" + return 1 + fi + return + fi + + # If multiple containers are running, let user choose + if [ $(echo "$running_containers" | grep -v '^$' | wc -l) -gt 1 ]; then + clear + echo "Multiple containers found for ${stack_name}_${service}:" + local i=1 + # Use regular arrays instead of associative arrays + container_ids=() + container_names=() + + while read -r container_id; do + if [ -n "$container_id" ]; then + local container_name=$(docker inspect --format '{{.Name}}' "$container_id" | sed 's/\///') + container_ids[$i]=$container_id + container_names[$i]=$container_name + echo "[$i] ${container_names[$i]} (${container_ids[$i]})" + i=$((i+1)) + fi + done <<< "$running_containers" + + echo -e "\nPlease select a container number:" + read -r selection + + if [[ "$selection" =~ ^[0-9]+$ ]] && [ -n "${container_ids[$selection]}" ]; then + local selected_container=${container_ids[$selection]} + clear + echo "Showing logs for container: ${container_names[$selection]}" + echo "Press Ctrl+C or 'q' to return to container selection" + + # Start watching logs in the background + docker container logs -f "$selected_container" & + local log_pid=$! + + while true; do + read -r -n 1 input + if [[ $input == "q" ]]; then + kill $log_pid 2>/dev/null + wait $log_pid 2>/dev/null + break + fi + done + clear + else + echo "Error: Invalid selection" + sleep 2 + fi + else + # Single container case + local container_name=$(docker inspect --format '{{.Name}}' "$running_containers" | sed 's/\///') + echo "Showing logs for container: $container_name" + echo "Press Ctrl+C or 'q' to exit logs" + docker container logs -f "$running_containers" & + local log_pid=$! + + while true; do + read -r -n 1 input + if [[ $input == "q" ]]; then + kill $log_pid 2>/dev/null + wait $log_pid 2>/dev/null + break + fi + done + break + fi + done +} + +function viewLogs(){ + + ARG_SERVICE_NAME=$2 + if [ -z "$ARG_SERVICE_NAME" ]; + then + echo + echo "Select a Service you want to view the logs for:" + echo " 1) Web" + echo " 2) Space" + echo " 3) API" + echo " 4) Worker" + echo " 5) Beat-Worker" + echo " 6) Migrator" + echo " 7) Proxy" + echo " 8) Redis" + echo " 9) Postgres" + echo " 10) Minio" + echo " 11) RabbitMQ" + echo " 0) Back to Main Menu" + echo + read -p "Service: " DOCKER_SERVICE_NAME + + until (( DOCKER_SERVICE_NAME >= 0 && DOCKER_SERVICE_NAME <= 11 )); do + echo "Invalid selection. Please enter a number between 0 and 11." + read -p "Service: " DOCKER_SERVICE_NAME + done + + if [ -z "$DOCKER_SERVICE_NAME" ]; + then + echo "INVALID SERVICE NAME SUPPLIED" + else + case $DOCKER_SERVICE_NAME in + 1) viewSpecificLogs "web";; + 2) viewSpecificLogs "space";; + 3) viewSpecificLogs "api";; + 4) viewSpecificLogs "worker";; + 5) viewSpecificLogs "beat-worker";; + 6) viewSpecificLogs "migrator";; + 7) viewSpecificLogs "proxy";; + 8) viewSpecificLogs "plane-redis";; + 9) viewSpecificLogs "plane-db";; + 10) viewSpecificLogs "plane-minio";; + 11) viewSpecificLogs "plane-mq";; + 0) askForAction;; + *) echo "INVALID SERVICE NAME SUPPLIED";; + esac + fi + elif [ -n "$ARG_SERVICE_NAME" ]; + then + ARG_SERVICE_NAME=$(echo "$ARG_SERVICE_NAME" | tr '[:upper:]' '[:lower:]') + case $ARG_SERVICE_NAME in + web) viewSpecificLogs "web";; + space) viewSpecificLogs "space";; + api) viewSpecificLogs "api";; + worker) viewSpecificLogs "worker";; + beat-worker) viewSpecificLogs "beat-worker";; + migrator) viewSpecificLogs "migrator";; + proxy) viewSpecificLogs "proxy";; + redis) viewSpecificLogs "plane-redis";; + postgres) viewSpecificLogs "plane-db";; + minio) viewSpecificLogs "plane-minio";; + rabbitmq) viewSpecificLogs "plane-mq";; + *) echo "INVALID SERVICE NAME SUPPLIED";; + esac + else + echo "INVALID SERVICE NAME SUPPLIED" + fi +} + + + +function askForAction() { + # Rest of askForAction remains the same but use $stack_name instead of $STACK_NAME + local DEFAULT_ACTION=$1 + + if [ -z "$DEFAULT_ACTION" ]; then + echo + echo "Select an Action you want to perform:" + echo " 1) Deploy Stack" + echo " 2) Remove Stack" + echo " 3) View Stack Status" + echo " 4) Redeploy Stack" + echo " 5) Upgrade" + echo " 6) View Logs" + echo " 7) Exit" + echo + read -p "Action [3]: " ACTION + until [[ -z "$ACTION" || "$ACTION" =~ ^[1-6]$ ]]; do + echo "$ACTION: invalid selection." + read -p "Action [3]: " ACTION + done + + if [ -z "$ACTION" ]; then + ACTION=3 + fi + echo + fi + + if [ "$ACTION" == "1" ] || [ "$DEFAULT_ACTION" == "deploy" ]; then + deployStack + elif [ "$ACTION" == "2" ] || [ "$DEFAULT_ACTION" == "remove" ]; then + removeStack + elif [ "$ACTION" == "3" ] || [ "$DEFAULT_ACTION" == "status" ]; then + viewStatus + elif [ "$ACTION" == "4" ] || [ "$DEFAULT_ACTION" == "redeploy" ]; then + redeployStack + elif [ "$ACTION" == "5" ] || [ "$DEFAULT_ACTION" == "upgrade" ]; then + upgrade + elif [ "$ACTION" == "6" ] || [ "$DEFAULT_ACTION" == "logs" ]; then + viewLogs "$@" + elif [ "$ACTION" == "7" ] || [ "$DEFAULT_ACTION" == "exit" ]; then + exit 0 + else + echo "INVALID ACTION SUPPLIED" + fi +} + +# Initialize stack name at script start + +if [ -z "$stack_name" ]; then + readStackName +fi + +# Sync environment variables +if [ -f "$DOCKER_ENV_PATH" ]; then + DOCKERHUB_USER=$(getEnvValue "DOCKERHUB_USER" "$DOCKER_ENV_PATH") + APP_RELEASE=$(getEnvValue "APP_RELEASE" "$DOCKER_ENV_PATH") + + if [ -z "$DOCKERHUB_USER" ]; then + DOCKERHUB_USER=artifacts.plane.so/makeplane + updateEnvFile "DOCKERHUB_USER" "$DOCKERHUB_USER" "$DOCKER_ENV_PATH" + fi + + if [ -z "$APP_RELEASE" ]; then + APP_RELEASE=stable + updateEnvFile "APP_RELEASE" "$APP_RELEASE" "$DOCKER_ENV_PATH" + fi +fi + + +# Main execution +print_header +askForAction "$@" diff --git a/supabase/code/docker-compose.yml b/supabase/code/docker-compose.yml index 9f98f0a..a8d8584 100644 --- a/supabase/code/docker-compose.yml +++ b/supabase/code/docker-compose.yml @@ -10,7 +10,7 @@ name: supabase services: studio: - image: supabase/studio:20250113-83c9420 + image: supabase/studio:2025.05.19-sha-3487831 restart: unless-stopped healthcheck: test: @@ -18,8 +18,8 @@ services: "CMD", "node", "-e", - "fetch('http://studio:3000/api/profile').then((r) => {if (r.status - !== 200) throw new Error(r.status)})" + "fetch('http://studio:3000/api/platform/profile').then((r) => {if + (r.status !== 200) throw new Error(r.status)})" ] timeout: 10s interval: 5s @@ -54,7 +54,7 @@ services: restart: unless-stopped volumes: # https://github.com/supabase/supabase/issues/12661 - - ./volumes/api/kong.yml:/home/kong/temp.yml:ro + - ./volumes/api/kong.yml:/home/kong/temp.yml:ro,z depends_on: analytics: condition: service_healthy @@ -75,7 +75,7 @@ services: /docker-entrypoint.sh kong docker-start' auth: - image: supabase/gotrue:v2.167.0 + image: supabase/gotrue:v2.172.1 restart: unless-stopped healthcheck: test: @@ -157,7 +157,7 @@ services: # GOTRUE_HOOK_SEND_EMAIL_SECRETS: "v1,whsec_VGhpcyBpcyBhbiBleGFtcGxlIG9mIGEgc2hvcnRlciBCYXNlNjQgc3RyaW5n" rest: - image: postgrest/postgrest:v12.2.0 + image: postgrest/postgrest:v12.2.12 restart: unless-stopped depends_on: db: @@ -177,7 +177,7 @@ services: realtime: # This container name looks inconsistent but is correct because realtime constructs tenant id by parsing the subdomain - image: supabase/realtime:v2.34.7 + image: supabase/realtime:v2.34.47 restart: unless-stopped depends_on: db: @@ -221,7 +221,7 @@ services: # To use S3 backed storage: docker compose -f docker-compose.yml -f docker-compose.s3.yml up storage: - image: supabase/storage-api:v1.14.5 + image: supabase/storage-api:v1.22.17 restart: unless-stopped volumes: - ./volumes/storage:/var/lib/storage:z @@ -279,7 +279,7 @@ services: IMGPROXY_ENABLE_WEBP_DETECTION: ${IMGPROXY_ENABLE_WEBP_DETECTION} meta: - image: supabase/postgres-meta:v0.84.2 + image: supabase/postgres-meta:v0.89.0 restart: unless-stopped depends_on: db: @@ -296,7 +296,7 @@ services: PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} functions: - image: supabase/edge-runtime:v1.67.0 + image: supabase/edge-runtime:v1.67.4 restart: unless-stopped volumes: - ./volumes/functions:/home/deno/functions:Z @@ -314,7 +314,7 @@ services: command: [ "start", "--main-service", "/home/deno/functions/main" ] analytics: - image: supabase/logflare:1.4.0 + image: supabase/logflare:1.12.0 restart: unless-stopped # Uncomment to use Big Query backend for analytics # volumes: @@ -354,7 +354,7 @@ services: # Comment out everything below this point if you are using an external Postgres database db: - image: supabase/postgres:15.8.1.020 + image: supabase/postgres:15.8.1.060 restart: unless-stopped volumes: - ./volumes/db/realtime.sql:/docker-entrypoint-initdb.d/migrations/99-realtime.sql:Z @@ -405,8 +405,8 @@ services: image: timberio/vector:0.28.1-alpine restart: unless-stopped volumes: - - ./volumes/logs/vector.yml:/etc/vector/vector.yml:ro - - ${DOCKER_SOCKET_LOCATION}:/var/run/docker.sock:ro + - ./volumes/logs/vector.yml:/etc/vector/vector.yml:ro,z + - ${DOCKER_SOCKET_LOCATION}:/var/run/docker.sock:ro,z healthcheck: test: [ @@ -423,13 +423,15 @@ services: environment: LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} command: [ "--config", "/etc/vector/vector.yml" ] + security_opt: + - "label=disable" # Update the DATABASE_URL if you are using an external Postgres database supavisor: - image: supabase/supavisor:1.1.56 + image: supabase/supavisor:2.5.1 restart: unless-stopped volumes: - - ./volumes/pooler/pooler.exs:/etc/pooler/pooler.exs:ro + - ./volumes/pooler/pooler.exs:/etc/pooler/pooler.exs:ro,z healthcheck: test: [ diff --git a/twenty/code/Makefile b/twenty/code/Makefile index 2d89c4c..bdda023 100644 --- a/twenty/code/Makefile +++ b/twenty/code/Makefile @@ -1,14 +1,27 @@ +# Makefile for building Twenty CRM docker images. +# Set the tag and/or target build platform using make command-line variables assignments. +# +# Optional make variables: +# PLATFORM - defaults to 'linux/amd64' +# TAG - defaults to 'latest' +# +# Example: make +# Example: make PLATFORM=linux/aarch64 TAG=my-tag + +PLATFORM ?= linux/amd64 +TAG ?= latest + prod-build: - @cd ../.. && docker build -f ./packages/twenty-docker/twenty/Dockerfile --tag twenty . && cd - + @cd ../.. && docker build -f ./packages/twenty-docker/twenty/Dockerfile --platform $(PLATFORM) --tag twenty:$(TAG) . && cd - prod-run: - @docker run -d -p 3000:3000 --name twenty twenty + @docker run -d -p 3000:3000 --name twenty twenty:$(TAG) prod-postgres-run: - @docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres --name twenty-postgres twenty-postgres + @docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres --name twenty-postgres twenty-postgres:$(TAG) prod-website-build: - @cd ../.. && docker build -f ./packages/twenty-docker/twenty-website/Dockerfile --tag twenty-website . && cd - + @cd ../.. && docker build -f ./packages/twenty-docker/twenty-website/Dockerfile --platform $(PLATFORM) --tag twenty-website:$(TAG) . && cd - prod-website-run: - @docker run -d -p 3000:3000 --name twenty-website twenty-website + @docker run -d -p 3000:3000 --name twenty-website twenty-website:$(TAG) diff --git a/twenty/code/docker-compose.yml b/twenty/code/docker-compose.yml index 7ea2bd6..2c0e188 100644 --- a/twenty/code/docker-compose.yml +++ b/twenty/code/docker-compose.yml @@ -1,21 +1,10 @@ name: twenty services: - change-vol-ownership: - image: ubuntu - user: root - volumes: - - server-local-data:/tmp/server-local-data - - docker-data:/tmp/docker-data - command: > - bash -c " chown -R 1000:1000 /tmp/server-local-data && chown -R 1000:1000 - /tmp/docker-data" - server: image: twentycrm/twenty:${TAG:-latest} volumes: - - server-local-data:/app/packages/twenty-server/${STORAGE_LOCAL_PATH:-.local-storage} - - docker-data:/app/docker-data + - server-local-data:/app/packages/twenty-server/.local-storage environment: NODE_PORT: 3000 PG_DATABASE_URL: postgres://${PG_DATABASE_USER:-postgres}:${PG_DATABASE_PASSWORD:-postgres}@${PG_DATABASE_HOST:-db}:${PG_DATABASE_PORT:-5432}/default @@ -27,21 +16,45 @@ services: STORAGE_S3_NAME: ${STORAGE_S3_NAME} STORAGE_S3_ENDPOINT: ${STORAGE_S3_ENDPOINT} - APP_SECRET: ${APP_SECRET} + APP_SECRET: ${APP_SECRET:-replace_me_with_a_random_string} + # MESSAGING_PROVIDER_GMAIL_ENABLED: ${MESSAGING_PROVIDER_GMAIL_ENABLED} + # CALENDAR_PROVIDER_GOOGLE_ENABLED: ${CALENDAR_PROVIDER_GOOGLE_ENABLED} + # AUTH_GOOGLE_CLIENT_ID: ${AUTH_GOOGLE_CLIENT_ID} + # AUTH_GOOGLE_CLIENT_SECRET: ${AUTH_GOOGLE_CLIENT_SECRET} + # AUTH_GOOGLE_CALLBACK_URL: ${AUTH_GOOGLE_CALLBACK_URL} + # AUTH_GOOGLE_APIS_CALLBACK_URL: ${AUTH_GOOGLE_APIS_CALLBACK_URL} + + # CALENDAR_PROVIDER_MICROSOFT_ENABLED: ${CALENDAR_PROVIDER_MICROSOFT_ENABLED} + # MESSAGING_PROVIDER_MICROSOFT_ENABLED: ${MESSAGING_PROVIDER_MICROSOFT_ENABLED} + # AUTH_MICROSOFT_ENABLED: ${AUTH_MICROSOFT_ENABLED} + # AUTH_MICROSOFT_CLIENT_ID: ${AUTH_MICROSOFT_CLIENT_ID} + # AUTH_MICROSOFT_CLIENT_SECRET: ${AUTH_MICROSOFT_CLIENT_SECRET} + # AUTH_MICROSOFT_CALLBACK_URL: ${AUTH_MICROSOFT_CALLBACK_URL} + # AUTH_MICROSOFT_APIS_CALLBACK_URL: ${AUTH_MICROSOFT_APIS_CALLBACK_URL} + + # EMAIL_FROM_ADDRESS: ${EMAIL_FROM_ADDRESS:-contact@yourdomain.com} + # EMAIL_FROM_NAME: ${EMAIL_FROM_NAME:-"John from YourDomain"} + # EMAIL_SYSTEM_ADDRESS: ${EMAIL_SYSTEM_ADDRESS:-system@yourdomain.com} + # EMAIL_DRIVER: ${EMAIL_DRIVER:-smtp} + # EMAIL_SMTP_HOST: ${EMAIL_SMTP_HOST:-smtp.gmail.com} + # EMAIL_SMTP_PORT: ${EMAIL_SMTP_PORT:-465} + # EMAIL_SMTP_USER: ${EMAIL_SMTP_USER:-} + # EMAIL_SMTP_PASSWORD: ${EMAIL_SMTP_PASSWORD:-} + depends_on: - change-vol-ownership: - condition: service_completed_successfully db: condition: service_healthy healthcheck: test: curl --fail http://localhost:3000/healthz interval: 5s timeout: 5s - retries: 10 + retries: 20 restart: always worker: image: twentycrm/twenty:${TAG:-latest} + volumes: + - server-local-data:/app/packages/twenty-server/${STORAGE_LOCAL_PATH:-.local-storage} command: [ "yarn", "worker:prod" ] environment: PG_DATABASE_URL: postgres://${PG_DATABASE_USER:-postgres}:${PG_DATABASE_PASSWORD:-postgres}@${PG_DATABASE_HOST:-db}:${PG_DATABASE_PORT:-5432}/default @@ -54,7 +67,31 @@ services: STORAGE_S3_NAME: ${STORAGE_S3_NAME} STORAGE_S3_ENDPOINT: ${STORAGE_S3_ENDPOINT} - APP_SECRET: ${APP_SECRET} + APP_SECRET: ${APP_SECRET:-replace_me_with_a_random_string} + # MESSAGING_PROVIDER_GMAIL_ENABLED: ${MESSAGING_PROVIDER_GMAIL_ENABLED} + # CALENDAR_PROVIDER_GOOGLE_ENABLED: ${CALENDAR_PROVIDER_GOOGLE_ENABLED} + # AUTH_GOOGLE_CLIENT_ID: ${AUTH_GOOGLE_CLIENT_ID} + # AUTH_GOOGLE_CLIENT_SECRET: ${AUTH_GOOGLE_CLIENT_SECRET} + # AUTH_GOOGLE_CALLBACK_URL: ${AUTH_GOOGLE_CALLBACK_URL} + # AUTH_GOOGLE_APIS_CALLBACK_URL: ${AUTH_GOOGLE_APIS_CALLBACK_URL} + + # CALENDAR_PROVIDER_MICROSOFT_ENABLED: ${CALENDAR_PROVIDER_MICROSOFT_ENABLED} + # MESSAGING_PROVIDER_MICROSOFT_ENABLED: ${MESSAGING_PROVIDER_MICROSOFT_ENABLED} + # AUTH_MICROSOFT_ENABLED: ${AUTH_MICROSOFT_ENABLED} + # AUTH_MICROSOFT_CLIENT_ID: ${AUTH_MICROSOFT_CLIENT_ID} + # AUTH_MICROSOFT_CLIENT_SECRET: ${AUTH_MICROSOFT_CLIENT_SECRET} + # AUTH_MICROSOFT_CALLBACK_URL: ${AUTH_MICROSOFT_CALLBACK_URL} + # AUTH_MICROSOFT_APIS_CALLBACK_URL: ${AUTH_MICROSOFT_APIS_CALLBACK_URL} + + # EMAIL_FROM_ADDRESS: ${EMAIL_FROM_ADDRESS:-contact@yourdomain.com} + # EMAIL_FROM_NAME: ${EMAIL_FROM_NAME:-"John from YourDomain"} + # EMAIL_SYSTEM_ADDRESS: ${EMAIL_SYSTEM_ADDRESS:-system@yourdomain.com} + # EMAIL_DRIVER: ${EMAIL_DRIVER:-smtp} + # EMAIL_SMTP_HOST: ${EMAIL_SMTP_HOST:-smtp.gmail.com} + # EMAIL_SMTP_PORT: ${EMAIL_SMTP_PORT:-465} + # EMAIL_SMTP_USER: ${EMAIL_SMTP_USER:-} + # EMAIL_SMTP_PASSWORD: ${EMAIL_SMTP_PASSWORD:-} + depends_on: db: condition: service_healthy @@ -63,14 +100,12 @@ services: restart: always db: - image: twentycrm/twenty-postgres-spilo:${TAG:-latest} + image: postgres:16 volumes: - - db-data:/home/postgres/pgdata + - db-data:/var/lib/postgresql/data environment: - PGUSER_SUPERUSER: ${PG_DATABASE_USER:-postgres} - PGPASSWORD_SUPERUSER: ${PG_DATABASE_PASSWORD:-postgres} - ALLOW_NOSSL: "true" - SPILO_PROVIDER: "local" + POSTGRES_USER: ${PG_DATABASE_USER:-postgres} + POSTGRES_PASSWORD: ${PG_DATABASE_PASSWORD:-postgres} healthcheck: test: pg_isready -U ${PG_DATABASE_USER:-postgres} -h localhost -d postgres interval: 5s @@ -81,8 +116,8 @@ services: redis: image: redis restart: always + command: [ "--maxmemory-policy", "noeviction" ] volumes: - docker-data: db-data: server-local-data: diff --git a/twenty/code/k8s/README.md b/twenty/code/k8s/README.md index df1cde7..d538d21 100644 --- a/twenty/code/k8s/README.md +++ b/twenty/code/k8s/README.md @@ -1,4 +1,7 @@ # README +DISCLAIMER: The k8s and podman deployments are not maintained by the core team. +These files are provided and maintained by the community. Twenty core team +maintains support for docker deployment. ## Overview diff --git a/twenty/code/k8s/manifests/deployment-redis.yaml b/twenty/code/k8s/manifests/deployment-redis.yaml index e09874a..bd7aaa3 100644 --- a/twenty/code/k8s/manifests/deployment-redis.yaml +++ b/twenty/code/k8s/manifests/deployment-redis.yaml @@ -25,9 +25,10 @@ spec: - name: redis image: redis/redis-stack-server:latest imagePullPolicy: Always + args: ["--maxmemory-policy", "noeviction"] env: - name: PORT - value: 6379 + value: "6379" ports: - containerPort: 6379 name: redis diff --git a/twenty/code/podman/README.md b/twenty/code/podman/README.md new file mode 100644 index 0000000..91aa202 --- /dev/null +++ b/twenty/code/podman/README.md @@ -0,0 +1,48 @@ +# How to deploy twenty on podman + +DISCLAIMER: The k8s and podman deployments are not maintained by the core team. +These files are provided and maintained by the community. Twenty core team +maintains support for docker deployment. + + +## How to use + +1. Edit `.env` file. At the minimum set `POSTGRES_PASSWORD`, `SERVER_URL`, and `APP_SECRET`. +2. Start twenty by running `podman-compose up -d`. + +If you need to stop twenty, you can do so by running `podman-compose down`. + + +### Install systemd service (optional) + +If you want to install a systemd service to run twenty, you can use the provided systemd service. + +Edit `twentycrm.service` and change these two variables: + + + WorkingDirectory=/opt/apps/twenty + EnvironmentFile=/opt/apps/twenty/.env + +`WorkingDirectory` should be changed to the path in which `podman-compose.yml` is located. + +`EnvironmentFile` should be changed to the path in which your `.env`file is located. + +You can run the script `install-systemd-user-service` to install the systemd service under the current user. + + + ./install-systemd-user-service + +Note: this script will enable the service and also start it. So it will assume that twenty is not currently running. +If you started it previously, bring it down using: + + podman-compose down + + + +## Compatibility + +These files should be compatible with podman 4.3+. + +I have tested this on Debian GNU/Linux 12 (bookworm) and with the podman that is distributed with the official Debian stable mirrors (podman v4.3.1+ds1-8+deb12u1, podman-compose v1.0.3-3). + + diff --git a/twenty/code/podman/install-systemd-user-service b/twenty/code/podman/install-systemd-user-service new file mode 100755 index 0000000..4e10ba1 --- /dev/null +++ b/twenty/code/podman/install-systemd-user-service @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +mkdir -p ~/.config/systemd/user +[ -f twentycrm.service ] || { echo "Error: twentycrm.service not found"; exit 1; } +cp twentycrm.service ~/.config/systemd/user + + +systemctl --user daemon-reload +systemctl --user enable twentycrm.service +systemctl --user start twentycrm.service diff --git a/twenty/code/podman/manual-steps-to-deploy-twenty-on-podman b/twenty/code/podman/manual-steps-to-deploy-twenty-on-podman new file mode 100644 index 0000000..beb9797 --- /dev/null +++ b/twenty/code/podman/manual-steps-to-deploy-twenty-on-podman @@ -0,0 +1,71 @@ +#!/bin/bash +set -e +if [ ! -f "$(dirname "$0")/.env" ]; then + echo "Error: .env file not found" + exit 1 +fi +source "$(dirname "$0")/.env" + +## steps to cleanup and start over from scratch - useful when testing +#podman pod stop twenty-pod +#podman pod rm twenty-pod +#podman volume rm twenty-db-data twenty-server-data twenty-redis-data +# end of cleanup + +podman volume create twenty-db-data +podman volume create twenty-server-data +podman volume create twenty-redis-data + +podman pod create --name twenty-pod -p 127.0.0.1:8080:3000 + +podman run -d --pod twenty-pod --name twenty-db \ + -e POSTGRES_DB=twenty \ + -e POSTGRES_USER=twenty \ + -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ + -v twenty-db-data:/var/lib/postgresql/data:Z \ + docker.io/library/postgres:16 + +podman run -d --pod twenty-pod --name twenty-redis \ + -v twenty-redis-data:/data:Z \ + docker.io/library/redis:latest + +podman run -d --pod twenty-pod --name twenty-server \ + -e NODE_PORT=3000 \ + -e SERVER_URL="$SERVER_URL" \ + -e PG_DATABASE_URL="postgresql://twenty:$POSTGRES_PASSWORD@twenty-db:5432/twenty" \ + -e REDIS_URL="redis://twenty-redis:6379" \ + -e APP_SECRET="$APP_SECRET" \ + -e NODE_ENV=production \ + -e LOG_LEVEL=info \ + -v twenty-server-data:/app/docker-data:Z \ + docker.io/twentycrm/twenty:latest + +podman run -d --pod twenty-pod --name twenty-worker \ + --init \ + -e SERVER_URL="$SERVER_URL" \ + -e PG_DATABASE_URL="postgresql://twenty:$POSTGRES_PASSWORD@twenty-db:5432/twenty" \ + -e REDIS_URL="redis://twenty-redis:6379" \ + -e APP_SECRET="$APP_SECRET" \ + -e DISABLE_DB_MIGRATIONS=true \ + -e NODE_ENV=production \ + -e LOG_LEVEL=info \ + -v twenty-server-data:/app/docker-data:Z \ + docker.io/twentycrm/twenty:latest \ + yarn worker:prod + +# wait some time, check status +sleep 30 +podman ps --pod -f name=twenty-pod + + +mkdir -p ~/.config/systemd/user +if [ ! -f "twentycrm.service" ]; then + echo "Error: twentycrm.service file not found" + exit 1 +fi +cp twentycrm.service ~/.config/systemd/user + +systemctl --user daemon-reload +systemctl --user enable twentycrm.service +systemctl --user start twentycrm.service + diff --git a/twenty/code/podman/podman-compose.yml b/twenty/code/podman/podman-compose.yml new file mode 100644 index 0000000..fa0ecae --- /dev/null +++ b/twenty/code/podman/podman-compose.yml @@ -0,0 +1,56 @@ +version: "3.8" + +services: + db: + container_name: twenty-db + image: postgres:16 + environment: + POSTGRES_DB: twenty + POSTGRES_USER: twenty + POSTGRES_PASSWORD: ${PG_DATABASE_PASSWORD} + volumes: + - twenty-db-data:/var/lib/postgresql/data:Z + + redis: + container_name: twenty-redis + image: redis:latest + command: ["--maxmemory-policy", "noeviction"] + volumes: + - twenty-redis-data:/data:Z + + server: + container_name: twenty-server + image: twentycrm/twenty:latest + environment: + NODE_PORT: "3000" + SERVER_URL: ${SERVER_URL} + PG_DATABASE_URL: "postgresql://twenty:${PG_DATABASE_PASSWORD}@twenty-db:5432/twenty" + REDIS_URL: "redis://twenty-redis:6379" + APP_SECRET: ${APP_SECRET} + NODE_ENV: production + LOG_LEVEL: info + volumes: + - twenty-server-data:/app/docker-data:Z + ports: + - 127.0.0.1:8080:3000 + + worker: + container_name: twenty-worker + image: twentycrm/twenty:latest + command: yarn worker:prod + environment: + SERVER_URL: ${SERVER_URL} + PG_DATABASE_URL: "postgresql://twenty:${PG_DATABASE_PASSWORD}@twenty-db:5432/twenty" + REDIS_URL: "redis://twenty-redis:6379" + APP_SECRET: ${APP_SECRET} + DISABLE_DB_MIGRATIONS: "true" + NODE_ENV: production + LOG_LEVEL: info + volumes: + - twenty-server-data:/app/docker-data:Z + init: true + +volumes: + twenty-db-data: + twenty-server-data: + twenty-redis-data: diff --git a/twenty/code/podman/twentycrm.service b/twenty/code/podman/twentycrm.service new file mode 100644 index 0000000..5364993 --- /dev/null +++ b/twenty/code/podman/twentycrm.service @@ -0,0 +1,15 @@ +[Unit] +Description=TwentyCRM Container Stack via Podman-Compose +After=network.target + +[Service] +Type=simple +WorkingDirectory=/opt/apps/twenty +EnvironmentFile=/opt/apps/twenty/.env +ExecStart=/usr/bin/podman-compose -f podman-compose.yml up -d +ExecStop=/usr/bin/podman-compose -f podman-compose.yml down +RemainAfterExit=yes + +[Install] +WantedBy=default.target + diff --git a/twenty/code/twenty-website/Dockerfile b/twenty/code/twenty-website/Dockerfile index e3b7420..3f64a07 100644 --- a/twenty/code/twenty-website/Dockerfile +++ b/twenty/code/twenty-website/Dockerfile @@ -8,10 +8,18 @@ COPY ./yarn.lock . COPY ./.yarnrc.yml . COPY ./.yarn/releases /app/.yarn/releases COPY ./tools/eslint-rules /app/tools/eslint-rules +COPY ./packages/twenty-ui/package.json /app/packages/twenty-ui/ +COPY ./packages/twenty-shared/package.json /app/packages/twenty-shared/ COPY ./packages/twenty-website/package.json /app/packages/twenty-website/package.json RUN yarn +ENV KEYSTATIC_GITHUB_CLIENT_ID="" +ENV KEYSTATIC_GITHUB_CLIENT_SECRET="" +ENV KEYSTATIC_SECRET="" +ENV NEXT_PUBLIC_KEYSTATIC_GITHUB_APP_SLUG="" + +COPY ./packages/twenty-ui /app/packages/twenty-ui COPY ./packages/twenty-website /app/packages/twenty-website RUN npx nx build twenty-website @@ -26,4 +34,9 @@ WORKDIR /app/packages/twenty-website LABEL org.opencontainers.image.source=https://github.com/twentyhq/twenty LABEL org.opencontainers.image.description="This image provides a consistent and reproducible environment for the website." +RUN chown -R 1000 /app + +# Use non root user with uid 1000 +USER 1000 + CMD ["/bin/sh", "-c", "npx nx start"] \ No newline at end of file diff --git a/twenty/code/twenty/Dockerfile b/twenty/code/twenty/Dockerfile index d171683..4100694 100644 --- a/twenty/code/twenty/Dockerfile +++ b/twenty/code/twenty/Dockerfile @@ -7,6 +7,7 @@ WORKDIR /app COPY ./package.json ./yarn.lock ./.yarnrc.yml ./tsconfig.base.json ./nx.json /app/ COPY ./.yarn/releases /app/.yarn/releases +COPY ./.prettierrc /app/ COPY ./packages/twenty-emails/package.json /app/packages/twenty-emails/ COPY ./packages/twenty-server/package.json /app/packages/twenty-server/ COPY ./packages/twenty-server/patches /app/packages/twenty-server/patches @@ -63,8 +64,9 @@ WORKDIR /app/packages/twenty-server ARG REACT_APP_SERVER_BASE_URL ENV REACT_APP_SERVER_BASE_URL $REACT_APP_SERVER_BASE_URL -ARG SENTRY_RELEASE -ENV SENTRY_RELEASE $SENTRY_RELEASE + +ARG APP_VERSION +ENV APP_VERSION $APP_VERSION # Copy built applications from previous stages COPY --chown=1000 --from=twenty-server-build /app /app @@ -75,8 +77,8 @@ COPY --chown=1000 --from=twenty-front-build /app/packages/twenty-front/build /ap LABEL org.opencontainers.image.source=https://github.com/twentyhq/twenty LABEL org.opencontainers.image.description="This image provides a consistent and reproducible environment for the backend and frontend, ensuring it deploys faster and runs the same way regardless of the deployment environment." -RUN mkdir /app/.local-storage -RUN chown -R 1000 /app +RUN mkdir -p /app/.local-storage /app/packages/twenty-server/.local-storage && \ + chown -R 1000:1000 /app # Use non root user with uid 1000 USER 1000 diff --git a/twenty/code/twenty/entrypoint.sh b/twenty/code/twenty/entrypoint.sh index 2b6985a..a7de30a 100755 --- a/twenty/code/twenty/entrypoint.sh +++ b/twenty/code/twenty/entrypoint.sh @@ -1,26 +1,33 @@ #!/bin/sh set -e -# Check if the initialization has already been done and that we enabled automatic migration -if [ "${DISABLE_DB_MIGRATIONS}" != "true" ] && [ ! -f /app/docker-data/db_status ]; then - echo "Running database setup and migrations..." +setup_and_migrate_db() { + if [ "${DISABLE_DB_MIGRATIONS}" = "true" ]; then + echo "Database setup and migrations are disabled, skipping..." + return + fi - # Creating the database if it doesn't exist + echo "Running database setup and migrations..." PGUSER=$(echo $PG_DATABASE_URL | awk -F '//' '{print $2}' | awk -F ':' '{print $1}') PGPASS=$(echo $PG_DATABASE_URL | awk -F ':' '{print $3}' | awk -F '@' '{print $1}') PGHOST=$(echo $PG_DATABASE_URL | awk -F '@' '{print $2}' | awk -F ':' '{print $1}') PGPORT=$(echo $PG_DATABASE_URL | awk -F ':' '{print $4}' | awk -F '/' '{print $1}') - PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'default'" | grep -q 1 || \ - PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d postgres -c "CREATE DATABASE \"default\"" + PGDATABASE=$(echo $PG_DATABASE_URL | awk -F ':' '{print $4}' | awk -F '/' '{print $2}') - # Run setup and migration scripts - NODE_OPTIONS="--max-old-space-size=1500" tsx ./scripts/setup-db.ts - yarn database:migrate:prod + # Creating the database if it doesn't exist + db_count=$(PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d postgres -tAc "SELECT COUNT(*) FROM pg_database WHERE datname = '${PGDATABASE}'") + if [ "$db_count" = "0" ]; then + echo "Database ${PGDATABASE} does not exist, creating..." + PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d postgres -c "CREATE DATABASE \"${PGDATABASE}\"" - # Mark initialization as done - echo "Successfuly migrated DB!" - touch /app/docker-data/db_status -fi + # Run setup and migration scripts + NODE_OPTIONS="--max-old-space-size=1500" tsx ./scripts/setup-db.ts + fi + + yarn command:prod upgrade + echo "Successfully migrated DB!" +} +setup_and_migrate_db # Continue with the original Docker command exec "$@"