From fa0b9127f12b3d7bff02b20db0614e289f844a6c Mon Sep 17 00:00:00 2001 From: Andrei Canta Date: Thu, 28 Aug 2025 15:52:36 +0300 Subject: [PATCH] run updates --- appwrite/code/.env.example | 2 +- appwrite/update.js | 6 + appwrite/update.sh | 14 -- dify/code/.env.example | 148 ++++++++++++++++++- dify/code/README.md | 103 +++++++------ dify/code/certbot/README.md | 8 +- dify/code/docker-compose-template.yaml | 37 ++++- dify/code/docker-compose.middleware.yaml | 4 +- dify/code/docker-compose.yaml | 104 ++++++++++++- dify/code/nginx/conf.d/default.conf.template | 5 +- dify/update.js | 6 + dify/update.sh | 13 -- plane/update.js | 24 +++ plane/update.sh | 14 -- supabase/code/.env.example | 2 +- supabase/code/docker-compose.yml | 8 +- supabase/update.js | 6 + supabase/update.sh | 11 -- twenty/code/twenty-website/Dockerfile | 5 +- twenty/code/twenty/Dockerfile | 19 +-- twenty/code/twenty/entrypoint.sh | 6 +- twenty/update.sh | 13 -- 22 files changed, 399 insertions(+), 159 deletions(-) delete mode 100644 appwrite/update.sh delete mode 100644 dify/update.sh delete mode 100644 plane/update.sh delete mode 100644 supabase/update.sh delete mode 100644 twenty/update.sh diff --git a/appwrite/code/.env.example b/appwrite/code/.env.example index ad44177..ad66348 100644 --- a/appwrite/code/.env.example +++ b/appwrite/code/.env.example @@ -6,7 +6,7 @@ _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled _APP_OPTIONS_ROUTER_FORCE_HTTPS=disabled _APP_OPTIONS_ROUTER_PROTECTION=disabled _APP_OPENSSL_KEY_V1=your-secret-key -_APP_DOMAIN=localhost +_APP_DOMAIN=$(PRIMARY_DOMAIN)localhost _APP_CUSTOM_DOMAIN_DENY_LIST=example.com,test.com,app.example.com _APP_DOMAIN_FUNCTIONS=functions.localhost _APP_DOMAIN_SITES=sites.localhost diff --git a/appwrite/update.js b/appwrite/update.js index c8308da..67a4d15 100644 --- a/appwrite/update.js +++ b/appwrite/update.js @@ -11,3 +11,9 @@ await utils.downloadFile( await utils.removeContainerNames("./code/docker-compose.yml"); await utils.removePorts("./code/docker-compose.yml"); + +await utils.searchReplace( + "./code/.env.example", + "_APP_DOMAIN=", + "_APP_DOMAIN=$(PRIMARY_DOMAIN)" +); diff --git a/appwrite/update.sh b/appwrite/update.sh deleted file mode 100644 index 2ae6244..0000000 --- a/appwrite/update.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [ ! -d "./repo" ]; then - git clone --depth 1 --branch main --single-branch https://github.com/appwrite/appwrite.git repo -else - cd repo - git pull - cd .. -fi - -curl -s https://appwrite.io/install/compose > ./code/docker-compose.yml -curl -s https://appwrite.io/install/env > ./code/.env.example - - diff --git a/dify/code/.env.example b/dify/code/.env.example index a024566..31aed97 100644 --- a/dify/code/.env.example +++ b/dify/code/.env.example @@ -34,7 +34,7 @@ APP_API_URL= # used to display WebAPP API Base Url to the front-end. # If empty, it is the same domain. # Example: https://app.dify.ai -APP_WEB_URL= +APP_WEB_URL=https://$(PRIMARY_DOMAIN) # File preview or download Url prefix. # used to display File preview or download Url to the front-end or as Multi-model inputs; @@ -47,6 +47,16 @@ APP_WEB_URL= # ensuring port 5001 is externally accessible (see docker-compose.yaml). FILES_URL= +# INTERNAL_FILES_URL is used for plugin daemon communication within Docker network. +# Set this to the internal Docker service URL for proper plugin file access. +# Example: INTERNAL_FILES_URL=http://api:5001 +INTERNAL_FILES_URL= + +# Ensure UTF-8 encoding +LANG=en_US.UTF-8 +LC_ALL=en_US.UTF-8 +PYTHONIOENCODING=utf-8 + # ------------------------------ # Server Configuration # ------------------------------ @@ -205,10 +215,16 @@ DB_DATABASE=dify # The size of the database connection pool. # The default is 30 connections, which can be appropriately increased. SQLALCHEMY_POOL_SIZE=30 +# The default is 10 connections, which allows temporary overflow beyond the pool size. +SQLALCHEMY_MAX_OVERFLOW=10 # Database connection pool recycling time, the default is 3600 seconds. SQLALCHEMY_POOL_RECYCLE=3600 # Whether to print SQL, default is false. SQLALCHEMY_ECHO=false +# If True, will test connections for liveness upon each checkout +SQLALCHEMY_POOL_PRE_PING=false +# Whether to enable the Last in first out option or use default FIFO queue if is false +SQLALCHEMY_POOL_USE_LIFO=false # Maximum number of connections to the database # Default is 100 @@ -250,6 +266,15 @@ REDIS_PORT=6379 REDIS_USERNAME= REDIS_PASSWORD=difyai123456 REDIS_USE_SSL=false +# SSL configuration for Redis (when REDIS_USE_SSL=true) +REDIS_SSL_CERT_REQS=CERT_NONE +# Options: CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED +REDIS_SSL_CA_CERTS= +# Path to CA certificate file for SSL verification +REDIS_SSL_CERTFILE= +# Path to client certificate file for SSL authentication +REDIS_SSL_KEYFILE= +# Path to client private key file for SSL authentication REDIS_DB=0 # Whether to use Redis Sentinel mode. @@ -274,12 +299,14 @@ REDIS_CLUSTERS_PASSWORD= # Celery Configuration # ------------------------------ -# Use redis as the broker, and redis db 1 for celery broker. -# Format as follows: `redis://:@:/` +# Use standalone redis as the broker, and redis db 1 for celery broker. (redis_username is usually set by defualt as empty) +# Format as follows: `redis://:@:/`. # Example: redis://:difyai123456@redis:6379/1 -# If use Redis Sentinel, format as follows: `sentinel://:@:/` -# Example: sentinel://localhost:26379/1;sentinel://localhost:26380/1;sentinel://localhost:26381/1 +# If use Redis Sentinel, format as follows: `sentinel://:@:/` +# For high availability, you can configure multiple Sentinel nodes (if provided) separated by semicolons like below example: +# Example: sentinel://:difyai123456@localhost:26379/1;sentinel://:difyai12345@localhost:26379/1;sentinel://:difyai12345@localhost:26379/1 CELERY_BROKER_URL=redis://:difyai123456@redis:6379/1 +CELERY_BACKEND=redis BROKER_USE_SSL=false # If you are using Redis Sentinel for high availability, configure the following settings. @@ -317,6 +344,25 @@ OPENDAL_SCHEME=fs # Configurations for OpenDAL Local File System. OPENDAL_FS_ROOT=storage +# ClickZetta Volume Configuration (for storage backend) +# To use ClickZetta Volume as storage backend, set STORAGE_TYPE=clickzetta-volume +# Note: ClickZetta Volume will reuse the existing CLICKZETTA_* connection parameters + +# Volume type selection (three types available): +# - user: Personal/small team use, simple config, user-level permissions +# - table: Enterprise multi-tenant, smart routing, table-level + user-level permissions +# - external: Data lake integration, external storage connection, volume-level + storage-level permissions +CLICKZETTA_VOLUME_TYPE=user + +# External Volume name (required only when TYPE=external) +CLICKZETTA_VOLUME_NAME= + +# Table Volume table prefix (used only when TYPE=table) +CLICKZETTA_VOLUME_TABLE_PREFIX=dataset_ + +# Dify file directory prefix (isolates from other apps, recommended to keep default) +CLICKZETTA_VOLUME_DIFY_PREFIX=dify_km + # S3 Configuration # S3_ENDPOINT= @@ -400,8 +446,10 @@ SUPABASE_URL=your-server-url # ------------------------------ # The type of vector store to use. -# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`, `opengauss`, `tablestore`,`vastbase`,`tidb`,`tidb_on_qdrant`,`baidu`,`lindorm`,`huawei_cloud`,`upstash`, `matrixone`. +# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`, `opengauss`, `tablestore`,`vastbase`,`tidb`,`tidb_on_qdrant`,`baidu`,`lindorm`,`huawei_cloud`,`upstash`, `matrixone`, `clickzetta`. VECTOR_STORE=weaviate +# Prefix used to create collection name in vector database +VECTOR_INDEX_NAME_PREFIX=Vector_index # The Weaviate endpoint URL. Only available when VECTOR_STORE is `weaviate`. WEAVIATE_ENDPOINT=http://weaviate:8080 @@ -565,6 +613,17 @@ ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=elastic KIBANA_PORT=5601 +# Using ElasticSearch Cloud Serverless, or not. +ELASTICSEARCH_USE_CLOUD=false +ELASTICSEARCH_CLOUD_URL=YOUR-ELASTICSEARCH_CLOUD_URL +ELASTICSEARCH_API_KEY=YOUR-ELASTICSEARCH_API_KEY + +ELASTICSEARCH_VERIFY_CERTS=False +ELASTICSEARCH_CA_CERTS= +ELASTICSEARCH_REQUEST_TIMEOUT=100000 +ELASTICSEARCH_RETRY_ON_TIMEOUT=True +ELASTICSEARCH_MAX_RETRIES=10 + # baidu vector configurations, only available when VECTOR_STORE is `baidu` BAIDU_VECTOR_DB_ENDPOINT=http://127.0.0.1:5287 BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS=30000 @@ -624,6 +683,21 @@ TABLESTORE_ENDPOINT=https://instance-name.cn-hangzhou.ots.aliyuncs.com TABLESTORE_INSTANCE_NAME=instance-name TABLESTORE_ACCESS_KEY_ID=xxx TABLESTORE_ACCESS_KEY_SECRET=xxx +TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE=false + +# Clickzetta configuration, only available when VECTOR_STORE is `clickzetta` +CLICKZETTA_USERNAME= +CLICKZETTA_PASSWORD= +CLICKZETTA_INSTANCE= +CLICKZETTA_SERVICE=api.clickzetta.com +CLICKZETTA_WORKSPACE=quick_start +CLICKZETTA_VCLUSTER=default_ap +CLICKZETTA_SCHEMA=dify +CLICKZETTA_BATCH_SIZE=100 +CLICKZETTA_ENABLE_INVERTED_INDEX=true +CLICKZETTA_ANALYZER_TYPE=chinese +CLICKZETTA_ANALYZER_MODE=smart +CLICKZETTA_VECTOR_DISTANCE_FUNCTION=cosine_distance # ------------------------------ # Knowledge Configuration @@ -705,6 +779,12 @@ API_SENTRY_PROFILES_SAMPLE_RATE=1.0 # If not set, Sentry error reporting will be disabled. WEB_SENTRY_DSN= +# Plugin_daemon Service Sentry DSN address, default is empty, when empty, +# all monitoring information is not reported to Sentry. +# If not set, Sentry error reporting will be disabled. +PLUGIN_SENTRY_ENABLED=false +PLUGIN_SENTRY_DSN= + # ------------------------------ # Notion Integration Configuration # Variables can be obtained by applying for Notion integration: https://www.notion.so/my-integrations @@ -763,6 +843,8 @@ INVITE_EXPIRY_HOURS=72 # Reset password token valid time (minutes), RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5 +CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES=5 +OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES=5 # The sandbox service endpoint. CODE_EXECUTION_ENDPOINT=http://sandbox:8194 @@ -794,6 +876,33 @@ WORKFLOW_FILE_UPLOAD_LIMIT=10 # hybrid: Save new data to object storage, read from both object storage and RDBMS WORKFLOW_NODE_EXECUTION_STORAGE=rdbms +# Repository configuration +# Core workflow execution repository implementation +# Options: +# - core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository (default) +# - core.repositories.celery_workflow_execution_repository.CeleryWorkflowExecutionRepository +CORE_WORKFLOW_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository + +# Core workflow node execution repository implementation +# Options: +# - core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository (default) +# - core.repositories.celery_workflow_node_execution_repository.CeleryWorkflowNodeExecutionRepository +CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository + +# API workflow run repository implementation +API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository + +# API workflow node execution repository implementation +API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository + +# Workflow log cleanup configuration +# Enable automatic cleanup of workflow run logs to manage database size +WORKFLOW_LOG_CLEANUP_ENABLED=false +# Number of days to retain workflow run logs (default: 30 days) +WORKFLOW_LOG_RETENTION_DAYS=30 +# Batch size for workflow log cleanup operations (default: 100) +WORKFLOW_LOG_CLEANUP_BATCH_SIZE=100 + # HTTP request node in workflow configuration HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576 @@ -826,6 +935,12 @@ MAX_ITERATIONS_NUM=99 # The timeout for the text generation in millisecond TEXT_GENERATION_TIMEOUT_MS=60000 +# Allow rendering unsafe URLs which have "data:" scheme. +ALLOW_UNSAFE_DATA_SCHEME=false + +# Maximum number of tree depth in the workflow +MAX_TREE_DEPTH=50 + # ------------------------------ # Environment Variables for db Service # ------------------------------ @@ -958,7 +1073,7 @@ NGINX_SSL_PROTOCOLS=TLSv1.1 TLSv1.2 TLSv1.3 # Nginx performance tuning NGINX_WORKER_PROCESSES=auto -NGINX_CLIENT_MAX_BODY_SIZE=15M +NGINX_CLIENT_MAX_BODY_SIZE=100M NGINX_KEEPALIVE_TIMEOUT=65 # Proxy settings @@ -1067,6 +1182,9 @@ MARKETPLACE_API_URL=https://marketplace.dify.ai FORCE_VERIFYING_SIGNATURE=true +PLUGIN_STDIO_BUFFER_SIZE=1024 +PLUGIN_STDIO_MAX_BUFFER_SIZE=5242880 + PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120 PLUGIN_MAX_EXECUTION_TIMEOUT=600 # PIP_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple @@ -1114,6 +1232,8 @@ PLUGIN_VOLCENGINE_TOS_REGION= # OTLP Collector Configuration # ------------------------------ ENABLE_OTEL=false +OTLP_TRACE_ENDPOINT= +OTLP_METRIC_ENDPOINT= OTLP_BASE_ENDPOINT=http://localhost:4318 OTLP_API_KEY= OTEL_EXPORTER_OTLP_PROTOCOL= @@ -1135,3 +1255,17 @@ QUEUE_MONITOR_THRESHOLD=200 QUEUE_MONITOR_ALERT_EMAILS= # Monitor interval in minutes, default is 30 minutes QUEUE_MONITOR_INTERVAL=30 + +# Swagger UI configuration +SWAGGER_UI_ENABLED=true +SWAGGER_UI_PATH=/swagger-ui.html + +# Celery schedule tasks configuration +ENABLE_CLEAN_EMBEDDING_CACHE_TASK=false +ENABLE_CLEAN_UNUSED_DATASETS_TASK=false +ENABLE_CREATE_TIDB_SERVERLESS_TASK=false +ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK=false +ENABLE_CLEAN_MESSAGES=false +ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK=false +ENABLE_DATASETS_QUEUE_MONITOR=false +ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK=true diff --git a/dify/code/README.md b/dify/code/README.md index 22dfe2c..b5c46eb 100644 --- a/dify/code/README.md +++ b/dify/code/README.md @@ -4,7 +4,7 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T ### What's Updated -- **Certbot Container**: `docker-compose.yaml` now contains `certbot` for managing SSL certificates. This container automatically renews certificates and ensures secure HTTPS connections. +- **Certbot Container**: `docker-compose.yaml` now contains `certbot` for managing SSL certificates. This container automatically renews certificates and ensures secure HTTPS connections.\ For more information, refer `docker/certbot/README.md`. - **Persistent Environment Variables**: Environment variables are now managed through a `.env` file, ensuring that your configurations persist across deployments. @@ -13,43 +13,44 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T > The `.env` file is a crucial component in Docker and Docker Compose environments, serving as a centralized configuration file where you can define environment variables that are accessible to the containers at runtime. This file simplifies the management of environment settings across different stages of development, testing, and production, providing consistency and ease of configuration to deployments. - **Unified Vector Database Services**: All vector database services are now managed from a single Docker Compose file `docker-compose.yaml`. You can switch between different vector databases by setting the `VECTOR_STORE` environment variable in your `.env` file. + - **Mandatory .env File**: A `.env` file is now required to run `docker compose up`. This file is crucial for configuring your deployment and for any custom settings to persist through upgrades. ### How to Deploy Dify with `docker-compose.yaml` 1. **Prerequisites**: Ensure Docker and Docker Compose are installed on your system. -2. **Environment Setup**: - - Navigate to the `docker` directory. - - Copy the `.env.example` file to a new file named `.env` by running `cp .env.example .env`. - - Customize the `.env` file as needed. Refer to the `.env.example` file for detailed configuration options. -3. **Running the Services**: - - Execute `docker compose up` from the `docker` directory to start the services. - - To specify a vector database, set the `VECTOR_STORE` variable in your `.env` file to your desired vector database service, such as `milvus`, `weaviate`, or `opensearch`. -4. **SSL Certificate Setup**: - - Refer `docker/certbot/README.md` to set up SSL certificates using Certbot. -5. **OpenTelemetry Collector Setup**: +1. **Environment Setup**: + - Navigate to the `docker` directory. + - Copy the `.env.example` file to a new file named `.env` by running `cp .env.example .env`. + - Customize the `.env` file as needed. Refer to the `.env.example` file for detailed configuration options. +1. **Running the Services**: + - Execute `docker compose up` from the `docker` directory to start the services. + - To specify a vector database, set the `VECTOR_STORE` variable in your `.env` file to your desired vector database service, such as `milvus`, `weaviate`, or `opensearch`. +1. **SSL Certificate Setup**: + - Refer `docker/certbot/README.md` to set up SSL certificates using Certbot. +1. **OpenTelemetry Collector Setup**: - Change `ENABLE_OTEL` to `true` in `.env`. - Configure `OTLP_BASE_ENDPOINT` properly. ### How to Deploy Middleware for Developing Dify 1. **Middleware Setup**: - - Use the `docker-compose.middleware.yaml` for setting up essential middleware services like databases and caches. - - Navigate to the `docker` directory. - - Ensure the `middleware.env` file is created by running `cp middleware.env.example middleware.env` (refer to the `middleware.env.example` file). -2. **Running Middleware Services**: - - Navigate to the `docker` directory. - - Execute `docker compose -f docker-compose.middleware.yaml --profile weaviate -p dify up -d` to start the middleware services. (Change the profile to other vector database if you are not using weaviate) + - Use the `docker-compose.middleware.yaml` for setting up essential middleware services like databases and caches. + - Navigate to the `docker` directory. + - Ensure the `middleware.env` file is created by running `cp middleware.env.example middleware.env` (refer to the `middleware.env.example` file). +1. **Running Middleware Services**: + - Navigate to the `docker` directory. + - Execute `docker compose -f docker-compose.middleware.yaml --profile weaviate -p dify up -d` to start the middleware services. (Change the profile to other vector database if you are not using weaviate) ### Migration for Existing Users For users migrating from the `docker-legacy` setup: 1. **Review Changes**: Familiarize yourself with the new `.env` configuration and Docker Compose setup. -2. **Transfer Customizations**: - - If you have customized configurations such as `docker-compose.yaml`, `ssrf_proxy/squid.conf`, or `nginx/conf.d/default.conf`, you will need to reflect these changes in the `.env` file you create. -3. **Data Migration**: - - Ensure that data from services like databases and caches is backed up and migrated appropriately to the new structure if necessary. +1. **Transfer Customizations**: + - If you have customized configurations such as `docker-compose.yaml`, `ssrf_proxy/squid.conf`, or `nginx/conf.d/default.conf`, you will need to reflect these changes in the `.env` file you create. +1. **Data Migration**: + - Ensure that data from services like databases and caches is backed up and migrated appropriately to the new structure if necessary. ### Overview of `.env` @@ -64,39 +65,49 @@ For users migrating from the `docker-legacy` setup: The `.env.example` file provided in the Docker setup is extensive and covers a wide range of configuration options. It is structured into several sections, each pertaining to different aspects of the application and its services. Here are some of the key sections and variables: 1. **Common Variables**: - - `CONSOLE_API_URL`, `SERVICE_API_URL`: URLs for different API services. - - `APP_WEB_URL`: Frontend application URL. - - `FILES_URL`: Base URL for file downloads and previews. -2. **Server Configuration**: - - `LOG_LEVEL`, `DEBUG`, `FLASK_DEBUG`: Logging and debug settings. - - `SECRET_KEY`: A key for encrypting session cookies and other sensitive data. + - `CONSOLE_API_URL`, `SERVICE_API_URL`: URLs for different API services. + - `APP_WEB_URL`: Frontend application URL. + - `FILES_URL`: Base URL for file downloads and previews. -3. **Database Configuration**: - - `DB_USERNAME`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`, `DB_DATABASE`: PostgreSQL database credentials and connection details. +1. **Server Configuration**: -4. **Redis Configuration**: - - `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`: Redis server connection settings. + - `LOG_LEVEL`, `DEBUG`, `FLASK_DEBUG`: Logging and debug settings. + - `SECRET_KEY`: A key for encrypting session cookies and other sensitive data. -5. **Celery Configuration**: - - `CELERY_BROKER_URL`: Configuration for Celery message broker. +1. **Database Configuration**: -6. **Storage Configuration**: - - `STORAGE_TYPE`, `S3_BUCKET_NAME`, `AZURE_BLOB_ACCOUNT_NAME`: Settings for file storage options like local, S3, Azure Blob, etc. + - `DB_USERNAME`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`, `DB_DATABASE`: PostgreSQL database credentials and connection details. -7. **Vector Database Configuration**: - - `VECTOR_STORE`: Type of vector database (e.g., `weaviate`, `milvus`). - - Specific settings for each vector store like `WEAVIATE_ENDPOINT`, `MILVUS_URI`. +1. **Redis Configuration**: -8. **CORS Configuration**: - - `WEB_API_CORS_ALLOW_ORIGINS`, `CONSOLE_CORS_ALLOW_ORIGINS`: Settings for cross-origin resource sharing. + - `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`: Redis server connection settings. -9. **OpenTelemetry Configuration**: - - `ENABLE_OTEL`: Enable OpenTelemetry collector in api. - - `OTLP_BASE_ENDPOINT`: Endpoint for your OTLP exporter. - -10. **Other Service-Specific Environment Variables**: - - Each service like `nginx`, `redis`, `db`, and vector databases have specific environment variables that are directly referenced in the `docker-compose.yaml`. +1. **Celery Configuration**: + + - `CELERY_BROKER_URL`: Configuration for Celery message broker. + +1. **Storage Configuration**: + + - `STORAGE_TYPE`, `S3_BUCKET_NAME`, `AZURE_BLOB_ACCOUNT_NAME`: Settings for file storage options like local, S3, Azure Blob, etc. + +1. **Vector Database Configuration**: + + - `VECTOR_STORE`: Type of vector database (e.g., `weaviate`, `milvus`). + - Specific settings for each vector store like `WEAVIATE_ENDPOINT`, `MILVUS_URI`. + +1. **CORS Configuration**: + + - `WEB_API_CORS_ALLOW_ORIGINS`, `CONSOLE_CORS_ALLOW_ORIGINS`: Settings for cross-origin resource sharing. + +1. **OpenTelemetry Configuration**: + + - `ENABLE_OTEL`: Enable OpenTelemetry collector in api. + - `OTLP_BASE_ENDPOINT`: Endpoint for your OTLP exporter. + +1. **Other Service-Specific Environment Variables**: + + - Each service like `nginx`, `redis`, `db`, and vector databases have specific environment variables that are directly referenced in the `docker-compose.yaml`. ### Additional Information diff --git a/dify/code/certbot/README.md b/dify/code/certbot/README.md index 21be34b..62b1eee 100644 --- a/dify/code/certbot/README.md +++ b/dify/code/certbot/README.md @@ -2,12 +2,12 @@ ## Short description -docker compose certbot configurations with Backward compatibility (without certbot container). +docker compose certbot configurations with Backward compatibility (without certbot container).\ Use `docker compose --profile certbot up` to use this features. ## The simplest way for launching new servers with SSL certificates -1. Get letsencrypt certs +1. Get letsencrypt certs\ set `.env` values ```properties NGINX_SSL_CERT_FILENAME=fullchain.pem @@ -25,7 +25,7 @@ Use `docker compose --profile certbot up` to use this features. ```shell docker compose exec -it certbot /bin/sh /update-cert.sh ``` -2. Edit `.env` file and `docker compose --profile certbot up` again. +1. Edit `.env` file and `docker compose --profile certbot up` again.\ set `.env` value additionally ```properties NGINX_HTTPS_ENABLED=true @@ -34,7 +34,7 @@ Use `docker compose --profile certbot up` to use this features. ```shell docker compose --profile certbot up -d --no-deps --force-recreate nginx ``` - Then you can access your serve with HTTPS. + Then you can access your serve with HTTPS.\ [https://your_domain.com](https://your_domain.com) ## SSL certificates renewal diff --git a/dify/code/docker-compose-template.yaml b/dify/code/docker-compose-template.yaml index d45f8f8..0e695e4 100644 --- a/dify/code/docker-compose-template.yaml +++ b/dify/code/docker-compose-template.yaml @@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:1.5.1 + image: langgenius/dify-api:1.8.0 restart: always environment: # Use the shared environment variables. @@ -31,7 +31,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:1.5.1 + image: langgenius/dify-api:1.8.0 restart: always environment: # Use the shared environment variables. @@ -55,9 +55,28 @@ services: - ssrf_proxy_network - default + # worker_beat service + # Celery beat for scheduling periodic tasks. + worker_beat: + image: langgenius/dify-api:1.8.0 + restart: always + environment: + # Use the shared environment variables. + <<: *shared-api-worker-env + # Startup mode, 'worker_beat' starts the Celery beat for scheduling periodic tasks. + MODE: beat + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + networks: + - ssrf_proxy_network + - default + # Frontend web application. web: - image: langgenius/dify-web:1.5.1 + image: langgenius/dify-web:1.8.0 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -67,6 +86,7 @@ services: TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} CSP_WHITELIST: ${CSP_WHITELIST:-} ALLOW_EMBED: ${ALLOW_EMBED:-false} + ALLOW_UNSAFE_DATA_SCHEME: ${ALLOW_UNSAFE_DATA_SCHEME:-false} MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai} TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-} @@ -76,6 +96,7 @@ services: MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99} + MAX_TREE_DEPTH: ${MAX_TREE_DEPTH:-50} ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} @@ -142,7 +163,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.1.3-local + image: langgenius/dify-plugin-daemon:0.2.0-local restart: always environment: # Use the shared environment variables. @@ -160,6 +181,8 @@ services: FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} + PLUGIN_STDIO_BUFFER_SIZE: ${PLUGIN_STDIO_BUFFER_SIZE:-1024} + PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880} PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} @@ -189,6 +212,8 @@ services: VOLCENGINE_TOS_ACCESS_KEY: ${PLUGIN_VOLCENGINE_TOS_ACCESS_KEY:-} VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-} VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-} + SENTRY_ENABLED: ${PLUGIN_SENTRY_ENABLED:-false} + SENTRY_DSN: ${PLUGIN_SENTRY_DSN:-} ports: - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}" volumes: @@ -265,7 +290,7 @@ services: NGINX_SSL_CERT_KEY_FILENAME: ${NGINX_SSL_CERT_KEY_FILENAME:-dify.key} NGINX_SSL_PROTOCOLS: ${NGINX_SSL_PROTOCOLS:-TLSv1.1 TLSv1.2 TLSv1.3} NGINX_WORKER_PROCESSES: ${NGINX_WORKER_PROCESSES:-auto} - NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-15M} + NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-100M} NGINX_KEEPALIVE_TIMEOUT: ${NGINX_KEEPALIVE_TIMEOUT:-65} NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s} NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s} @@ -518,7 +543,7 @@ services: milvus-standalone: container_name: milvus-standalone - image: milvusdb/milvus:v2.5.0-beta + image: milvusdb/milvus:v2.5.15 profiles: - milvus command: [ 'milvus', 'run', 'standalone' ] diff --git a/dify/code/docker-compose.middleware.yaml b/dify/code/docker-compose.middleware.yaml index 0b18857..9f7cc72 100644 --- a/dify/code/docker-compose.middleware.yaml +++ b/dify/code/docker-compose.middleware.yaml @@ -20,7 +20,7 @@ services: ports: - "${EXPOSE_POSTGRES_PORT:-5432}:5432" healthcheck: - test: [ "CMD", "pg_isready" ] + test: [ 'CMD', 'pg_isready', '-h', 'db', '-U', '${PGUSER:-postgres}', '-d', '${POSTGRES_DB:-dify}' ] interval: 1s timeout: 3s retries: 30 @@ -71,7 +71,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.1.3-local + image: langgenius/dify-plugin-daemon:0.2.0-local restart: always env_file: - ./middleware.env diff --git a/dify/code/docker-compose.yaml b/dify/code/docker-compose.yaml index 9b65af5..7f24227 100644 --- a/dify/code/docker-compose.yaml +++ b/dify/code/docker-compose.yaml @@ -11,6 +11,10 @@ x-shared-env: &shared-api-worker-env APP_API_URL: ${APP_API_URL:-} APP_WEB_URL: ${APP_WEB_URL:-} FILES_URL: ${FILES_URL:-} + INTERNAL_FILES_URL: ${INTERNAL_FILES_URL:-} + LANG: ${LANG:-en_US.UTF-8} + LC_ALL: ${LC_ALL:-en_US.UTF-8} + PYTHONIOENCODING: ${PYTHONIOENCODING:-utf-8} LOG_LEVEL: ${LOG_LEVEL:-INFO} LOG_FILE: ${LOG_FILE:-/app/logs/server.log} LOG_FILE_MAX_SIZE: ${LOG_FILE_MAX_SIZE:-20} @@ -53,8 +57,11 @@ x-shared-env: &shared-api-worker-env DB_PORT: ${DB_PORT:-5432} DB_DATABASE: ${DB_DATABASE:-dify} SQLALCHEMY_POOL_SIZE: ${SQLALCHEMY_POOL_SIZE:-30} + SQLALCHEMY_MAX_OVERFLOW: ${SQLALCHEMY_MAX_OVERFLOW:-10} SQLALCHEMY_POOL_RECYCLE: ${SQLALCHEMY_POOL_RECYCLE:-3600} SQLALCHEMY_ECHO: ${SQLALCHEMY_ECHO:-false} + SQLALCHEMY_POOL_PRE_PING: ${SQLALCHEMY_POOL_PRE_PING:-false} + SQLALCHEMY_POOL_USE_LIFO: ${SQLALCHEMY_POOL_USE_LIFO:-false} POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-100} POSTGRES_SHARED_BUFFERS: ${POSTGRES_SHARED_BUFFERS:-128MB} POSTGRES_WORK_MEM: ${POSTGRES_WORK_MEM:-4MB} @@ -65,6 +72,10 @@ x-shared-env: &shared-api-worker-env REDIS_USERNAME: ${REDIS_USERNAME:-} REDIS_PASSWORD: ${REDIS_PASSWORD:-difyai123456} REDIS_USE_SSL: ${REDIS_USE_SSL:-false} + REDIS_SSL_CERT_REQS: ${REDIS_SSL_CERT_REQS:-CERT_NONE} + REDIS_SSL_CA_CERTS: ${REDIS_SSL_CA_CERTS:-} + REDIS_SSL_CERTFILE: ${REDIS_SSL_CERTFILE:-} + REDIS_SSL_KEYFILE: ${REDIS_SSL_KEYFILE:-} REDIS_DB: ${REDIS_DB:-0} REDIS_USE_SENTINEL: ${REDIS_USE_SENTINEL:-false} REDIS_SENTINELS: ${REDIS_SENTINELS:-} @@ -76,6 +87,7 @@ x-shared-env: &shared-api-worker-env REDIS_CLUSTERS: ${REDIS_CLUSTERS:-} REDIS_CLUSTERS_PASSWORD: ${REDIS_CLUSTERS_PASSWORD:-} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://:difyai123456@redis:6379/1} + CELERY_BACKEND: ${CELERY_BACKEND:-redis} BROKER_USE_SSL: ${BROKER_USE_SSL:-false} CELERY_USE_SENTINEL: ${CELERY_USE_SENTINEL:-false} CELERY_SENTINEL_MASTER_NAME: ${CELERY_SENTINEL_MASTER_NAME:-} @@ -86,6 +98,10 @@ x-shared-env: &shared-api-worker-env STORAGE_TYPE: ${STORAGE_TYPE:-opendal} OPENDAL_SCHEME: ${OPENDAL_SCHEME:-fs} OPENDAL_FS_ROOT: ${OPENDAL_FS_ROOT:-storage} + CLICKZETTA_VOLUME_TYPE: ${CLICKZETTA_VOLUME_TYPE:-user} + CLICKZETTA_VOLUME_NAME: ${CLICKZETTA_VOLUME_NAME:-} + CLICKZETTA_VOLUME_TABLE_PREFIX: ${CLICKZETTA_VOLUME_TABLE_PREFIX:-dataset_} + CLICKZETTA_VOLUME_DIFY_PREFIX: ${CLICKZETTA_VOLUME_DIFY_PREFIX:-dify_km} S3_ENDPOINT: ${S3_ENDPOINT:-} S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET_NAME: ${S3_BUCKET_NAME:-difyai} @@ -132,6 +148,7 @@ x-shared-env: &shared-api-worker-env SUPABASE_API_KEY: ${SUPABASE_API_KEY:-your-access-key} SUPABASE_URL: ${SUPABASE_URL:-your-server-url} VECTOR_STORE: ${VECTOR_STORE:-weaviate} + VECTOR_INDEX_NAME_PREFIX: ${VECTOR_INDEX_NAME_PREFIX:-Vector_index} WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080} WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} QDRANT_URL: ${QDRANT_URL:-http://qdrant:6333} @@ -253,6 +270,14 @@ x-shared-env: &shared-api-worker-env ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_USERNAME:-elastic} ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-elastic} KIBANA_PORT: ${KIBANA_PORT:-5601} + ELASTICSEARCH_USE_CLOUD: ${ELASTICSEARCH_USE_CLOUD:-false} + ELASTICSEARCH_CLOUD_URL: ${ELASTICSEARCH_CLOUD_URL:-YOUR-ELASTICSEARCH_CLOUD_URL} + ELASTICSEARCH_API_KEY: ${ELASTICSEARCH_API_KEY:-YOUR-ELASTICSEARCH_API_KEY} + ELASTICSEARCH_VERIFY_CERTS: ${ELASTICSEARCH_VERIFY_CERTS:-False} + ELASTICSEARCH_CA_CERTS: ${ELASTICSEARCH_CA_CERTS:-} + ELASTICSEARCH_REQUEST_TIMEOUT: ${ELASTICSEARCH_REQUEST_TIMEOUT:-100000} + ELASTICSEARCH_RETRY_ON_TIMEOUT: ${ELASTICSEARCH_RETRY_ON_TIMEOUT:-True} + ELASTICSEARCH_MAX_RETRIES: ${ELASTICSEARCH_MAX_RETRIES:-10} BAIDU_VECTOR_DB_ENDPOINT: ${BAIDU_VECTOR_DB_ENDPOINT:-http://127.0.0.1:5287} BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS: ${BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS:-30000} BAIDU_VECTOR_DB_ACCOUNT: ${BAIDU_VECTOR_DB_ACCOUNT:-root} @@ -296,6 +321,19 @@ x-shared-env: &shared-api-worker-env TABLESTORE_INSTANCE_NAME: ${TABLESTORE_INSTANCE_NAME:-instance-name} TABLESTORE_ACCESS_KEY_ID: ${TABLESTORE_ACCESS_KEY_ID:-xxx} TABLESTORE_ACCESS_KEY_SECRET: ${TABLESTORE_ACCESS_KEY_SECRET:-xxx} + TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE: ${TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE:-false} + CLICKZETTA_USERNAME: ${CLICKZETTA_USERNAME:-} + CLICKZETTA_PASSWORD: ${CLICKZETTA_PASSWORD:-} + CLICKZETTA_INSTANCE: ${CLICKZETTA_INSTANCE:-} + CLICKZETTA_SERVICE: ${CLICKZETTA_SERVICE:-api.clickzetta.com} + CLICKZETTA_WORKSPACE: ${CLICKZETTA_WORKSPACE:-quick_start} + CLICKZETTA_VCLUSTER: ${CLICKZETTA_VCLUSTER:-default_ap} + CLICKZETTA_SCHEMA: ${CLICKZETTA_SCHEMA:-dify} + CLICKZETTA_BATCH_SIZE: ${CLICKZETTA_BATCH_SIZE:-100} + CLICKZETTA_ENABLE_INVERTED_INDEX: ${CLICKZETTA_ENABLE_INVERTED_INDEX:-true} + CLICKZETTA_ANALYZER_TYPE: ${CLICKZETTA_ANALYZER_TYPE:-chinese} + CLICKZETTA_ANALYZER_MODE: ${CLICKZETTA_ANALYZER_MODE:-smart} + CLICKZETTA_VECTOR_DISTANCE_FUNCTION: ${CLICKZETTA_VECTOR_DISTANCE_FUNCTION:-cosine_distance} UPLOAD_FILE_SIZE_LIMIT: ${UPLOAD_FILE_SIZE_LIMIT:-15} UPLOAD_FILE_BATCH_LIMIT: ${UPLOAD_FILE_BATCH_LIMIT:-5} ETL_TYPE: ${ETL_TYPE:-dify} @@ -314,6 +352,8 @@ x-shared-env: &shared-api-worker-env API_SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} API_SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} WEB_SENTRY_DSN: ${WEB_SENTRY_DSN:-} + PLUGIN_SENTRY_ENABLED: ${PLUGIN_SENTRY_ENABLED:-false} + PLUGIN_SENTRY_DSN: ${PLUGIN_SENTRY_DSN:-} NOTION_INTEGRATION_TYPE: ${NOTION_INTEGRATION_TYPE:-public} NOTION_CLIENT_SECRET: ${NOTION_CLIENT_SECRET:-} NOTION_CLIENT_ID: ${NOTION_CLIENT_ID:-} @@ -332,6 +372,8 @@ x-shared-env: &shared-api-worker-env INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000} INVITE_EXPIRY_HOURS: ${INVITE_EXPIRY_HOURS:-72} RESET_PASSWORD_TOKEN_EXPIRY_MINUTES: ${RESET_PASSWORD_TOKEN_EXPIRY_MINUTES:-5} + CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES: ${CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES:-5} + OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES: ${OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES:-5} CODE_EXECUTION_ENDPOINT: ${CODE_EXECUTION_ENDPOINT:-http://sandbox:8194} CODE_EXECUTION_API_KEY: ${CODE_EXECUTION_API_KEY:-dify-sandbox} CODE_MAX_NUMBER: ${CODE_MAX_NUMBER:-9223372036854775807} @@ -353,6 +395,13 @@ x-shared-env: &shared-api-worker-env WORKFLOW_PARALLEL_DEPTH_LIMIT: ${WORKFLOW_PARALLEL_DEPTH_LIMIT:-3} WORKFLOW_FILE_UPLOAD_LIMIT: ${WORKFLOW_FILE_UPLOAD_LIMIT:-10} WORKFLOW_NODE_EXECUTION_STORAGE: ${WORKFLOW_NODE_EXECUTION_STORAGE:-rdbms} + CORE_WORKFLOW_EXECUTION_REPOSITORY: ${CORE_WORKFLOW_EXECUTION_REPOSITORY:-core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository} + CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY:-core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository} + API_WORKFLOW_RUN_REPOSITORY: ${API_WORKFLOW_RUN_REPOSITORY:-repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository} + API_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${API_WORKFLOW_NODE_EXECUTION_REPOSITORY:-repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository} + WORKFLOW_LOG_CLEANUP_ENABLED: ${WORKFLOW_LOG_CLEANUP_ENABLED:-false} + WORKFLOW_LOG_RETENTION_DAYS: ${WORKFLOW_LOG_RETENTION_DAYS:-30} + WORKFLOW_LOG_CLEANUP_BATCH_SIZE: ${WORKFLOW_LOG_CLEANUP_BATCH_SIZE:-100} HTTP_REQUEST_NODE_MAX_BINARY_SIZE: ${HTTP_REQUEST_NODE_MAX_BINARY_SIZE:-10485760} HTTP_REQUEST_NODE_MAX_TEXT_SIZE: ${HTTP_REQUEST_NODE_MAX_TEXT_SIZE:-1048576} HTTP_REQUEST_NODE_SSL_VERIFY: ${HTTP_REQUEST_NODE_SSL_VERIFY:-True} @@ -364,6 +413,8 @@ x-shared-env: &shared-api-worker-env MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99} TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} + ALLOW_UNSAFE_DATA_SCHEME: ${ALLOW_UNSAFE_DATA_SCHEME:-false} + MAX_TREE_DEPTH: ${MAX_TREE_DEPTH:-50} POSTGRES_USER: ${POSTGRES_USER:-${DB_USERNAME}} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-${DB_PASSWORD}} POSTGRES_DB: ${POSTGRES_DB:-${DB_DATABASE}} @@ -420,7 +471,7 @@ x-shared-env: &shared-api-worker-env NGINX_SSL_CERT_KEY_FILENAME: ${NGINX_SSL_CERT_KEY_FILENAME:-dify.key} NGINX_SSL_PROTOCOLS: ${NGINX_SSL_PROTOCOLS:-TLSv1.1 TLSv1.2 TLSv1.3} NGINX_WORKER_PROCESSES: ${NGINX_WORKER_PROCESSES:-auto} - NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-15M} + NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-100M} NGINX_KEEPALIVE_TIMEOUT: ${NGINX_KEEPALIVE_TIMEOUT:-65} NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s} NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s} @@ -465,6 +516,8 @@ x-shared-env: &shared-api-worker-env MARKETPLACE_ENABLED: ${MARKETPLACE_ENABLED:-true} MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + PLUGIN_STDIO_BUFFER_SIZE: ${PLUGIN_STDIO_BUFFER_SIZE:-1024} + PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880} PLUGIN_PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} @@ -498,6 +551,8 @@ x-shared-env: &shared-api-worker-env PLUGIN_VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-} PLUGIN_VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-} ENABLE_OTEL: ${ENABLE_OTEL:-false} + OTLP_TRACE_ENDPOINT: ${OTLP_TRACE_ENDPOINT:-} + OTLP_METRIC_ENDPOINT: ${OTLP_METRIC_ENDPOINT:-} OTLP_BASE_ENDPOINT: ${OTLP_BASE_ENDPOINT:-http://localhost:4318} OTLP_API_KEY: ${OTLP_API_KEY:-} OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-} @@ -513,11 +568,21 @@ x-shared-env: &shared-api-worker-env QUEUE_MONITOR_THRESHOLD: ${QUEUE_MONITOR_THRESHOLD:-200} QUEUE_MONITOR_ALERT_EMAILS: ${QUEUE_MONITOR_ALERT_EMAILS:-} QUEUE_MONITOR_INTERVAL: ${QUEUE_MONITOR_INTERVAL:-30} + SWAGGER_UI_ENABLED: ${SWAGGER_UI_ENABLED:-true} + SWAGGER_UI_PATH: ${SWAGGER_UI_PATH:-/swagger-ui.html} + ENABLE_CLEAN_EMBEDDING_CACHE_TASK: ${ENABLE_CLEAN_EMBEDDING_CACHE_TASK:-false} + ENABLE_CLEAN_UNUSED_DATASETS_TASK: ${ENABLE_CLEAN_UNUSED_DATASETS_TASK:-false} + ENABLE_CREATE_TIDB_SERVERLESS_TASK: ${ENABLE_CREATE_TIDB_SERVERLESS_TASK:-false} + ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK: ${ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK:-false} + ENABLE_CLEAN_MESSAGES: ${ENABLE_CLEAN_MESSAGES:-false} + ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK: ${ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK:-false} + ENABLE_DATASETS_QUEUE_MONITOR: ${ENABLE_DATASETS_QUEUE_MONITOR:-false} + ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK: ${ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK:-true} services: # API service api: - image: langgenius/dify-api:1.5.1 + image: langgenius/dify-api:1.8.0 restart: always environment: # Use the shared environment variables. @@ -546,7 +611,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:1.5.1 + image: langgenius/dify-api:1.8.0 restart: always environment: # Use the shared environment variables. @@ -570,9 +635,28 @@ services: - ssrf_proxy_network - default + # worker_beat service + # Celery beat for scheduling periodic tasks. + worker_beat: + image: langgenius/dify-api:1.8.0 + restart: always + environment: + # Use the shared environment variables. + <<: *shared-api-worker-env + # Startup mode, 'worker_beat' starts the Celery beat for scheduling periodic tasks. + MODE: beat + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + networks: + - ssrf_proxy_network + - default + # Frontend web application. web: - image: langgenius/dify-web:1.5.1 + image: langgenius/dify-web:1.8.0 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -582,6 +666,7 @@ services: TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} CSP_WHITELIST: ${CSP_WHITELIST:-} ALLOW_EMBED: ${ALLOW_EMBED:-false} + ALLOW_UNSAFE_DATA_SCHEME: ${ALLOW_UNSAFE_DATA_SCHEME:-false} MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai} TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-} @@ -591,6 +676,7 @@ services: MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99} + MAX_TREE_DEPTH: ${MAX_TREE_DEPTH:-50} ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} @@ -667,7 +753,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.1.3-local + image: langgenius/dify-plugin-daemon:0.2.0-local restart: always environment: # Use the shared environment variables. @@ -685,6 +771,8 @@ services: FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} + PLUGIN_STDIO_BUFFER_SIZE: ${PLUGIN_STDIO_BUFFER_SIZE:-1024} + PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880} PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} @@ -714,6 +802,8 @@ services: VOLCENGINE_TOS_ACCESS_KEY: ${PLUGIN_VOLCENGINE_TOS_ACCESS_KEY:-} VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-} VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-} + SENTRY_ENABLED: ${PLUGIN_SENTRY_ENABLED:-false} + SENTRY_DSN: ${PLUGIN_SENTRY_DSN:-} volumes: - ./volumes/plugin_daemon:/app/storage depends_on: @@ -802,7 +892,7 @@ services: NGINX_SSL_CERT_KEY_FILENAME: ${NGINX_SSL_CERT_KEY_FILENAME:-dify.key} NGINX_SSL_PROTOCOLS: ${NGINX_SSL_PROTOCOLS:-TLSv1.1 TLSv1.2 TLSv1.3} NGINX_WORKER_PROCESSES: ${NGINX_WORKER_PROCESSES:-auto} - NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-15M} + NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-100M} NGINX_KEEPALIVE_TIMEOUT: ${NGINX_KEEPALIVE_TIMEOUT:-65} NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s} NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s} @@ -1055,7 +1145,7 @@ services: - milvus milvus-standalone: - image: milvusdb/milvus:v2.5.0-beta + image: milvusdb/milvus:v2.5.15 profiles: - milvus command: [ 'milvus', 'run', 'standalone' ] diff --git a/dify/code/nginx/conf.d/default.conf.template b/dify/code/nginx/conf.d/default.conf.template index a458412..48d7da8 100644 --- a/dify/code/nginx/conf.d/default.conf.template +++ b/dify/code/nginx/conf.d/default.conf.template @@ -39,7 +39,10 @@ server { proxy_pass http://web:3000; include proxy.conf; } - + location /mcp { + proxy_pass http://api:5001; + include proxy.conf; + } # placeholder for acme challenge location ${ACME_CHALLENGE_LOCATION} diff --git a/dify/update.js b/dify/update.js index faff6b0..527f581 100644 --- a/dify/update.js +++ b/dify/update.js @@ -4,3 +4,9 @@ await utils.cloneOrPullRepo({ repo: "https://github.com/langgenius/dify.git" }); await utils.copyDir("./repo/docker", "./code"); await utils.removeContainerNames("./code/docker-compose.yaml"); await utils.removePorts("./code/docker-compose.yaml"); + +await utils.searchReplace( + "./code/.env.example", + "APP_WEB_URL=", + "APP_WEB_URL=https://$(PRIMARY_DOMAIN)" +); diff --git a/dify/update.sh b/dify/update.sh deleted file mode 100644 index 433d23c..0000000 --- a/dify/update.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -if [ ! -d "./repo" ]; then - git clone --depth 1 --branch main --single-branch https://github.com/langgenius/dify.git repo -else - cd repo - git pull - cd .. -fi - -cp -r ./repo/docker/. ./code - - diff --git a/plane/update.js b/plane/update.js index 5abe74f..a5ef43f 100644 --- a/plane/update.js +++ b/plane/update.js @@ -1,5 +1,11 @@ import utils from "../utils.js"; +console.log( + "Plan file structure has changed. This script needs to be reworked." +); + +process.exit(); + await utils.cloneOrPullRepo({ repo: "https://github.com/makeplane/plane.git", path: "./repo", @@ -11,3 +17,21 @@ await utils.renameFile("./code/variables.env", "./code/.env.example"); await utils.removeContainerNames("./code/docker-compose.yml"); await utils.removePorts("./code/docker-compose.yml"); + +await utils.searchReplace( + "./code/.env.example", + "APP_DOMAIN=localhost", + "APP_DOMAIN=$(PRIMARY_DOMAIN)" +); + +await utils.searchReplace( + "./code/.env.example", + "WEB_URL=http://${APP_DOMAIN}", + "WEB_URL=https://$(PRIMARY_DOMAIN)" +); + +await utils.searchReplace( + "./code/.env.example", + "CORS_ALLOWED_ORIGINS=http://${APP_DOMAIN}", + "CORS_ALLOWED_ORIGINS=https://$(PRIMARY_DOMAIN)" +); diff --git a/plane/update.sh b/plane/update.sh deleted file mode 100644 index 3bd7313..0000000 --- a/plane/update.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [ ! -d "./repo" ]; then - git clone --depth 1 --branch preview --single-branch https://github.com/makeplane/plane.git repo -else - cd repo - git pull - cd .. -fi - -cp -r ./repo/deploy/selfhost/. ./code -mv ./code/variables.env ./code/.env.example - - diff --git a/supabase/code/.env.example b/supabase/code/.env.example index 8ee5f75..fcb6310 100644 --- a/supabase/code/.env.example +++ b/supabase/code/.env.example @@ -59,7 +59,7 @@ PGRST_DB_SCHEMAS=public,storage,graphql_public ############ ## General -SITE_URL=http://localhost:3000 +SITE_URL=https://$(PRIMARY_DOMAIN) ADDITIONAL_REDIRECT_URLS= JWT_EXPIRY=3600 DISABLE_SIGNUP=false diff --git a/supabase/code/docker-compose.yml b/supabase/code/docker-compose.yml index 9a4e48e..b1fbf86 100644 --- a/supabase/code/docker-compose.yml +++ b/supabase/code/docker-compose.yml @@ -75,7 +75,7 @@ services: /docker-entrypoint.sh kong docker-start' auth: - image: supabase/gotrue:v2.176.1 + image: supabase/gotrue:v2.177.0 restart: unless-stopped healthcheck: test: @@ -221,7 +221,7 @@ services: # To use S3 backed storage: docker compose -f docker-compose.yml -f docker-compose.s3.yml up storage: - image: supabase/storage-api:v1.24.7 + image: supabase/storage-api:v1.25.7 restart: unless-stopped volumes: - ./volumes/storage:/var/lib/storage:z @@ -279,7 +279,7 @@ services: IMGPROXY_ENABLE_WEBP_DETECTION: ${IMGPROXY_ENABLE_WEBP_DETECTION} meta: - image: supabase/postgres-meta:v0.89.3 + image: supabase/postgres-meta:v0.91.0 restart: unless-stopped depends_on: db: @@ -429,7 +429,7 @@ services: # Update the DATABASE_URL if you are using an external Postgres database supavisor: - image: supabase/supavisor:2.5.6 + image: supabase/supavisor:2.5.7 restart: unless-stopped volumes: - ./volumes/pooler/pooler.exs:/etc/pooler/pooler.exs:ro,z diff --git a/supabase/update.js b/supabase/update.js index ca3df4f..fdcaa71 100644 --- a/supabase/update.js +++ b/supabase/update.js @@ -5,3 +5,9 @@ await utils.copyDir("./repo/docker", "./code"); await utils.removeContainerNames("./code/docker-compose.yml"); await utils.removePorts("./code/docker-compose.yml"); + +await utils.searchReplace( + "./code/.env.example", + "SITE_URL=http://localhost:3000", + "SITE_URL=https://$(PRIMARY_DOMAIN)" +); diff --git a/supabase/update.sh b/supabase/update.sh deleted file mode 100644 index 24abd5a..0000000 --- a/supabase/update.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ ! -d "./repo" ]; then - git clone --depth 1 --branch master --single-branch https://github.com/supabase/supabase repo -else - cd repo - git pull - cd .. -fi - -cp -r ./repo/docker/. ./code diff --git a/twenty/code/twenty-website/Dockerfile b/twenty/code/twenty-website/Dockerfile index c760216..2ff7cb8 100644 --- a/twenty/code/twenty-website/Dockerfile +++ b/twenty/code/twenty-website/Dockerfile @@ -1,4 +1,4 @@ -FROM node:22-alpine as twenty-website-build +FROM node:24-alpine AS twenty-website-build WORKDIR /app @@ -7,6 +7,7 @@ COPY ./package.json . COPY ./yarn.lock . COPY ./.yarnrc.yml . COPY ./.yarn/releases /app/.yarn/releases +COPY ./.yarn/patches /app/.yarn/patches COPY ./tools/eslint-rules /app/tools/eslint-rules COPY ./packages/twenty-ui/package.json /app/packages/twenty-ui/ COPY ./packages/twenty-shared/package.json /app/packages/twenty-shared/ @@ -23,7 +24,7 @@ COPY ./packages/twenty-ui /app/packages/twenty-ui COPY ./packages/twenty-website /app/packages/twenty-website RUN npx nx build twenty-website -FROM node:22-alpine as twenty-website +FROM node:24-alpine AS twenty-website WORKDIR /app/packages/twenty-website diff --git a/twenty/code/twenty/Dockerfile b/twenty/code/twenty/Dockerfile index d5f7197..7818be7 100644 --- a/twenty/code/twenty/Dockerfile +++ b/twenty/code/twenty/Dockerfile @@ -1,11 +1,12 @@ # Base image for common dependencies -FROM node:22-alpine as common-deps +FROM node:24-alpine AS common-deps WORKDIR /app # Copy only the necessary files for dependency resolution COPY ./package.json ./yarn.lock ./.yarnrc.yml ./tsconfig.base.json ./nx.json /app/ COPY ./.yarn/releases /app/.yarn/releases +COPY ./.yarn/patches /app/.yarn/patches COPY ./.prettierrc /app/ COPY ./packages/twenty-emails/package.json /app/packages/twenty-emails/ @@ -20,7 +21,7 @@ RUN yarn && yarn cache clean && npx nx reset # Build the back -FROM common-deps as twenty-server-build +FROM common-deps AS twenty-server-build # Copy sourcecode after installing dependences to accelerate subsequents builds COPY ./packages/twenty-emails /app/packages/twenty-emails @@ -28,17 +29,11 @@ COPY ./packages/twenty-shared /app/packages/twenty-shared COPY ./packages/twenty-server /app/packages/twenty-server RUN npx nx run twenty-server:build -RUN mv /app/packages/twenty-server/dist /app/packages/twenty-server/build -RUN npx nx run twenty-server:build:packageJson -RUN mv /app/packages/twenty-server/dist/package.json /app/packages/twenty-server/package.json -RUN rm -rf /app/packages/twenty-server/dist -RUN mv /app/packages/twenty-server/build /app/packages/twenty-server/dist RUN yarn workspaces focus --production twenty-emails twenty-shared twenty-server - # Build the front -FROM common-deps as twenty-front-build +FROM common-deps AS twenty-front-build ARG REACT_APP_SERVER_BASE_URL @@ -49,7 +44,7 @@ RUN npx nx build twenty-front # Final stage: Run the application -FROM node:22-alpine as twenty +FROM node:24-alpine AS twenty # Used to run healthcheck in docker RUN apk add --no-cache curl jq @@ -63,10 +58,10 @@ RUN chmod +x /app/entrypoint.sh WORKDIR /app/packages/twenty-server ARG REACT_APP_SERVER_BASE_URL -ENV REACT_APP_SERVER_BASE_URL $REACT_APP_SERVER_BASE_URL +ENV REACT_APP_SERVER_BASE_URL=$REACT_APP_SERVER_BASE_URL ARG APP_VERSION -ENV APP_VERSION $APP_VERSION +ENV APP_VERSION=$APP_VERSION # Copy built applications from previous stages COPY --chown=1000 --from=twenty-server-build /app /app diff --git a/twenty/code/twenty/entrypoint.sh b/twenty/code/twenty/entrypoint.sh index 9990491..0175a0a 100755 --- a/twenty/code/twenty/entrypoint.sh +++ b/twenty/code/twenty/entrypoint.sh @@ -19,8 +19,12 @@ setup_and_migrate_db() { if [ "$db_count" = "0" ]; then echo "Database ${PGDATABASE} does not exist, creating..." PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d postgres -c "CREATE DATABASE \"${PGDATABASE}\"" + fi - # Run setup and migration scripts + # Run setup and migration scripts + has_schema=$(PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d ${PGDATABASE} -tAc "SELECT EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'core')") + if [ "$has_schema" = "f" ]; then + echo "Database appears to be empty, running migrations." NODE_OPTIONS="--max-old-space-size=1500" tsx ./scripts/setup-db.ts yarn database:migrate:prod fi diff --git a/twenty/update.sh b/twenty/update.sh deleted file mode 100644 index 468d076..0000000 --- a/twenty/update.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -if [ ! -d "./repo" ]; then - git clone --depth 1 --branch main --single-branch git@github.com:twentyhq/twenty.git repo -else - cd repo - git pull - cd .. -fi - -cp -r ./repo/packages/twenty-docker/. ./code - -