run updates

appwrite/code/.env.example

@@ -6,7 +6,7 @@ _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled
 _APP_OPTIONS_ROUTER_FORCE_HTTPS=disabled
 _APP_OPTIONS_ROUTER_PROTECTION=disabled
 _APP_OPENSSL_KEY_V1=your-secret-key
-_APP_DOMAIN=localhost
+_APP_DOMAIN=$(PRIMARY_DOMAIN)localhost
 _APP_CUSTOM_DOMAIN_DENY_LIST=example.com,test.com,app.example.com
 _APP_DOMAIN_FUNCTIONS=functions.localhost
 _APP_DOMAIN_SITES=sites.localhost

appwrite/update.js

@@ -11,3 +11,9 @@ await utils.downloadFile(
 
 await utils.removeContainerNames("./code/docker-compose.yml");
 await utils.removePorts("./code/docker-compose.yml");
+
+await utils.searchReplace(
+  "./code/.env.example",
+  "_APP_DOMAIN=",
+  "_APP_DOMAIN=$(PRIMARY_DOMAIN)"
+);

appwrite/update.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-if [ ! -d "./repo" ]; then
-    git clone --depth 1 --branch main --single-branch https://github.com/appwrite/appwrite.git repo
-else
-    cd repo
-    git pull
-    cd ..
-fi
-
-curl -s https://appwrite.io/install/compose > ./code/docker-compose.yml
-curl -s https://appwrite.io/install/env > ./code/.env.example
-
-

dify/code/.env.example

@@ -34,7 +34,7 @@ APP_API_URL=
 # used to display WebAPP API Base Url to the front-end.
 # If empty, it is the same domain.
 # Example: https://app.dify.ai
-APP_WEB_URL=
+APP_WEB_URL=https://$(PRIMARY_DOMAIN)
 
 # File preview or download Url prefix.
 # used to display File preview or download Url to the front-end or as Multi-model inputs;
@@ -47,6 +47,16 @@ APP_WEB_URL=
 #   ensuring port 5001 is externally accessible (see docker-compose.yaml).
 FILES_URL=
 
+# INTERNAL_FILES_URL is used for plugin daemon communication within Docker network.
+# Set this to the internal Docker service URL for proper plugin file access.
+# Example: INTERNAL_FILES_URL=http://api:5001
+INTERNAL_FILES_URL=
+
+# Ensure UTF-8 encoding
+LANG=en_US.UTF-8
+LC_ALL=en_US.UTF-8
+PYTHONIOENCODING=utf-8
+
 # ------------------------------
 # Server Configuration
 # ------------------------------
@@ -205,10 +215,16 @@ DB_DATABASE=dify
 # The size of the database connection pool.
 # The default is 30 connections, which can be appropriately increased.
 SQLALCHEMY_POOL_SIZE=30
+# The default is 10 connections, which allows temporary overflow beyond the pool size.
+SQLALCHEMY_MAX_OVERFLOW=10
 # Database connection pool recycling time, the default is 3600 seconds.
 SQLALCHEMY_POOL_RECYCLE=3600
 # Whether to print SQL, default is false.
 SQLALCHEMY_ECHO=false
+# If True, will test connections for liveness upon each checkout
+SQLALCHEMY_POOL_PRE_PING=false
+# Whether to enable the Last in first out option or use default FIFO queue if is false
+SQLALCHEMY_POOL_USE_LIFO=false
 
 # Maximum number of connections to the database
 # Default is 100
@@ -250,6 +266,15 @@ REDIS_PORT=6379
 REDIS_USERNAME=
 REDIS_PASSWORD=difyai123456
 REDIS_USE_SSL=false
+# SSL configuration for Redis (when REDIS_USE_SSL=true)
+REDIS_SSL_CERT_REQS=CERT_NONE
+# Options: CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
+REDIS_SSL_CA_CERTS=
+# Path to CA certificate file for SSL verification
+REDIS_SSL_CERTFILE=
+# Path to client certificate file for SSL authentication
+REDIS_SSL_KEYFILE=
+# Path to client private key file for SSL authentication
 REDIS_DB=0
 
 # Whether to use Redis Sentinel mode.
@@ -274,12 +299,14 @@ REDIS_CLUSTERS_PASSWORD=
 # Celery Configuration
 # ------------------------------
 
-# Use redis as the broker, and redis db 1 for celery broker.
+# Use standalone redis as the broker, and redis db 1 for celery broker. (redis_username is usually set by defualt as empty)
-# Format as follows: `redis://<redis_username>:<redis_password>@<redis_host>:<redis_port>/<redis_database>`
+# Format as follows: `redis://<redis_username>:<redis_password>@<redis_host>:<redis_port>/<redis_database>`.
 # Example: redis://:difyai123456@redis:6379/1
-# If use Redis Sentinel, format as follows: `sentinel://<sentinel_username>:<sentinel_password>@<sentinel_host>:<sentinel_port>/<redis_database>`
+# If use Redis Sentinel, format as follows: `sentinel://<redis_username>:<redis_password>@<sentinel_host1>:<sentinel_port>/<redis_database>`
-# Example: sentinel://localhost:26379/1;sentinel://localhost:26380/1;sentinel://localhost:26381/1
+# For high availability, you can configure multiple Sentinel nodes (if provided) separated by semicolons like below example:
+# Example: sentinel://:difyai123456@localhost:26379/1;sentinel://:difyai12345@localhost:26379/1;sentinel://:difyai12345@localhost:26379/1
 CELERY_BROKER_URL=redis://:difyai123456@redis:6379/1
+CELERY_BACKEND=redis
 BROKER_USE_SSL=false
 
 # If you are using Redis Sentinel for high availability, configure the following settings.
@@ -317,6 +344,25 @@ OPENDAL_SCHEME=fs
 # Configurations for OpenDAL Local File System.
 OPENDAL_FS_ROOT=storage
 
+# ClickZetta Volume Configuration (for storage backend)
+# To use ClickZetta Volume as storage backend, set STORAGE_TYPE=clickzetta-volume
+# Note: ClickZetta Volume will reuse the existing CLICKZETTA_* connection parameters
+
+# Volume type selection (three types available):
+# - user: Personal/small team use, simple config, user-level permissions
+# - table: Enterprise multi-tenant, smart routing, table-level + user-level permissions
+# - external: Data lake integration, external storage connection, volume-level + storage-level permissions
+CLICKZETTA_VOLUME_TYPE=user
+
+# External Volume name (required only when TYPE=external)
+CLICKZETTA_VOLUME_NAME=
+
+# Table Volume table prefix (used only when TYPE=table)
+CLICKZETTA_VOLUME_TABLE_PREFIX=dataset_
+
+# Dify file directory prefix (isolates from other apps, recommended to keep default)
+CLICKZETTA_VOLUME_DIFY_PREFIX=dify_km
+
 # S3 Configuration
 #
 S3_ENDPOINT=
@@ -400,8 +446,10 @@ SUPABASE_URL=your-server-url
 # ------------------------------
 
 # The type of vector store to use.
-# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`, `opengauss`, `tablestore`,`vastbase`,`tidb`,`tidb_on_qdrant`,`baidu`,`lindorm`,`huawei_cloud`,`upstash`, `matrixone`.
+# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`, `opengauss`, `tablestore`,`vastbase`,`tidb`,`tidb_on_qdrant`,`baidu`,`lindorm`,`huawei_cloud`,`upstash`, `matrixone`, `clickzetta`.
 VECTOR_STORE=weaviate
+# Prefix used to create collection name in vector database
+VECTOR_INDEX_NAME_PREFIX=Vector_index
 
 # The Weaviate endpoint URL. Only available when VECTOR_STORE is `weaviate`.
 WEAVIATE_ENDPOINT=http://weaviate:8080
@@ -565,6 +613,17 @@ ELASTICSEARCH_USERNAME=elastic
 ELASTICSEARCH_PASSWORD=elastic
 KIBANA_PORT=5601
 
+# Using ElasticSearch Cloud Serverless, or not.
+ELASTICSEARCH_USE_CLOUD=false
+ELASTICSEARCH_CLOUD_URL=YOUR-ELASTICSEARCH_CLOUD_URL
+ELASTICSEARCH_API_KEY=YOUR-ELASTICSEARCH_API_KEY
+
+ELASTICSEARCH_VERIFY_CERTS=False
+ELASTICSEARCH_CA_CERTS=
+ELASTICSEARCH_REQUEST_TIMEOUT=100000
+ELASTICSEARCH_RETRY_ON_TIMEOUT=True
+ELASTICSEARCH_MAX_RETRIES=10
+
 # baidu vector configurations, only available when VECTOR_STORE is `baidu`
 BAIDU_VECTOR_DB_ENDPOINT=http://127.0.0.1:5287
 BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS=30000
@@ -624,6 +683,21 @@ TABLESTORE_ENDPOINT=https://instance-name.cn-hangzhou.ots.aliyuncs.com
 TABLESTORE_INSTANCE_NAME=instance-name
 TABLESTORE_ACCESS_KEY_ID=xxx
 TABLESTORE_ACCESS_KEY_SECRET=xxx
+TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE=false
+
+# Clickzetta configuration, only available when VECTOR_STORE is `clickzetta`
+CLICKZETTA_USERNAME=
+CLICKZETTA_PASSWORD=
+CLICKZETTA_INSTANCE=
+CLICKZETTA_SERVICE=api.clickzetta.com
+CLICKZETTA_WORKSPACE=quick_start
+CLICKZETTA_VCLUSTER=default_ap
+CLICKZETTA_SCHEMA=dify
+CLICKZETTA_BATCH_SIZE=100
+CLICKZETTA_ENABLE_INVERTED_INDEX=true
+CLICKZETTA_ANALYZER_TYPE=chinese
+CLICKZETTA_ANALYZER_MODE=smart
+CLICKZETTA_VECTOR_DISTANCE_FUNCTION=cosine_distance
 
 # ------------------------------
 # Knowledge Configuration
@@ -705,6 +779,12 @@ API_SENTRY_PROFILES_SAMPLE_RATE=1.0
 # If not set, Sentry error reporting will be disabled.
 WEB_SENTRY_DSN=
 
+# Plugin_daemon Service Sentry DSN address, default is empty, when empty,
+# all monitoring information is not reported to Sentry.
+# If not set, Sentry error reporting will be disabled.
+PLUGIN_SENTRY_ENABLED=false
+PLUGIN_SENTRY_DSN=
+
 # ------------------------------
 # Notion Integration Configuration
 # Variables can be obtained by applying for Notion integration: https://www.notion.so/my-integrations
@@ -763,6 +843,8 @@ INVITE_EXPIRY_HOURS=72
 
 # Reset password token valid time (minutes),
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
+CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES=5
+OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES=5
 
 # The sandbox service endpoint.
 CODE_EXECUTION_ENDPOINT=http://sandbox:8194
@@ -794,6 +876,33 @@ WORKFLOW_FILE_UPLOAD_LIMIT=10
 # hybrid: Save new data to object storage, read from both object storage and RDBMS
 WORKFLOW_NODE_EXECUTION_STORAGE=rdbms
 
+# Repository configuration
+# Core workflow execution repository implementation
+# Options:
+#   - core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository (default)
+#   - core.repositories.celery_workflow_execution_repository.CeleryWorkflowExecutionRepository
+CORE_WORKFLOW_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository
+
+# Core workflow node execution repository implementation
+# Options:
+#   - core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository (default)
+#   - core.repositories.celery_workflow_node_execution_repository.CeleryWorkflowNodeExecutionRepository
+CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository
+
+# API workflow run repository implementation
+API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository
+
+# API workflow node execution repository implementation
+API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository
+
+# Workflow log cleanup configuration
+# Enable automatic cleanup of workflow run logs to manage database size
+WORKFLOW_LOG_CLEANUP_ENABLED=false
+# Number of days to retain workflow run logs (default: 30 days)
+WORKFLOW_LOG_RETENTION_DAYS=30
+# Batch size for workflow log cleanup operations (default: 100)
+WORKFLOW_LOG_CLEANUP_BATCH_SIZE=100
+
 # HTTP request node in workflow configuration
 HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760
 HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576
@@ -826,6 +935,12 @@ MAX_ITERATIONS_NUM=99
 # The timeout for the text generation in millisecond
 TEXT_GENERATION_TIMEOUT_MS=60000
 
+# Allow rendering unsafe URLs which have "data:" scheme.
+ALLOW_UNSAFE_DATA_SCHEME=false
+
+# Maximum number of tree depth in the workflow
+MAX_TREE_DEPTH=50
+
 # ------------------------------
 # Environment Variables for db Service
 # ------------------------------
@@ -958,7 +1073,7 @@ NGINX_SSL_PROTOCOLS=TLSv1.1 TLSv1.2 TLSv1.3
 
 # Nginx performance tuning
 NGINX_WORKER_PROCESSES=auto
-NGINX_CLIENT_MAX_BODY_SIZE=15M
+NGINX_CLIENT_MAX_BODY_SIZE=100M
 NGINX_KEEPALIVE_TIMEOUT=65
 
 # Proxy settings
@@ -1067,6 +1182,9 @@ MARKETPLACE_API_URL=https://marketplace.dify.ai
 
 FORCE_VERIFYING_SIGNATURE=true
 
+PLUGIN_STDIO_BUFFER_SIZE=1024
+PLUGIN_STDIO_MAX_BUFFER_SIZE=5242880
+
 PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120
 PLUGIN_MAX_EXECUTION_TIMEOUT=600
 # PIP_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple
@@ -1114,6 +1232,8 @@ PLUGIN_VOLCENGINE_TOS_REGION=
 # OTLP Collector Configuration
 # ------------------------------
 ENABLE_OTEL=false
+OTLP_TRACE_ENDPOINT=
+OTLP_METRIC_ENDPOINT=
 OTLP_BASE_ENDPOINT=http://localhost:4318
 OTLP_API_KEY=
 OTEL_EXPORTER_OTLP_PROTOCOL=
@@ -1135,3 +1255,17 @@ QUEUE_MONITOR_THRESHOLD=200
 QUEUE_MONITOR_ALERT_EMAILS=
 # Monitor interval in minutes, default is 30 minutes
 QUEUE_MONITOR_INTERVAL=30
+
+# Swagger UI configuration
+SWAGGER_UI_ENABLED=true
+SWAGGER_UI_PATH=/swagger-ui.html
+
+# Celery schedule tasks configuration
+ENABLE_CLEAN_EMBEDDING_CACHE_TASK=false
+ENABLE_CLEAN_UNUSED_DATASETS_TASK=false
+ENABLE_CREATE_TIDB_SERVERLESS_TASK=false
+ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK=false
+ENABLE_CLEAN_MESSAGES=false
+ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK=false
+ENABLE_DATASETS_QUEUE_MONITOR=false
+ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK=true

dify/code/README.md

@@ -4,7 +4,7 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T
 
 ### What's Updated
 
-- **Certbot Container**: `docker-compose.yaml` now contains `certbot` for managing SSL certificates. This container automatically renews certificates and ensures secure HTTPS connections.  
+- **Certbot Container**: `docker-compose.yaml` now contains `certbot` for managing SSL certificates. This container automatically renews certificates and ensures secure HTTPS connections.\
   For more information, refer `docker/certbot/README.md`.
 
 - **Persistent Environment Variables**: Environment variables are now managed through a `.env` file, ensuring that your configurations persist across deployments.
@@ -13,43 +13,44 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T
   > The `.env` file is a crucial component in Docker and Docker Compose environments, serving as a centralized configuration file where you can define environment variables that are accessible to the containers at runtime. This file simplifies the management of environment settings across different stages of development, testing, and production, providing consistency and ease of configuration to deployments.
 
 - **Unified Vector Database Services**: All vector database services are now managed from a single Docker Compose file `docker-compose.yaml`. You can switch between different vector databases by setting the `VECTOR_STORE` environment variable in your `.env` file.
+
 - **Mandatory .env File**: A `.env` file is now required to run `docker compose up`. This file is crucial for configuring your deployment and for any custom settings to persist through upgrades.
 
 ### How to Deploy Dify with `docker-compose.yaml`
 
 1. **Prerequisites**: Ensure Docker and Docker Compose are installed on your system.
-2. **Environment Setup**:
+1. **Environment Setup**:
-    - Navigate to the `docker` directory.
+   - Navigate to the `docker` directory.
-    - Copy the `.env.example` file to a new file named `.env` by running `cp .env.example .env`.
+   - Copy the `.env.example` file to a new file named `.env` by running `cp .env.example .env`.
-    - Customize the `.env` file as needed. Refer to the `.env.example` file for detailed configuration options.
+   - Customize the `.env` file as needed. Refer to the `.env.example` file for detailed configuration options.
-3. **Running the Services**:
+1. **Running the Services**:
-    - Execute `docker compose up` from the `docker` directory to start the services.
+   - Execute `docker compose up` from the `docker` directory to start the services.
-    - To specify a vector database, set the `VECTOR_STORE` variable in your `.env` file to your desired vector database service, such as `milvus`, `weaviate`, or `opensearch`.
+   - To specify a vector database, set the `VECTOR_STORE` variable in your `.env` file to your desired vector database service, such as `milvus`, `weaviate`, or `opensearch`.
-4. **SSL Certificate Setup**:
+1. **SSL Certificate Setup**:
-    - Refer `docker/certbot/README.md` to set up SSL certificates using Certbot.
+   - Refer `docker/certbot/README.md` to set up SSL certificates using Certbot.
-5. **OpenTelemetry Collector Setup**:
+1. **OpenTelemetry Collector Setup**:
    - Change `ENABLE_OTEL` to `true` in `.env`.
    - Configure `OTLP_BASE_ENDPOINT` properly.
 
 ### How to Deploy Middleware for Developing Dify
 
 1. **Middleware Setup**:
-    - Use the `docker-compose.middleware.yaml` for setting up essential middleware services like databases and caches.
+   - Use the `docker-compose.middleware.yaml` for setting up essential middleware services like databases and caches.
-    - Navigate to the `docker` directory.
+   - Navigate to the `docker` directory.
-    - Ensure the `middleware.env` file is created by running `cp middleware.env.example middleware.env` (refer to the `middleware.env.example` file).
+   - Ensure the `middleware.env` file is created by running `cp middleware.env.example middleware.env` (refer to the `middleware.env.example` file).
-2. **Running Middleware Services**:
+1. **Running Middleware Services**:
-    - Navigate to the `docker` directory.
+   - Navigate to the `docker` directory.
-    - Execute `docker compose -f docker-compose.middleware.yaml --profile weaviate -p dify up -d` to start the middleware services. (Change the profile to other vector database if you are not using weaviate)
+   - Execute `docker compose -f docker-compose.middleware.yaml --profile weaviate -p dify up -d` to start the middleware services. (Change the profile to other vector database if you are not using weaviate)
 
 ### Migration for Existing Users
 
 For users migrating from the `docker-legacy` setup:
 
 1. **Review Changes**: Familiarize yourself with the new `.env` configuration and Docker Compose setup.
-2. **Transfer Customizations**:
+1. **Transfer Customizations**:
-    - If you have customized configurations such as `docker-compose.yaml`, `ssrf_proxy/squid.conf`, or `nginx/conf.d/default.conf`, you will need to reflect these changes in the `.env` file you create.
+   - If you have customized configurations such as `docker-compose.yaml`, `ssrf_proxy/squid.conf`, or `nginx/conf.d/default.conf`, you will need to reflect these changes in the `.env` file you create.
-3. **Data Migration**:
+1. **Data Migration**:
-    - Ensure that data from services like databases and caches is backed up and migrated appropriately to the new structure if necessary.
+   - Ensure that data from services like databases and caches is backed up and migrated appropriately to the new structure if necessary.
 
 ### Overview of `.env`
 
@@ -64,39 +65,49 @@ For users migrating from the `docker-legacy` setup:
 The `.env.example` file provided in the Docker setup is extensive and covers a wide range of configuration options. It is structured into several sections, each pertaining to different aspects of the application and its services. Here are some of the key sections and variables:
 
 1. **Common Variables**:
-    - `CONSOLE_API_URL`, `SERVICE_API_URL`: URLs for different API services.
-    - `APP_WEB_URL`: Frontend application URL.
-    - `FILES_URL`: Base URL for file downloads and previews.
 
-2. **Server Configuration**:
+   - `CONSOLE_API_URL`, `SERVICE_API_URL`: URLs for different API services.
-    - `LOG_LEVEL`, `DEBUG`, `FLASK_DEBUG`: Logging and debug settings.
+   - `APP_WEB_URL`: Frontend application URL.
-    - `SECRET_KEY`: A key for encrypting session cookies and other sensitive data.
+   - `FILES_URL`: Base URL for file downloads and previews.
 
-3. **Database Configuration**:
+1. **Server Configuration**:
-    - `DB_USERNAME`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`, `DB_DATABASE`: PostgreSQL database credentials and connection details.
 
-4. **Redis Configuration**:
+   - `LOG_LEVEL`, `DEBUG`, `FLASK_DEBUG`: Logging and debug settings.
-    - `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`: Redis server connection settings.
+   - `SECRET_KEY`: A key for encrypting session cookies and other sensitive data.
 
-5. **Celery Configuration**:
+1. **Database Configuration**:
-    - `CELERY_BROKER_URL`: Configuration for Celery message broker.
 
-6. **Storage Configuration**:
+   - `DB_USERNAME`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`, `DB_DATABASE`: PostgreSQL database credentials and connection details.
-    - `STORAGE_TYPE`, `S3_BUCKET_NAME`, `AZURE_BLOB_ACCOUNT_NAME`: Settings for file storage options like local, S3, Azure Blob, etc.
 
-7. **Vector Database Configuration**:
+1. **Redis Configuration**:
-    - `VECTOR_STORE`: Type of vector database (e.g., `weaviate`, `milvus`).
-    - Specific settings for each vector store like `WEAVIATE_ENDPOINT`, `MILVUS_URI`.
 
-8. **CORS Configuration**:
+   - `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`: Redis server connection settings.
-    - `WEB_API_CORS_ALLOW_ORIGINS`, `CONSOLE_CORS_ALLOW_ORIGINS`: Settings for cross-origin resource sharing.
 
-9. **OpenTelemetry Configuration**:
+1. **Celery Configuration**:
-    - `ENABLE_OTEL`: Enable OpenTelemetry collector in api.
+
-    - `OTLP_BASE_ENDPOINT`: Endpoint for your OTLP exporter.
+   - `CELERY_BROKER_URL`: Configuration for Celery message broker.
-  
+
-10. **Other Service-Specific Environment Variables**:
+1. **Storage Configuration**:
-    - Each service like `nginx`, `redis`, `db`, and vector databases have specific environment variables that are directly referenced in the `docker-compose.yaml`.
+
+   - `STORAGE_TYPE`, `S3_BUCKET_NAME`, `AZURE_BLOB_ACCOUNT_NAME`: Settings for file storage options like local, S3, Azure Blob, etc.
+
+1. **Vector Database Configuration**:
+
+   - `VECTOR_STORE`: Type of vector database (e.g., `weaviate`, `milvus`).
+   - Specific settings for each vector store like `WEAVIATE_ENDPOINT`, `MILVUS_URI`.
+
+1. **CORS Configuration**:
+
+   - `WEB_API_CORS_ALLOW_ORIGINS`, `CONSOLE_CORS_ALLOW_ORIGINS`: Settings for cross-origin resource sharing.
+
+1. **OpenTelemetry Configuration**:
+
+   - `ENABLE_OTEL`: Enable OpenTelemetry collector in api.
+   - `OTLP_BASE_ENDPOINT`: Endpoint for your OTLP exporter.
+
+1. **Other Service-Specific Environment Variables**:
+
+   - Each service like `nginx`, `redis`, `db`, and vector databases have specific environment variables that are directly referenced in the `docker-compose.yaml`.
 
 ### Additional Information
 

dify/code/certbot/README.md

@@ -2,12 +2,12 @@
 
 ## Short description
 
-docker compose certbot configurations with Backward compatibility (without certbot container).  
+docker compose certbot configurations with Backward compatibility (without certbot container).\
 Use `docker compose --profile certbot up` to use this features.
 
 ## The simplest way for launching new servers with SSL certificates
 
-1. Get letsencrypt certs  
+1. Get letsencrypt certs\
    set `.env` values
    ```properties
    NGINX_SSL_CERT_FILENAME=fullchain.pem
@@ -25,7 +25,7 @@ Use `docker compose --profile certbot up` to use this features.
    ```shell
    docker compose exec -it certbot /bin/sh /update-cert.sh
    ```
-2. Edit `.env` file and `docker compose --profile certbot up` again.  
+1. Edit `.env` file and `docker compose --profile certbot up` again.\
    set `.env` value additionally
    ```properties
    NGINX_HTTPS_ENABLED=true
@@ -34,7 +34,7 @@ Use `docker compose --profile certbot up` to use this features.
    ```shell
    docker compose --profile certbot up -d --no-deps --force-recreate nginx
    ```
-   Then you can access your serve with HTTPS.  
+   Then you can access your serve with HTTPS.\
    [https://your_domain.com](https://your_domain.com)
 
 ## SSL certificates renewal

dify/code/docker-compose-template.yaml

@@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env
 services:
   # API service
   api:
-    image: langgenius/dify-api:1.5.1
+    image: langgenius/dify-api:1.8.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -31,7 +31,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:1.5.1
+    image: langgenius/dify-api:1.8.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -55,9 +55,28 @@ services:
       - ssrf_proxy_network
       - default
 
+  # worker_beat service
+  # Celery beat for scheduling periodic tasks.
+  worker_beat:
+    image: langgenius/dify-api:1.8.0
+    restart: always
+    environment:
+      # Use the shared environment variables.
+      <<: *shared-api-worker-env
+      # Startup mode, 'worker_beat' starts the Celery beat for scheduling periodic tasks.
+      MODE: beat
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_started
+    networks:
+      - ssrf_proxy_network
+      - default
+
   # Frontend web application.
   web:
-    image: langgenius/dify-web:1.5.1
+    image: langgenius/dify-web:1.8.0
     restart: always
     environment:
       CONSOLE_API_URL: ${CONSOLE_API_URL:-}
@@ -67,6 +86,7 @@ services:
       TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000}
       CSP_WHITELIST: ${CSP_WHITELIST:-}
       ALLOW_EMBED: ${ALLOW_EMBED:-false}
+      ALLOW_UNSAFE_DATA_SCHEME: ${ALLOW_UNSAFE_DATA_SCHEME:-false}
       MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai}
       MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai}
       TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-}
@@ -76,6 +96,7 @@ services:
       MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10}
       MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10}
       MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99}
+      MAX_TREE_DEPTH: ${MAX_TREE_DEPTH:-50}
       ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true}
       ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true}
       ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true}
@@ -142,7 +163,7 @@ services:
 
   # plugin daemon
   plugin_daemon:
-    image: langgenius/dify-plugin-daemon:0.1.3-local
+    image: langgenius/dify-plugin-daemon:0.2.0-local
     restart: always
     environment:
       # Use the shared environment variables.
@@ -160,6 +181,8 @@ services:
       FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true}
       PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}
       PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600}
+      PLUGIN_STDIO_BUFFER_SIZE: ${PLUGIN_STDIO_BUFFER_SIZE:-1024}
+      PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880}
       PIP_MIRROR_URL: ${PIP_MIRROR_URL:-}
       PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local}
       PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage}
@@ -189,6 +212,8 @@ services:
       VOLCENGINE_TOS_ACCESS_KEY: ${PLUGIN_VOLCENGINE_TOS_ACCESS_KEY:-}
       VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-}
       VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-}
+      SENTRY_ENABLED: ${PLUGIN_SENTRY_ENABLED:-false}
+      SENTRY_DSN: ${PLUGIN_SENTRY_DSN:-}
     ports:
       - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}"
     volumes:
@@ -265,7 +290,7 @@ services:
       NGINX_SSL_CERT_KEY_FILENAME: ${NGINX_SSL_CERT_KEY_FILENAME:-dify.key}
       NGINX_SSL_PROTOCOLS: ${NGINX_SSL_PROTOCOLS:-TLSv1.1 TLSv1.2 TLSv1.3}
       NGINX_WORKER_PROCESSES: ${NGINX_WORKER_PROCESSES:-auto}
-      NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-15M}
+      NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-100M}
       NGINX_KEEPALIVE_TIMEOUT: ${NGINX_KEEPALIVE_TIMEOUT:-65}
       NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s}
       NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s}
@@ -518,7 +543,7 @@ services:
 
   milvus-standalone:
     container_name: milvus-standalone
-    image: milvusdb/milvus:v2.5.0-beta
+    image: milvusdb/milvus:v2.5.15
     profiles:
       - milvus
     command: [ 'milvus', 'run', 'standalone' ]

dify/code/docker-compose.middleware.yaml

@@ -20,7 +20,7 @@ services:
     ports:
       - "${EXPOSE_POSTGRES_PORT:-5432}:5432"
     healthcheck:
-      test: [ "CMD", "pg_isready" ]
+      test: [ 'CMD', 'pg_isready', '-h', 'db', '-U', '${PGUSER:-postgres}', '-d', '${POSTGRES_DB:-dify}' ]
       interval: 1s
       timeout: 3s
       retries: 30
@@ -71,7 +71,7 @@ services:
 
   # plugin daemon
   plugin_daemon:
-    image: langgenius/dify-plugin-daemon:0.1.3-local
+    image: langgenius/dify-plugin-daemon:0.2.0-local
     restart: always
     env_file:
       - ./middleware.env

dify/code/docker-compose.yaml

@@ -11,6 +11,10 @@ x-shared-env: &shared-api-worker-env
   APP_API_URL: ${APP_API_URL:-}
   APP_WEB_URL: ${APP_WEB_URL:-}
   FILES_URL: ${FILES_URL:-}
+  INTERNAL_FILES_URL: ${INTERNAL_FILES_URL:-}
+  LANG: ${LANG:-en_US.UTF-8}
+  LC_ALL: ${LC_ALL:-en_US.UTF-8}
+  PYTHONIOENCODING: ${PYTHONIOENCODING:-utf-8}
   LOG_LEVEL: ${LOG_LEVEL:-INFO}
   LOG_FILE: ${LOG_FILE:-/app/logs/server.log}
   LOG_FILE_MAX_SIZE: ${LOG_FILE_MAX_SIZE:-20}
@@ -53,8 +57,11 @@ x-shared-env: &shared-api-worker-env
   DB_PORT: ${DB_PORT:-5432}
   DB_DATABASE: ${DB_DATABASE:-dify}
   SQLALCHEMY_POOL_SIZE: ${SQLALCHEMY_POOL_SIZE:-30}
+  SQLALCHEMY_MAX_OVERFLOW: ${SQLALCHEMY_MAX_OVERFLOW:-10}
   SQLALCHEMY_POOL_RECYCLE: ${SQLALCHEMY_POOL_RECYCLE:-3600}
   SQLALCHEMY_ECHO: ${SQLALCHEMY_ECHO:-false}
+  SQLALCHEMY_POOL_PRE_PING: ${SQLALCHEMY_POOL_PRE_PING:-false}
+  SQLALCHEMY_POOL_USE_LIFO: ${SQLALCHEMY_POOL_USE_LIFO:-false}
   POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-100}
   POSTGRES_SHARED_BUFFERS: ${POSTGRES_SHARED_BUFFERS:-128MB}
   POSTGRES_WORK_MEM: ${POSTGRES_WORK_MEM:-4MB}
@@ -65,6 +72,10 @@ x-shared-env: &shared-api-worker-env
   REDIS_USERNAME: ${REDIS_USERNAME:-}
   REDIS_PASSWORD: ${REDIS_PASSWORD:-difyai123456}
   REDIS_USE_SSL: ${REDIS_USE_SSL:-false}
+  REDIS_SSL_CERT_REQS: ${REDIS_SSL_CERT_REQS:-CERT_NONE}
+  REDIS_SSL_CA_CERTS: ${REDIS_SSL_CA_CERTS:-}
+  REDIS_SSL_CERTFILE: ${REDIS_SSL_CERTFILE:-}
+  REDIS_SSL_KEYFILE: ${REDIS_SSL_KEYFILE:-}
   REDIS_DB: ${REDIS_DB:-0}
   REDIS_USE_SENTINEL: ${REDIS_USE_SENTINEL:-false}
   REDIS_SENTINELS: ${REDIS_SENTINELS:-}
@@ -76,6 +87,7 @@ x-shared-env: &shared-api-worker-env
   REDIS_CLUSTERS: ${REDIS_CLUSTERS:-}
   REDIS_CLUSTERS_PASSWORD: ${REDIS_CLUSTERS_PASSWORD:-}
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://:difyai123456@redis:6379/1}
+  CELERY_BACKEND: ${CELERY_BACKEND:-redis}
   BROKER_USE_SSL: ${BROKER_USE_SSL:-false}
   CELERY_USE_SENTINEL: ${CELERY_USE_SENTINEL:-false}
   CELERY_SENTINEL_MASTER_NAME: ${CELERY_SENTINEL_MASTER_NAME:-}
@@ -86,6 +98,10 @@ x-shared-env: &shared-api-worker-env
   STORAGE_TYPE: ${STORAGE_TYPE:-opendal}
   OPENDAL_SCHEME: ${OPENDAL_SCHEME:-fs}
   OPENDAL_FS_ROOT: ${OPENDAL_FS_ROOT:-storage}
+  CLICKZETTA_VOLUME_TYPE: ${CLICKZETTA_VOLUME_TYPE:-user}
+  CLICKZETTA_VOLUME_NAME: ${CLICKZETTA_VOLUME_NAME:-}
+  CLICKZETTA_VOLUME_TABLE_PREFIX: ${CLICKZETTA_VOLUME_TABLE_PREFIX:-dataset_}
+  CLICKZETTA_VOLUME_DIFY_PREFIX: ${CLICKZETTA_VOLUME_DIFY_PREFIX:-dify_km}
   S3_ENDPOINT: ${S3_ENDPOINT:-}
   S3_REGION: ${S3_REGION:-us-east-1}
   S3_BUCKET_NAME: ${S3_BUCKET_NAME:-difyai}
@@ -132,6 +148,7 @@ x-shared-env: &shared-api-worker-env
   SUPABASE_API_KEY: ${SUPABASE_API_KEY:-your-access-key}
   SUPABASE_URL: ${SUPABASE_URL:-your-server-url}
   VECTOR_STORE: ${VECTOR_STORE:-weaviate}
+  VECTOR_INDEX_NAME_PREFIX: ${VECTOR_INDEX_NAME_PREFIX:-Vector_index}
   WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080}
   WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih}
   QDRANT_URL: ${QDRANT_URL:-http://qdrant:6333}
@@ -253,6 +270,14 @@ x-shared-env: &shared-api-worker-env
   ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_USERNAME:-elastic}
   ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-elastic}
   KIBANA_PORT: ${KIBANA_PORT:-5601}
+  ELASTICSEARCH_USE_CLOUD: ${ELASTICSEARCH_USE_CLOUD:-false}
+  ELASTICSEARCH_CLOUD_URL: ${ELASTICSEARCH_CLOUD_URL:-YOUR-ELASTICSEARCH_CLOUD_URL}
+  ELASTICSEARCH_API_KEY: ${ELASTICSEARCH_API_KEY:-YOUR-ELASTICSEARCH_API_KEY}
+  ELASTICSEARCH_VERIFY_CERTS: ${ELASTICSEARCH_VERIFY_CERTS:-False}
+  ELASTICSEARCH_CA_CERTS: ${ELASTICSEARCH_CA_CERTS:-}
+  ELASTICSEARCH_REQUEST_TIMEOUT: ${ELASTICSEARCH_REQUEST_TIMEOUT:-100000}
+  ELASTICSEARCH_RETRY_ON_TIMEOUT: ${ELASTICSEARCH_RETRY_ON_TIMEOUT:-True}
+  ELASTICSEARCH_MAX_RETRIES: ${ELASTICSEARCH_MAX_RETRIES:-10}
   BAIDU_VECTOR_DB_ENDPOINT: ${BAIDU_VECTOR_DB_ENDPOINT:-http://127.0.0.1:5287}
   BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS: ${BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS:-30000}
   BAIDU_VECTOR_DB_ACCOUNT: ${BAIDU_VECTOR_DB_ACCOUNT:-root}
@@ -296,6 +321,19 @@ x-shared-env: &shared-api-worker-env
   TABLESTORE_INSTANCE_NAME: ${TABLESTORE_INSTANCE_NAME:-instance-name}
   TABLESTORE_ACCESS_KEY_ID: ${TABLESTORE_ACCESS_KEY_ID:-xxx}
   TABLESTORE_ACCESS_KEY_SECRET: ${TABLESTORE_ACCESS_KEY_SECRET:-xxx}
+  TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE: ${TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE:-false}
+  CLICKZETTA_USERNAME: ${CLICKZETTA_USERNAME:-}
+  CLICKZETTA_PASSWORD: ${CLICKZETTA_PASSWORD:-}
+  CLICKZETTA_INSTANCE: ${CLICKZETTA_INSTANCE:-}
+  CLICKZETTA_SERVICE: ${CLICKZETTA_SERVICE:-api.clickzetta.com}
+  CLICKZETTA_WORKSPACE: ${CLICKZETTA_WORKSPACE:-quick_start}
+  CLICKZETTA_VCLUSTER: ${CLICKZETTA_VCLUSTER:-default_ap}
+  CLICKZETTA_SCHEMA: ${CLICKZETTA_SCHEMA:-dify}
+  CLICKZETTA_BATCH_SIZE: ${CLICKZETTA_BATCH_SIZE:-100}
+  CLICKZETTA_ENABLE_INVERTED_INDEX: ${CLICKZETTA_ENABLE_INVERTED_INDEX:-true}
+  CLICKZETTA_ANALYZER_TYPE: ${CLICKZETTA_ANALYZER_TYPE:-chinese}
+  CLICKZETTA_ANALYZER_MODE: ${CLICKZETTA_ANALYZER_MODE:-smart}
+  CLICKZETTA_VECTOR_DISTANCE_FUNCTION: ${CLICKZETTA_VECTOR_DISTANCE_FUNCTION:-cosine_distance}
   UPLOAD_FILE_SIZE_LIMIT: ${UPLOAD_FILE_SIZE_LIMIT:-15}
   UPLOAD_FILE_BATCH_LIMIT: ${UPLOAD_FILE_BATCH_LIMIT:-5}
   ETL_TYPE: ${ETL_TYPE:-dify}
@@ -314,6 +352,8 @@ x-shared-env: &shared-api-worker-env
   API_SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0}
   API_SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0}
   WEB_SENTRY_DSN: ${WEB_SENTRY_DSN:-}
+  PLUGIN_SENTRY_ENABLED: ${PLUGIN_SENTRY_ENABLED:-false}
+  PLUGIN_SENTRY_DSN: ${PLUGIN_SENTRY_DSN:-}
   NOTION_INTEGRATION_TYPE: ${NOTION_INTEGRATION_TYPE:-public}
   NOTION_CLIENT_SECRET: ${NOTION_CLIENT_SECRET:-}
   NOTION_CLIENT_ID: ${NOTION_CLIENT_ID:-}
@@ -332,6 +372,8 @@ x-shared-env: &shared-api-worker-env
   INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000}
   INVITE_EXPIRY_HOURS: ${INVITE_EXPIRY_HOURS:-72}
   RESET_PASSWORD_TOKEN_EXPIRY_MINUTES: ${RESET_PASSWORD_TOKEN_EXPIRY_MINUTES:-5}
+  CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES: ${CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES:-5}
+  OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES: ${OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES:-5}
   CODE_EXECUTION_ENDPOINT: ${CODE_EXECUTION_ENDPOINT:-http://sandbox:8194}
   CODE_EXECUTION_API_KEY: ${CODE_EXECUTION_API_KEY:-dify-sandbox}
   CODE_MAX_NUMBER: ${CODE_MAX_NUMBER:-9223372036854775807}
@@ -353,6 +395,13 @@ x-shared-env: &shared-api-worker-env
   WORKFLOW_PARALLEL_DEPTH_LIMIT: ${WORKFLOW_PARALLEL_DEPTH_LIMIT:-3}
   WORKFLOW_FILE_UPLOAD_LIMIT: ${WORKFLOW_FILE_UPLOAD_LIMIT:-10}
   WORKFLOW_NODE_EXECUTION_STORAGE: ${WORKFLOW_NODE_EXECUTION_STORAGE:-rdbms}
+  CORE_WORKFLOW_EXECUTION_REPOSITORY: ${CORE_WORKFLOW_EXECUTION_REPOSITORY:-core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository}
+  CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY:-core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository}
+  API_WORKFLOW_RUN_REPOSITORY: ${API_WORKFLOW_RUN_REPOSITORY:-repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository}
+  API_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${API_WORKFLOW_NODE_EXECUTION_REPOSITORY:-repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository}
+  WORKFLOW_LOG_CLEANUP_ENABLED: ${WORKFLOW_LOG_CLEANUP_ENABLED:-false}
+  WORKFLOW_LOG_RETENTION_DAYS: ${WORKFLOW_LOG_RETENTION_DAYS:-30}
+  WORKFLOW_LOG_CLEANUP_BATCH_SIZE: ${WORKFLOW_LOG_CLEANUP_BATCH_SIZE:-100}
   HTTP_REQUEST_NODE_MAX_BINARY_SIZE: ${HTTP_REQUEST_NODE_MAX_BINARY_SIZE:-10485760}
   HTTP_REQUEST_NODE_MAX_TEXT_SIZE: ${HTTP_REQUEST_NODE_MAX_TEXT_SIZE:-1048576}
   HTTP_REQUEST_NODE_SSL_VERIFY: ${HTTP_REQUEST_NODE_SSL_VERIFY:-True}
@@ -364,6 +413,8 @@ x-shared-env: &shared-api-worker-env
   MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10}
   MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99}
   TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000}
+  ALLOW_UNSAFE_DATA_SCHEME: ${ALLOW_UNSAFE_DATA_SCHEME:-false}
+  MAX_TREE_DEPTH: ${MAX_TREE_DEPTH:-50}
   POSTGRES_USER: ${POSTGRES_USER:-${DB_USERNAME}}
   POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-${DB_PASSWORD}}
   POSTGRES_DB: ${POSTGRES_DB:-${DB_DATABASE}}
@@ -420,7 +471,7 @@ x-shared-env: &shared-api-worker-env
   NGINX_SSL_CERT_KEY_FILENAME: ${NGINX_SSL_CERT_KEY_FILENAME:-dify.key}
   NGINX_SSL_PROTOCOLS: ${NGINX_SSL_PROTOCOLS:-TLSv1.1 TLSv1.2 TLSv1.3}
   NGINX_WORKER_PROCESSES: ${NGINX_WORKER_PROCESSES:-auto}
-  NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-15M}
+  NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-100M}
   NGINX_KEEPALIVE_TIMEOUT: ${NGINX_KEEPALIVE_TIMEOUT:-65}
   NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s}
   NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s}
@@ -465,6 +516,8 @@ x-shared-env: &shared-api-worker-env
   MARKETPLACE_ENABLED: ${MARKETPLACE_ENABLED:-true}
   MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai}
   FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true}
+  PLUGIN_STDIO_BUFFER_SIZE: ${PLUGIN_STDIO_BUFFER_SIZE:-1024}
+  PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880}
   PLUGIN_PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}
   PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600}
   PIP_MIRROR_URL: ${PIP_MIRROR_URL:-}
@@ -498,6 +551,8 @@ x-shared-env: &shared-api-worker-env
   PLUGIN_VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-}
   PLUGIN_VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-}
   ENABLE_OTEL: ${ENABLE_OTEL:-false}
+  OTLP_TRACE_ENDPOINT: ${OTLP_TRACE_ENDPOINT:-}
+  OTLP_METRIC_ENDPOINT: ${OTLP_METRIC_ENDPOINT:-}
   OTLP_BASE_ENDPOINT: ${OTLP_BASE_ENDPOINT:-http://localhost:4318}
   OTLP_API_KEY: ${OTLP_API_KEY:-}
   OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-}
@@ -513,11 +568,21 @@ x-shared-env: &shared-api-worker-env
   QUEUE_MONITOR_THRESHOLD: ${QUEUE_MONITOR_THRESHOLD:-200}
   QUEUE_MONITOR_ALERT_EMAILS: ${QUEUE_MONITOR_ALERT_EMAILS:-}
   QUEUE_MONITOR_INTERVAL: ${QUEUE_MONITOR_INTERVAL:-30}
+  SWAGGER_UI_ENABLED: ${SWAGGER_UI_ENABLED:-true}
+  SWAGGER_UI_PATH: ${SWAGGER_UI_PATH:-/swagger-ui.html}
+  ENABLE_CLEAN_EMBEDDING_CACHE_TASK: ${ENABLE_CLEAN_EMBEDDING_CACHE_TASK:-false}
+  ENABLE_CLEAN_UNUSED_DATASETS_TASK: ${ENABLE_CLEAN_UNUSED_DATASETS_TASK:-false}
+  ENABLE_CREATE_TIDB_SERVERLESS_TASK: ${ENABLE_CREATE_TIDB_SERVERLESS_TASK:-false}
+  ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK: ${ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK:-false}
+  ENABLE_CLEAN_MESSAGES: ${ENABLE_CLEAN_MESSAGES:-false}
+  ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK: ${ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK:-false}
+  ENABLE_DATASETS_QUEUE_MONITOR: ${ENABLE_DATASETS_QUEUE_MONITOR:-false}
+  ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK: ${ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK:-true}
 
 services:
   # API service
   api:
-    image: langgenius/dify-api:1.5.1
+    image: langgenius/dify-api:1.8.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -546,7 +611,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:1.5.1
+    image: langgenius/dify-api:1.8.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -570,9 +635,28 @@ services:
       - ssrf_proxy_network
       - default
 
+  # worker_beat service
+  # Celery beat for scheduling periodic tasks.
+  worker_beat:
+    image: langgenius/dify-api:1.8.0
+    restart: always
+    environment:
+      # Use the shared environment variables.
+      <<: *shared-api-worker-env
+      # Startup mode, 'worker_beat' starts the Celery beat for scheduling periodic tasks.
+      MODE: beat
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_started
+    networks:
+      - ssrf_proxy_network
+      - default
+
   # Frontend web application.
   web:
-    image: langgenius/dify-web:1.5.1
+    image: langgenius/dify-web:1.8.0
     restart: always
     environment:
       CONSOLE_API_URL: ${CONSOLE_API_URL:-}
@@ -582,6 +666,7 @@ services:
       TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000}
       CSP_WHITELIST: ${CSP_WHITELIST:-}
       ALLOW_EMBED: ${ALLOW_EMBED:-false}
+      ALLOW_UNSAFE_DATA_SCHEME: ${ALLOW_UNSAFE_DATA_SCHEME:-false}
       MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai}
       MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai}
       TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-}
@@ -591,6 +676,7 @@ services:
       MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10}
       MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10}
       MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99}
+      MAX_TREE_DEPTH: ${MAX_TREE_DEPTH:-50}
       ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true}
       ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true}
       ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true}
@@ -667,7 +753,7 @@ services:
 
   # plugin daemon
   plugin_daemon:
-    image: langgenius/dify-plugin-daemon:0.1.3-local
+    image: langgenius/dify-plugin-daemon:0.2.0-local
     restart: always
     environment:
       # Use the shared environment variables.
@@ -685,6 +771,8 @@ services:
       FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true}
       PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}
       PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600}
+      PLUGIN_STDIO_BUFFER_SIZE: ${PLUGIN_STDIO_BUFFER_SIZE:-1024}
+      PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880}
       PIP_MIRROR_URL: ${PIP_MIRROR_URL:-}
       PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local}
       PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage}
@@ -714,6 +802,8 @@ services:
       VOLCENGINE_TOS_ACCESS_KEY: ${PLUGIN_VOLCENGINE_TOS_ACCESS_KEY:-}
       VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-}
       VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-}
+      SENTRY_ENABLED: ${PLUGIN_SENTRY_ENABLED:-false}
+      SENTRY_DSN: ${PLUGIN_SENTRY_DSN:-}
     volumes:
       - ./volumes/plugin_daemon:/app/storage
     depends_on:
@@ -802,7 +892,7 @@ services:
       NGINX_SSL_CERT_KEY_FILENAME: ${NGINX_SSL_CERT_KEY_FILENAME:-dify.key}
       NGINX_SSL_PROTOCOLS: ${NGINX_SSL_PROTOCOLS:-TLSv1.1 TLSv1.2 TLSv1.3}
       NGINX_WORKER_PROCESSES: ${NGINX_WORKER_PROCESSES:-auto}
-      NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-15M}
+      NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-100M}
       NGINX_KEEPALIVE_TIMEOUT: ${NGINX_KEEPALIVE_TIMEOUT:-65}
       NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s}
       NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s}
@@ -1055,7 +1145,7 @@ services:
       - milvus
 
   milvus-standalone:
-    image: milvusdb/milvus:v2.5.0-beta
+    image: milvusdb/milvus:v2.5.15
     profiles:
       - milvus
     command: [ 'milvus', 'run', 'standalone' ]

dify/code/nginx/conf.d/default.conf.template

@@ -39,7 +39,10 @@ server {
       proxy_pass http://web:3000;
       include proxy.conf;
     }
-
+    location /mcp {
+      proxy_pass http://api:5001;
+      include proxy.conf;
+    }
     # placeholder for acme challenge location
     ${ACME_CHALLENGE_LOCATION}
 

dify/update.js

@@ -4,3 +4,9 @@ await utils.cloneOrPullRepo({ repo: "https://github.com/langgenius/dify.git" });
 await utils.copyDir("./repo/docker", "./code");
 await utils.removeContainerNames("./code/docker-compose.yaml");
 await utils.removePorts("./code/docker-compose.yaml");
+
+await utils.searchReplace(
+  "./code/.env.example",
+  "APP_WEB_URL=",
+  "APP_WEB_URL=https://$(PRIMARY_DOMAIN)"
+);

dify/update.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-if [ ! -d "./repo" ]; then
-    git clone --depth 1 --branch main --single-branch https://github.com/langgenius/dify.git repo
-else
-    cd repo
-    git pull
-    cd ..
-fi
-
-cp -r ./repo/docker/. ./code
-
-

plane/update.js

@@ -1,5 +1,11 @@
 import utils from "../utils.js";
 
+console.log(
+  "Plan file structure has changed. This script needs to be reworked."
+);
+
+process.exit();
+
 await utils.cloneOrPullRepo({
   repo: "https://github.com/makeplane/plane.git",
   path: "./repo",
@@ -11,3 +17,21 @@ await utils.renameFile("./code/variables.env", "./code/.env.example");
 
 await utils.removeContainerNames("./code/docker-compose.yml");
 await utils.removePorts("./code/docker-compose.yml");
+
+await utils.searchReplace(
+  "./code/.env.example",
+  "APP_DOMAIN=localhost",
+  "APP_DOMAIN=$(PRIMARY_DOMAIN)"
+);
+
+await utils.searchReplace(
+  "./code/.env.example",
+  "WEB_URL=http://${APP_DOMAIN}",
+  "WEB_URL=https://$(PRIMARY_DOMAIN)"
+);
+
+await utils.searchReplace(
+  "./code/.env.example",
+  "CORS_ALLOWED_ORIGINS=http://${APP_DOMAIN}",
+  "CORS_ALLOWED_ORIGINS=https://$(PRIMARY_DOMAIN)"
+);

plane/update.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-if [ ! -d "./repo" ]; then
-    git clone --depth 1 --branch preview --single-branch https://github.com/makeplane/plane.git repo
-else
-    cd repo
-    git pull
-    cd ..
-fi
-
-cp -r ./repo/deploy/selfhost/. ./code
-mv ./code/variables.env ./code/.env.example
-
-

supabase/code/.env.example

@@ -59,7 +59,7 @@ PGRST_DB_SCHEMAS=public,storage,graphql_public
 ############
 
 ## General
-SITE_URL=http://localhost:3000
+SITE_URL=https://$(PRIMARY_DOMAIN)
 ADDITIONAL_REDIRECT_URLS=
 JWT_EXPIRY=3600
 DISABLE_SIGNUP=false

supabase/code/docker-compose.yml

@@ -75,7 +75,7 @@ services:
       /docker-entrypoint.sh kong docker-start'
 
   auth:
-    image: supabase/gotrue:v2.176.1
+    image: supabase/gotrue:v2.177.0
     restart: unless-stopped
     healthcheck:
       test:
@@ -221,7 +221,7 @@ services:
 
   # To use S3 backed storage: docker compose -f docker-compose.yml -f docker-compose.s3.yml up
   storage:
-    image: supabase/storage-api:v1.24.7
+    image: supabase/storage-api:v1.25.7
     restart: unless-stopped
     volumes:
       - ./volumes/storage:/var/lib/storage:z
@@ -279,7 +279,7 @@ services:
       IMGPROXY_ENABLE_WEBP_DETECTION: ${IMGPROXY_ENABLE_WEBP_DETECTION}
 
   meta:
-    image: supabase/postgres-meta:v0.89.3
+    image: supabase/postgres-meta:v0.91.0
     restart: unless-stopped
     depends_on:
       db:
@@ -429,7 +429,7 @@ services:
 
   # Update the DATABASE_URL if you are using an external Postgres database
   supavisor:
-    image: supabase/supavisor:2.5.6
+    image: supabase/supavisor:2.5.7
     restart: unless-stopped
     volumes:
       - ./volumes/pooler/pooler.exs:/etc/pooler/pooler.exs:ro,z

supabase/update.js

@@ -5,3 +5,9 @@ await utils.copyDir("./repo/docker", "./code");
 
 await utils.removeContainerNames("./code/docker-compose.yml");
 await utils.removePorts("./code/docker-compose.yml");
+
+await utils.searchReplace(
+  "./code/.env.example",
+  "SITE_URL=http://localhost:3000",
+  "SITE_URL=https://$(PRIMARY_DOMAIN)"
+);

supabase/update.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-if [ ! -d "./repo" ]; then
-    git clone --depth 1 --branch master --single-branch https://github.com/supabase/supabase repo
-else
-    cd repo
-    git pull
-    cd ..
-fi
-
-cp -r ./repo/docker/. ./code

twenty/code/twenty-website/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:22-alpine as twenty-website-build
+FROM node:24-alpine AS twenty-website-build
 
 
 WORKDIR /app
@@ -7,6 +7,7 @@ COPY ./package.json .
 COPY ./yarn.lock .
 COPY ./.yarnrc.yml .
 COPY ./.yarn/releases /app/.yarn/releases
+COPY ./.yarn/patches /app/.yarn/patches
 COPY ./tools/eslint-rules /app/tools/eslint-rules
 COPY ./packages/twenty-ui/package.json /app/packages/twenty-ui/
 COPY ./packages/twenty-shared/package.json /app/packages/twenty-shared/
@@ -23,7 +24,7 @@ COPY ./packages/twenty-ui /app/packages/twenty-ui
 COPY ./packages/twenty-website /app/packages/twenty-website
 RUN npx nx build twenty-website
 
-FROM node:22-alpine as twenty-website
+FROM node:24-alpine AS twenty-website
 
 WORKDIR /app/packages/twenty-website
 

twenty/code/twenty/Dockerfile

@@ -1,11 +1,12 @@
 # Base image for common dependencies
-FROM node:22-alpine as common-deps
+FROM node:24-alpine AS common-deps
 
 WORKDIR /app
 
 # Copy only the necessary files for dependency resolution
 COPY ./package.json ./yarn.lock ./.yarnrc.yml ./tsconfig.base.json ./nx.json /app/
 COPY ./.yarn/releases /app/.yarn/releases
+COPY ./.yarn/patches /app/.yarn/patches
 
 COPY ./.prettierrc /app/
 COPY ./packages/twenty-emails/package.json /app/packages/twenty-emails/
@@ -20,7 +21,7 @@ RUN yarn && yarn cache clean && npx nx reset
 
 
 # Build the back
-FROM common-deps as twenty-server-build
+FROM common-deps AS twenty-server-build
 
 # Copy sourcecode after installing dependences to accelerate subsequents builds
 COPY ./packages/twenty-emails /app/packages/twenty-emails
@@ -28,17 +29,11 @@ COPY ./packages/twenty-shared /app/packages/twenty-shared
 COPY ./packages/twenty-server /app/packages/twenty-server
 
 RUN npx nx run twenty-server:build
-RUN mv /app/packages/twenty-server/dist /app/packages/twenty-server/build
-RUN npx nx run twenty-server:build:packageJson
-RUN mv /app/packages/twenty-server/dist/package.json /app/packages/twenty-server/package.json
-RUN rm -rf /app/packages/twenty-server/dist
-RUN mv /app/packages/twenty-server/build /app/packages/twenty-server/dist
 
 RUN yarn workspaces focus --production twenty-emails twenty-shared twenty-server
 
-
 # Build the front
-FROM common-deps as twenty-front-build
+FROM common-deps AS twenty-front-build
 
 ARG REACT_APP_SERVER_BASE_URL
 
@@ -49,7 +44,7 @@ RUN npx nx build twenty-front
 
 
 # Final stage: Run the application
-FROM node:22-alpine as twenty
+FROM node:24-alpine AS twenty
 
 # Used to run healthcheck in docker
 RUN apk add --no-cache curl jq
@@ -63,10 +58,10 @@ RUN chmod +x /app/entrypoint.sh
 WORKDIR /app/packages/twenty-server
 
 ARG REACT_APP_SERVER_BASE_URL
-ENV REACT_APP_SERVER_BASE_URL $REACT_APP_SERVER_BASE_URL
+ENV REACT_APP_SERVER_BASE_URL=$REACT_APP_SERVER_BASE_URL
 
 ARG APP_VERSION
-ENV APP_VERSION $APP_VERSION
+ENV APP_VERSION=$APP_VERSION
 
 # Copy built applications from previous stages
 COPY --chown=1000 --from=twenty-server-build /app /app

twenty/code/twenty/entrypoint.sh

@@ -19,8 +19,12 @@ setup_and_migrate_db() {
     if [ "$db_count" = "0" ]; then
         echo "Database ${PGDATABASE} does not exist, creating..."
         PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d postgres -c "CREATE DATABASE \"${PGDATABASE}\""
+    fi
 
-        # Run setup and migration scripts
+    # Run setup and migration scripts
+    has_schema=$(PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d ${PGDATABASE} -tAc "SELECT EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'core')")
+    if [ "$has_schema" = "f" ]; then
+        echo "Database appears to be empty, running migrations."
         NODE_OPTIONS="--max-old-space-size=1500" tsx ./scripts/setup-db.ts
         yarn database:migrate:prod
     fi

twenty/update.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-if [ ! -d "./repo" ]; then
-    git clone --depth 1 --branch main --single-branch git@github.com:twentyhq/twenty.git repo
-else
-    cd repo
-    git pull
-    cd ..
-fi
-
-cp -r ./repo/packages/twenty-docker/. ./code
-
-