diff --git a/appwrite/code/.env.example b/appwrite/code/.env.example index 81df039..e7d5040 100644 --- a/appwrite/code/.env.example +++ b/appwrite/code/.env.example @@ -100,6 +100,7 @@ _APP_FUNCTIONS_MEMORY_SWAP=0 _APP_FUNCTIONS_RUNTIMES=node-16.0,php-8.0,python-3.9,ruby-3.0 _APP_EXECUTOR_SECRET=your-secret-key _APP_EXECUTOR_HOST=http://exc1/v1 +_APP_BROWSER_HOST=http://appwrite-browser:3000/v1 _APP_EXECUTOR_RUNTIME_NETWORK=appwrite_runtimes _APP_FUNCTIONS_ENVS=node-16.0,php-7.4,python-3.9,ruby-3.0 _APP_FUNCTIONS_INACTIVE_THRESHOLD=60 diff --git a/appwrite/code/docker-compose.yml b/appwrite/code/docker-compose.yml index 3ee4acd..29db96c 100644 --- a/appwrite/code/docker-compose.yml +++ b/appwrite/code/docker-compose.yml @@ -28,7 +28,7 @@ services: - appwrite appwrite: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 <<: *x-logging restart: unless-stopped networks: @@ -166,7 +166,7 @@ services: - _APP_ASSISTANT_OPENAI_API_KEY appwrite-console: <<: *x-logging - image: appwrite/console:7.4.7 + image: appwrite/console:7.5.7 restart: unless-stopped networks: - appwrite @@ -186,7 +186,7 @@ services: - traefik.http.routers.appwrite_console_https.tls=true appwrite-realtime: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: realtime <<: *x-logging restart: unless-stopped @@ -228,7 +228,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-audits: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-audits <<: *x-logging restart: unless-stopped @@ -253,7 +253,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-webhooks: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-webhooks <<: *x-logging restart: unless-stopped @@ -280,7 +280,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-deletes: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-deletes <<: *x-logging restart: unless-stopped @@ -342,7 +342,7 @@ services: - _APP_EMAIL_CERTIFICATES appwrite-worker-databases: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-databases <<: *x-logging restart: unless-stopped @@ -367,7 +367,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-builds: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-builds <<: *x-logging restart: unless-stopped @@ -434,7 +434,7 @@ services: - _APP_DOMAIN_SITES appwrite-worker-certificates: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-certificates <<: *x-logging restart: unless-stopped @@ -470,7 +470,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-functions: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-functions <<: *x-logging restart: unless-stopped @@ -508,7 +508,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-mails: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-mails <<: *x-logging restart: unless-stopped @@ -541,7 +541,7 @@ services: - _APP_OPTIONS_FORCE_HTTPS appwrite-worker-messaging: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-messaging <<: *x-logging restart: unless-stopped @@ -591,7 +591,7 @@ services: - _APP_STORAGE_WASABI_BUCKET appwrite-worker-migrations: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-migrations <<: *x-logging restart: unless-stopped @@ -626,7 +626,7 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET appwrite-task-maintenance: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: maintenance <<: *x-logging restart: unless-stopped @@ -664,7 +664,7 @@ services: - _APP_MAINTENANCE_RETENTION_SCHEDULES appwrite-task-stats-resources: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: stats-resources <<: *x-logging restart: unless-stopped @@ -692,7 +692,7 @@ services: - _APP_STATS_RESOURCES_INTERVAL appwrite-worker-stats-resources: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-stats-resources <<: *x-logging restart: unless-stopped @@ -719,7 +719,7 @@ services: - _APP_STATS_RESOURCES_INTERVAL appwrite-worker-stats-usage: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: worker-stats-usage <<: *x-logging restart: unless-stopped @@ -746,7 +746,7 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL appwrite-task-scheduler-functions: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: schedule-functions <<: *x-logging restart: unless-stopped @@ -770,7 +770,7 @@ services: - _APP_DB_PASS appwrite-task-scheduler-executions: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: schedule-executions <<: *x-logging restart: unless-stopped @@ -794,7 +794,7 @@ services: - _APP_DB_PASS appwrite-task-scheduler-messages: - image: appwrite/appwrite:1.8.0 + image: appwrite/appwrite:1.8.1 entrypoint: schedule-messages <<: *x-logging restart: unless-stopped @@ -818,7 +818,7 @@ services: - _APP_DB_PASS appwrite-assistant: - image: appwrite/assistant:0.8.3 + image: appwrite/assistant:0.8.4 <<: *x-logging restart: unless-stopped networks: @@ -827,7 +827,7 @@ services: - _APP_ASSISTANT_OPENAI_API_KEY appwrite-browser: - image: appwrite/browser:0.2.4 + image: appwrite/browser:0.3.2 <<: *x-logging restart: unless-stopped networks: diff --git a/dify/code/.env.example b/dify/code/.env.example index 3032216..9f2fa47 100644 --- a/dify/code/.env.example +++ b/dify/code/.env.example @@ -58,8 +58,8 @@ FILES_URL= INTERNAL_FILES_URL= # Ensure UTF-8 encoding -LANG=en_US.UTF-8 -LC_ALL=en_US.UTF-8 +LANG=C.UTF-8 +LC_ALL=C.UTF-8 PYTHONIOENCODING=utf-8 # ------------------------------ @@ -69,6 +69,8 @@ PYTHONIOENCODING=utf-8 # The log level for the application. # Supported values are `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` LOG_LEVEL=INFO +# Log output format: text or json +LOG_OUTPUT_FORMAT=text # Log file path LOG_FILE=/app/logs/server.log # Log file max size, the unit is MB @@ -231,7 +233,7 @@ NEXT_PUBLIC_ENABLE_SINGLE_DOLLAR_LATEX=false # You can adjust the database configuration according to your needs. # ------------------------------ -# Database type, supported values are `postgresql` and `mysql` +# Database type, supported values are `postgresql`, `mysql`, `oceanbase`, `seekdb` DB_TYPE=postgresql # For MySQL, only `root` user is supported for now DB_USERNAME=postgres @@ -447,6 +449,15 @@ S3_SECRET_KEY= # If set to false, the access key and secret key must be provided. S3_USE_AWS_MANAGED_IAM=false +# Workflow run and Conversation archive storage (S3-compatible) +ARCHIVE_STORAGE_ENABLED=false +ARCHIVE_STORAGE_ENDPOINT= +ARCHIVE_STORAGE_ARCHIVE_BUCKET= +ARCHIVE_STORAGE_EXPORT_BUCKET= +ARCHIVE_STORAGE_ACCESS_KEY= +ARCHIVE_STORAGE_SECRET_KEY= +ARCHIVE_STORAGE_REGION=auto + # Azure Blob Configuration # AZURE_BLOB_ACCOUNT_NAME=difyai @@ -522,7 +533,7 @@ SUPABASE_URL=your-server-url # ------------------------------ # The type of vector store to use. -# Supported values are `weaviate`, `oceanbase`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `opengauss`, `tablestore`,`vastbase`,`tidb`,`tidb_on_qdrant`,`baidu`,`lindorm`,`huawei_cloud`,`upstash`, `matrixone`, `clickzetta`, `alibabacloud_mysql`, `iris`. +# Supported values are `weaviate`, `oceanbase`, `seekdb`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `opengauss`, `tablestore`, `vastbase`, `tidb`, `tidb_on_qdrant`, `baidu`, `lindorm`, `huawei_cloud`, `upstash`, `matrixone`, `clickzetta`, `alibabacloud_mysql`, `iris`. VECTOR_STORE=weaviate # Prefix used to create collection name in vector database VECTOR_INDEX_NAME_PREFIX=Vector_index @@ -533,9 +544,9 @@ WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih WEAVIATE_GRPC_ENDPOINT=grpc://weaviate:50051 WEAVIATE_TOKENIZATION=word -# For OceanBase metadata database configuration, available when `DB_TYPE` is `mysql` and `COMPOSE_PROFILES` includes `oceanbase`. +# For OceanBase metadata database configuration, available when `DB_TYPE` is `oceanbase`. # For OceanBase vector database configuration, available when `VECTOR_STORE` is `oceanbase` -# If you want to use OceanBase as both vector database and metadata database, you need to set `DB_TYPE` to `mysql`, `COMPOSE_PROFILES` is `oceanbase`, and set Database Configuration is the same as the vector database. +# If you want to use OceanBase as both vector database and metadata database, you need to set both `DB_TYPE` and `VECTOR_STORE` to `oceanbase`, and set Database Configuration is the same as the vector database. # seekdb is the lite version of OceanBase and shares the connection configuration with OceanBase. OCEANBASE_VECTOR_HOST=oceanbase OCEANBASE_VECTOR_PORT=2881 @@ -957,6 +968,8 @@ SMTP_USERNAME= SMTP_PASSWORD= SMTP_USE_TLS=true SMTP_OPPORTUNISTIC_TLS=false +# Optional: override the local hostname used for SMTP HELO/EHLO +SMTP_LOCAL_HOSTNAME= # Sendgid configuration SENDGRID_API_KEY= @@ -1026,18 +1039,26 @@ WORKFLOW_NODE_EXECUTION_STORAGE=rdbms # Options: # - core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository (default) # - core.repositories.celery_workflow_execution_repository.CeleryWorkflowExecutionRepository +# - extensions.logstore.repositories.logstore_workflow_execution_repository.LogstoreWorkflowExecutionRepository CORE_WORKFLOW_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository # Core workflow node execution repository implementation # Options: # - core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository (default) # - core.repositories.celery_workflow_node_execution_repository.CeleryWorkflowNodeExecutionRepository +# - extensions.logstore.repositories.logstore_workflow_node_execution_repository.LogstoreWorkflowNodeExecutionRepository CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository # API workflow run repository implementation +# Options: +# - repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository (default) +# - extensions.logstore.repositories.logstore_api_workflow_run_repository.LogstoreAPIWorkflowRunRepository API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository # API workflow node execution repository implementation +# Options: +# - repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository (default) +# - extensions.logstore.repositories.logstore_api_workflow_node_execution_repository.LogstoreAPIWorkflowNodeExecutionRepository API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository # Workflow log cleanup configuration @@ -1066,6 +1087,10 @@ LOGSTORE_DUAL_WRITE_ENABLED=false # Enable dual-read fallback to SQL database when LogStore returns no results (default: true) # Useful for migration scenarios where historical data exists only in SQL database LOGSTORE_DUAL_READ_ENABLED=true +# Control flag for whether to write the `graph` field to LogStore. +# If LOGSTORE_ENABLE_PUT_GRAPH_FIELD is "true", write the full `graph` field; +# otherwise write an empty {} instead. Defaults to writing the `graph` field. +LOGSTORE_ENABLE_PUT_GRAPH_FIELD=true # HTTP request node in workflow configuration HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 @@ -1463,6 +1488,7 @@ ENABLE_CLEAN_UNUSED_DATASETS_TASK=false ENABLE_CREATE_TIDB_SERVERLESS_TASK=false ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK=false ENABLE_CLEAN_MESSAGES=false +ENABLE_WORKFLOW_RUN_CLEANUP_TASK=false ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK=false ENABLE_DATASETS_QUEUE_MONITOR=false ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK=true diff --git a/dify/code/docker-compose-template.yaml b/dify/code/docker-compose-template.yaml index 3c88cdd..9659990 100644 --- a/dify/code/docker-compose-template.yaml +++ b/dify/code/docker-compose-template.yaml @@ -21,7 +21,7 @@ services: # API service api: - image: langgenius/dify-api:1.11.2 + image: langgenius/dify-api:1.11.4 restart: always environment: # Use the shared environment variables. @@ -63,7 +63,7 @@ services: # worker service # The Celery worker for processing all queues (dataset, workflow, mail, etc.) worker: - image: langgenius/dify-api:1.11.2 + image: langgenius/dify-api:1.11.4 restart: always environment: # Use the shared environment variables. @@ -102,7 +102,7 @@ services: # worker_beat service # Celery beat for scheduling periodic tasks. worker_beat: - image: langgenius/dify-api:1.11.2 + image: langgenius/dify-api:1.11.4 restart: always environment: # Use the shared environment variables. @@ -132,7 +132,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.11.2 + image: langgenius/dify-web:1.11.4 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -475,7 +475,8 @@ services: OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} OB_SERVER_IP: 127.0.0.1 MODE: mini - LANG: en_US.UTF-8 + LANG: C.UTF-8 + LC_ALL: C.UTF-8 ports: - "${OCEANBASE_VECTOR_PORT:-2881}:2881" healthcheck: diff --git a/dify/code/docker-compose.middleware.yaml b/dify/code/docker-compose.middleware.yaml index dba61d1..81c34fc 100644 --- a/dify/code/docker-compose.middleware.yaml +++ b/dify/code/docker-compose.middleware.yaml @@ -129,6 +129,7 @@ services: - ./middleware.env environment: # Use the shared environment variables. + LOG_OUTPUT_FORMAT: ${LOG_OUTPUT_FORMAT:-text} DB_DATABASE: ${DB_PLUGIN_DATABASE:-dify_plugin} REDIS_HOST: ${REDIS_HOST:-redis} REDIS_PORT: ${REDIS_PORT:-6379} diff --git a/dify/code/docker-compose.yaml b/dify/code/docker-compose.yaml index 1099adf..d8153b7 100644 --- a/dify/code/docker-compose.yaml +++ b/dify/code/docker-compose.yaml @@ -13,10 +13,11 @@ x-shared-env: &shared-api-worker-env APP_WEB_URL: ${APP_WEB_URL:-} FILES_URL: ${FILES_URL:-} INTERNAL_FILES_URL: ${INTERNAL_FILES_URL:-} - LANG: ${LANG:-en_US.UTF-8} - LC_ALL: ${LC_ALL:-en_US.UTF-8} + LANG: ${LANG:-C.UTF-8} + LC_ALL: ${LC_ALL:-C.UTF-8} PYTHONIOENCODING: ${PYTHONIOENCODING:-utf-8} LOG_LEVEL: ${LOG_LEVEL:-INFO} + LOG_OUTPUT_FORMAT: ${LOG_OUTPUT_FORMAT:-text} LOG_FILE: ${LOG_FILE:-/app/logs/server.log} LOG_FILE_MAX_SIZE: ${LOG_FILE_MAX_SIZE:-20} LOG_FILE_BACKUP_COUNT: ${LOG_FILE_BACKUP_COUNT:-5} @@ -122,6 +123,13 @@ x-shared-env: &shared-api-worker-env S3_ACCESS_KEY: ${S3_ACCESS_KEY:-} S3_SECRET_KEY: ${S3_SECRET_KEY:-} S3_USE_AWS_MANAGED_IAM: ${S3_USE_AWS_MANAGED_IAM:-false} + ARCHIVE_STORAGE_ENABLED: ${ARCHIVE_STORAGE_ENABLED:-false} + ARCHIVE_STORAGE_ENDPOINT: ${ARCHIVE_STORAGE_ENDPOINT:-} + ARCHIVE_STORAGE_ARCHIVE_BUCKET: ${ARCHIVE_STORAGE_ARCHIVE_BUCKET:-} + ARCHIVE_STORAGE_EXPORT_BUCKET: ${ARCHIVE_STORAGE_EXPORT_BUCKET:-} + ARCHIVE_STORAGE_ACCESS_KEY: ${ARCHIVE_STORAGE_ACCESS_KEY:-} + ARCHIVE_STORAGE_SECRET_KEY: ${ARCHIVE_STORAGE_SECRET_KEY:-} + ARCHIVE_STORAGE_REGION: ${ARCHIVE_STORAGE_REGION:-auto} AZURE_BLOB_ACCOUNT_NAME: ${AZURE_BLOB_ACCOUNT_NAME:-difyai} AZURE_BLOB_ACCOUNT_KEY: ${AZURE_BLOB_ACCOUNT_KEY:-difyai} AZURE_BLOB_CONTAINER_NAME: ${AZURE_BLOB_CONTAINER_NAME:-difyai-container} @@ -417,6 +425,7 @@ x-shared-env: &shared-api-worker-env SMTP_PASSWORD: ${SMTP_PASSWORD:-} SMTP_USE_TLS: ${SMTP_USE_TLS:-true} SMTP_OPPORTUNISTIC_TLS: ${SMTP_OPPORTUNISTIC_TLS:-false} + SMTP_LOCAL_HOSTNAME: ${SMTP_LOCAL_HOSTNAME:-} SENDGRID_API_KEY: ${SENDGRID_API_KEY:-} INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000} INVITE_EXPIRY_HOURS: ${INVITE_EXPIRY_HOURS:-72} @@ -467,6 +476,7 @@ x-shared-env: &shared-api-worker-env ALIYUN_SLS_LOGSTORE_TTL: ${ALIYUN_SLS_LOGSTORE_TTL:-365} LOGSTORE_DUAL_WRITE_ENABLED: ${LOGSTORE_DUAL_WRITE_ENABLED:-false} LOGSTORE_DUAL_READ_ENABLED: ${LOGSTORE_DUAL_READ_ENABLED:-true} + LOGSTORE_ENABLE_PUT_GRAPH_FIELD: ${LOGSTORE_ENABLE_PUT_GRAPH_FIELD:-true} HTTP_REQUEST_NODE_MAX_BINARY_SIZE: ${HTTP_REQUEST_NODE_MAX_BINARY_SIZE:-10485760} HTTP_REQUEST_NODE_MAX_TEXT_SIZE: ${HTTP_REQUEST_NODE_MAX_TEXT_SIZE:-1048576} HTTP_REQUEST_NODE_SSL_VERIFY: ${HTTP_REQUEST_NODE_SSL_VERIFY:-True} @@ -653,6 +663,7 @@ x-shared-env: &shared-api-worker-env ENABLE_CREATE_TIDB_SERVERLESS_TASK: ${ENABLE_CREATE_TIDB_SERVERLESS_TASK:-false} ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK: ${ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK:-false} ENABLE_CLEAN_MESSAGES: ${ENABLE_CLEAN_MESSAGES:-false} + ENABLE_WORKFLOW_RUN_CLEANUP_TASK: ${ENABLE_WORKFLOW_RUN_CLEANUP_TASK:-false} ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK: ${ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK:-false} ENABLE_DATASETS_QUEUE_MONITOR: ${ENABLE_DATASETS_QUEUE_MONITOR:-false} ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK: ${ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK:-true} @@ -694,7 +705,7 @@ services: # API service api: - image: langgenius/dify-api:1.11.2 + image: langgenius/dify-api:1.11.4 restart: always environment: # Use the shared environment variables. @@ -736,7 +747,7 @@ services: # worker service # The Celery worker for processing all queues (dataset, workflow, mail, etc.) worker: - image: langgenius/dify-api:1.11.2 + image: langgenius/dify-api:1.11.4 restart: always environment: # Use the shared environment variables. @@ -775,7 +786,7 @@ services: # worker_beat service # Celery beat for scheduling periodic tasks. worker_beat: - image: langgenius/dify-api:1.11.2 + image: langgenius/dify-api:1.11.4 restart: always environment: # Use the shared environment variables. @@ -805,7 +816,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.11.2 + image: langgenius/dify-web:1.11.4 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -1146,7 +1157,8 @@ services: OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} OB_SERVER_IP: 127.0.0.1 MODE: mini - LANG: en_US.UTF-8 + LANG: C.UTF-8 + LC_ALL: C.UTF-8 healthcheck: test: [ diff --git a/dify/code/middleware.env.example b/dify/code/middleware.env.example index f7e0252..c88dbe5 100644 --- a/dify/code/middleware.env.example +++ b/dify/code/middleware.env.example @@ -233,4 +233,8 @@ ALIYUN_SLS_LOGSTORE_TTL=365 LOGSTORE_DUAL_WRITE_ENABLED=true # Enable dual-read fallback to SQL database when LogStore returns no results (default: true) # Useful for migration scenarios where historical data exists only in SQL database -LOGSTORE_DUAL_READ_ENABLED=true \ No newline at end of file +LOGSTORE_DUAL_READ_ENABLED=true +# Control flag for whether to write the `graph` field to LogStore. +# If LOGSTORE_ENABLE_PUT_GRAPH_FIELD is "true", write the full `graph` field; +# otherwise write an empty {} instead. Defaults to writing the `graph` field. +LOGSTORE_ENABLE_PUT_GRAPH_FIELD=true \ No newline at end of file diff --git a/dify/code/ssrf_proxy/squid.conf.template b/dify/code/ssrf_proxy/squid.conf.template index 1775a1f..256e669 100644 --- a/dify/code/ssrf_proxy/squid.conf.template +++ b/dify/code/ssrf_proxy/squid.conf.template @@ -54,3 +54,52 @@ http_access allow src_all # Unless the option's size is increased, an error will occur when uploading more than two files. client_request_buffer_max_size 100 MB + +################################## Performance & Concurrency ############################### +# Increase file descriptor limit for high concurrency +max_filedescriptors 65536 + +# Timeout configurations for image requests +connect_timeout 30 seconds +request_timeout 2 minutes +read_timeout 2 minutes +client_lifetime 5 minutes +shutdown_lifetime 30 seconds + +# Persistent connections - improve performance for multiple requests +server_persistent_connections on +client_persistent_connections on +persistent_request_timeout 30 seconds +pconn_timeout 1 minute + +# Connection pool and concurrency limits +client_db on +server_idle_pconn_timeout 2 minutes +client_idle_pconn_timeout 2 minutes + +# Quick abort settings - don't abort requests that are mostly done +quick_abort_min 16 KB +quick_abort_max 16 MB +quick_abort_pct 95 + +# Memory and cache optimization +memory_cache_mode disk +cache_mem 256 MB +maximum_object_size_in_memory 512 KB + +# DNS resolver settings for better performance +dns_timeout 30 seconds +dns_retransmit_interval 5 seconds +# By default, Squid uses the system's configured DNS resolvers. +# If you need to override them, set dns_nameservers to appropriate servers +# for your environment (for example, internal/corporate DNS). The following +# is an example using public DNS and SHOULD be customized before use: +# dns_nameservers 8.8.8.8 8.8.4.4 + +# Logging format for better debugging +logformat dify_log %ts.%03tu %6tr %>a %Ss/%03>Hs % Stopping and removing all containers..." + +if [ -f ".env" ]; then + docker compose -f docker-compose.yml -f ./dev/docker-compose.dev.yml down -v --remove-orphans +elif [ -f ".env.example" ]; then + echo "No .env found, using .env.example for docker compose down..." + docker compose --env-file .env.example -f docker-compose.yml -f ./dev/docker-compose.dev.yml down -v --remove-orphans +else + echo "Skipping 'docker compose down' because there's no env-file." +fi + +echo "===> Cleaning up bind-mounted directories..." +BIND_MOUNTS="./volumes/db/data ./volumes/storage" + +for dir in $BIND_MOUNTS; do + if [ -d "$dir" ]; then + echo "Removing $dir..." + confirm + rm -rf "$dir" + else + echo "$dir not found." + fi done -echo "Resetting .env file..." -if [ -f ".env" ]; then - echo "Removing existing .env file..." - rm -f .env +echo "===> Resetting .env file (will save backup to .env.old)..." +confirm +if [ -f ".env" ] || [ -L ".env" ]; then + echo "Renaming existing .env file to .env.old" + mv .env .env.old else - echo "No .env file found. Skipping .env removal step..." + echo "No .env file found." fi if [ -f ".env.example" ]; then - echo "Copying .env.example to .env..." - cp .env.example .env + echo "===> Copying .env.example to .env" + cp .env.example .env else - echo ".env.example file not found. Skipping .env reset step..." + echo "No .env.example found, can't restore .env to default values." fi -echo "Cleanup complete!" \ No newline at end of file +echo "Cleanup complete!" +echo "Re-run 'docker compose pull' to update images." +echo "" diff --git a/supabase/code/utils/db-passwd.sh b/supabase/code/utils/db-passwd.sh new file mode 100644 index 0000000..40c9122 --- /dev/null +++ b/supabase/code/utils/db-passwd.sh @@ -0,0 +1,157 @@ +#!/bin/sh +# +# Portions of this code are derived from Inder Singh's update-db-pass.sh +# Copyright 2025 Inder Singh. Licensed under Apache License 2.0. +# Original source: +# https://github.com/singh-inder/supabase-automated-self-host/blob/main/docker/update-db-pass.sh +# +# GitHub discussion here: +# https://github.com/supabase/supabase/issues/22605#issuecomment-3323382144 +# +# Changed: +# - POSIX shell compatibility +# - No hardcoded values for database service and admin user +# - Use .env for the admin user and database service port +# - Does _not_ set password for supabase_read_only_user (this role is not +# supposed to have a password) +# - Print all values and confirm before updating +# - Stop on any errors +# +# Heads up: +# - Updating _analytics.source_backends is not needed after PR logflare#2069 +# - Newer Logflare versions use a different table and update connection string +# + +set -e + +if ! docker compose version > /dev/null 2>&1; then + echo "Docker Compose not found." + exit 1 +fi + +if [ ! -f .env ]; then + echo "Missing .env file. Exiting." + exit 1 +fi + +# Generate random hex-only password to avoid issues with SQL/shell +new_passwd="$(openssl rand -hex 16)" +# If replacing with a custom password, avoid using @/?#:& +# https://supabase.com/docs/guides/database/postgres/roles#passwords +# new_passwd="d0notUseSpecialSymbolsForPq123-" + +# Check Postgres service +db_image_prefix="supabase.postgres:" + +compose_output=$(docker compose ps \ + --format '{{.Image}}\t{{.Service}}\t{{.Status}}' 2>/dev/null | \ + grep -m1 "^$db_image_prefix" || true) + +if [ -z "$compose_output" ]; then + echo "Postgres container not found. Exiting." + exit 1 +fi + +db_image=$(echo "$compose_output" | cut -f1) +db_srv_name=$(echo "$compose_output" | cut -f2) +db_srv_status=$(echo "$compose_output" | cut -f3) + +case "$db_srv_status" in + Up*) + ;; + *) + echo "Postgres container status: $db_srv_status" + echo "Exiting." + exit 1 + ;; +esac + +db_srv_port=$(grep "^POSTGRES_PORT=" .env | cut -d '=' -f 2) +port_source=" (.env):" +if [ -z "$db_srv_port" ]; then + db_srv_port="5432" + port_source=" (default):" +fi + +db_admin_user="supabase_admin" + +echo "" +echo "*** Check configuration below before updating database passwords! ***" +echo "" +echo "Service name: $db_srv_name" +echo "Service status: $db_srv_status" +echo "Service port${port_source} $db_srv_port" +echo "Image: $db_image" +echo "" +echo "Admin user: $db_admin_user" + +if ! test -t 0; then + echo "" + echo "Running non-interactively. Not updating passwords." + exit 0 +fi + +echo "New database password: $new_passwd" +echo "" + +printf "Update database passwords? (y/N) " +read -r REPLY +case "$REPLY" in + [Yy]) + ;; + *) + echo "Canceled. Not updating passwords." + exit 0 + ;; +esac + +echo "Updating passwords..." +echo "Connecting to the database service container..." + +docker compose exec -T "$db_srv_name" psql -U "$db_admin_user" -d "_supabase" -v ON_ERROR_STOP=1 </dev/null 2>&1; then + echo "Error: openssl is required but not found." + exit 1 +fi + +jwt_secret="$(gen_base64 30)" + +# Used in get_token() +header='{"alg":"HS256","typ":"JWT"}' +iat=$(date +%s) +exp=$((iat + 5 * 3600 * 24 * 365)) # 5 years + +# Normalizes JSON formatting so that the token matches https://www.jwt.io/ results +anon_payload="{\"role\":\"anon\",\"iss\":\"supabase\",\"iat\":$iat,\"exp\":$exp}" +service_role_payload="{\"role\":\"service_role\",\"iss\":\"supabase\",\"iat\":$iat,\"exp\":$exp}" + +#echo "anon_payload=$anon_payload" +#echo "service_role_payload=$service_role_payload" + +anon_key=$(gen_token "$anon_payload") +service_role_key=$(gen_token "$service_role_payload") + +secret_key_base=$(gen_base64 48) +vault_enc_key=$(gen_hex 16) +pg_meta_crypto_key=$(gen_base64 24) + +logflare_public_access_token=$(gen_base64 24) +logflare_private_access_token=$(gen_base64 24) + +s3_protocol_access_key_id=$(gen_hex 16) +s3_protocol_access_key_secret=$(gen_hex 32) + +echo "" +echo "JWT_SECRET=${jwt_secret}" +echo "" +#echo "Issued at: $iat" +#echo "Expire: $exp" +echo "ANON_KEY=${anon_key}" +echo "SERVICE_ROLE_KEY=${service_role_key}" +echo "" +echo "SECRET_KEY_BASE=${secret_key_base}" +echo "VAULT_ENC_KEY=${vault_enc_key}" +echo "PG_META_CRYPTO_KEY=${pg_meta_crypto_key}" +echo "LOGFLARE_PUBLIC_ACCESS_TOKEN=${logflare_public_access_token}" +echo "LOGFLARE_PRIVATE_ACCESS_TOKEN=${logflare_private_access_token}" +echo "S3_PROTOCOL_ACCESS_KEY_ID=${s3_protocol_access_key_id}" +echo "S3_PROTOCOL_ACCESS_KEY_SECRET=${s3_protocol_access_key_secret}" +echo "" + +postgres_password=$(gen_hex 16) +dashboard_password=$(gen_hex 16) + +echo "POSTGRES_PASSWORD=${postgres_password}" +echo "DASHBOARD_PASSWORD=${dashboard_password}" +echo "" + +if ! test -t 0; then + echo "Running non-interactively. Skipping .env update." + exit 0 +fi + +printf "Update .env file? (y/N) " +read -r REPLY +case "$REPLY" in + [Yy]) + ;; + *) + echo "Not updating .env" + exit 0 + ;; +esac + +echo "Updating .env..." + +sed \ + -i.old \ + -e "s|^JWT_SECRET=.*$|JWT_SECRET=${jwt_secret}|" \ + -e "s|^ANON_KEY=.*$|ANON_KEY=${anon_key}|" \ + -e "s|^SERVICE_ROLE_KEY=.*$|SERVICE_ROLE_KEY=${service_role_key}|" \ + -e "s|^SECRET_KEY_BASE=.*$|SECRET_KEY_BASE=${secret_key_base}|" \ + -e "s|^VAULT_ENC_KEY=.*$|VAULT_ENC_KEY=${vault_enc_key}|" \ + -e "s|^PG_META_CRYPTO_KEY=.*$|PG_META_CRYPTO_KEY=${pg_meta_crypto_key}|" \ + -e "s|^LOGFLARE_PUBLIC_ACCESS_TOKEN=.*$|LOGFLARE_PUBLIC_ACCESS_TOKEN=${logflare_public_access_token}|" \ + -e "s|^LOGFLARE_PRIVATE_ACCESS_TOKEN=.*$|LOGFLARE_PRIVATE_ACCESS_TOKEN=${logflare_private_access_token}|" \ + -e "s|^S3_PROTOCOL_ACCESS_KEY_ID=.*$|S3_PROTOCOL_ACCESS_KEY_ID=${s3_protocol_access_key_id}|" \ + -e "s|^S3_PROTOCOL_ACCESS_KEY_SECRET=.*$|S3_PROTOCOL_ACCESS_KEY_SECRET=${s3_protocol_access_key_secret}|" \ + -e "s|^POSTGRES_PASSWORD=.*$|POSTGRES_PASSWORD=${postgres_password}|" \ + -e "s|^DASHBOARD_PASSWORD=.*$|DASHBOARD_PASSWORD=${dashboard_password}|" \ + .env diff --git a/supabase/code/versions.md b/supabase/code/versions.md new file mode 100644 index 0000000..a26aef0 --- /dev/null +++ b/supabase/code/versions.md @@ -0,0 +1,83 @@ +# Docker Image Versions + +## 2025-12-18 +- supabase/studio:2025.12.17-sha-43f4f7f (prev supabase/studio:2025.12.09-sha-434634f) +- supabase/gotrue:v2.184.0 (prev supabase/gotrue:v2.183.0) +- supabase/postgres-meta:v0.95.1 (prev supabase/postgres-meta:v0.93.1) +- supabase/logflare:1.27.0 (prev supabase/logflare:1.26.25) + +## 2025-12-10 +- supabase/studio:2025.12.09-sha-434634f (prev supabase/studio:2025.11.26-sha-8f096b5) +- postgrest/postgrest:v14.1 (prev postgrest/postgrest:v13.0.7) +- supabase/realtime:v2.68.0 (prev supabase/realtime:v2.65.3) +- supabase/storage-api:v1.33.0 (prev supabase/storage-api:v1.32.0) +- supabase/edge-runtime:v1.69.28 (prev supabase/edge-runtime:v1.69.25) +- supabase/logflare:1.26.25 (prev supabase/logflare:1.26.13) + +## 2025-11-26 +- supabase/studio:2025.11.26-sha-8f096b5 (prev supabase/studio:2025.11.24-sha-d990ae8) +- supabase/realtime:v2.65.3 (prev supabase/realtime:v2.65.2) +- supabase/logflare:1.26.13 (prev supabase/logflare:1.26.12) + +## 2025-11-25 +- supabase/studio:2025.11.24-sha-d990ae8 (prev supabase/studio:2025.11.10-sha-5291fe3) +- supabase/gotrue:v2.183.0 (prev supabase/gotrue:v2.182.1) +- supabase/realtime:v2.65.2 (prev supabase/realtime:v2.63.0) +- supabase/storage-api:v1.32.0 (prev supabase/storage-api:v1.29.0) +- supabase/edge-runtime:v1.69.25 (prev supabase/edge-runtime:v1.69.23) +- supabase/logflare:1.26.12 (prev supabase/logflare:1.22.6) + +## 2025-11-12 +- supabase/studio:2025.11.10-sha-5291fe3 (prev 2025.10.27-sha-85b84e0) +- supabase/gotrue:v2.182.1 (prev v2.180.0) +- supabase/realtime:v2.63.0 (prev v2.57.2) +- supabase/storage-api:v1.29.0 (prev v1.28.2) +- supabase/edge-runtime:v1.69.23 (prev v1.69.15) +- supabase/supavisor:2.7.4 (prev 2.7.3) + +## 2025-10-28 +- supabase/studio:2025.10.27-sha-85b84e0 (prev 2025.10.20-sha-5005fc6) +- supabase/realtime:v2.57.2 (prev v2.56.0) +- supabase/storage-api:v1.28.2 (prev v1.28.1) +- supabase/postgres-meta:v0.93.1 (prev v0.93.0) +- supabase/edge-runtime:v1.69.15 (prev v1.69.14) + +## 2025-10-21 +- supabase/studio:2025.10.20-sha-5005fc6 (prev 2025.10.01-sha-8460121) +- supabase/realtime:v2.56.0 (prev v2.51.11) +- supabase/storage-api:v1.28.1 (prev v1.28.0) +- supabase/postgres-meta:v0.93.0 (prev v0.91.6) +- supabase/edge-runtime:v1.69.14 (prev v1.69.6) +- supabase/supavisor:2.7.3 (prev 2.7.0) + +## 2025-10-13 +- supabase/logflare:1.22.6 (prev 1.22.4) + +## 2025-10-08 +- supabase/studio:2025.10.01-sha-8460121 (prev 2025.06.30-sha-6f5982d) +- supabase/gotrue:v2.180.0 (prev v2.177.0) +- postgrest/postgrest:v13.0.7 (prev v12.2.12) +- supabase/realtime:v2.51.11 (prev v2.34.47) +- supabase/storage-api:v1.28.0 (prev v1.25.7) +- supabase/postgres-meta:v0.91.6 (prev v0.91.0) +- supabase/logflare:1.22.4 (prev 1.14.2) +- supabase/postgres:15.8.1.085 (prev 15.8.1.060) +- supabase/supavisor:2.7.0 (prev 2.5.7) + +## 2025-07-15 +- supabase/gotrue:v2.177.0 (prev v2.176.1) +- supabase/storage-api:v1.25.7 (prev v1.24.7) +- supabase/postgres-meta:v0.91.0 (prev v0.89.3) +- supabase/supavisor:2.5.7 (prev 2.5.6) + +## 2025-07-02 +- supabase/studio:2025.06.30-sha-6f5982d (prev 2025.06.02-sha-8f2993d) +- supabase/gotrue:v2.176.1 (prev v2.174.0) +- supabase/storage-api:v1.24.7 (prev v1.23.0) +- supabase/supavisor:2.5.6 (prev 2.5.1) + +## 2025-06-03 +- supabase/studio:2025.06.02-sha-8f2993d (prev 2025.05.19-sha-3487831) +- supabase/gotrue:v2.174.0 (prev v2.172.1) +- supabase/storage-api:v1.23.0 (prev v1.22.17) +- supabase/postgres-meta:v0.89.3 (prev v0.89.0) diff --git a/supabase/code/volumes/api/kong.yml b/supabase/code/volumes/api/kong.yml index 7abf425..168634f 100644 --- a/supabase/code/volumes/api/kong.yml +++ b/supabase/code/volumes/api/kong.yml @@ -225,6 +225,48 @@ services: allow: - admin + ## Block access to /api/mcp + - name: mcp-blocker + _comment: 'Block direct access to /api/mcp' + url: http://studio:3000/api/mcp + routes: + - name: mcp-blocker-route + strip_path: true + paths: + - /api/mcp + plugins: + - name: request-termination + config: + status_code: 403 + message: "Access is forbidden." + + ## MCP endpoint - local access + - name: mcp + _comment: 'MCP: /mcp -> http://studio:3000/api/mcp (local access)' + url: http://studio:3000/api/mcp + routes: + - name: mcp + strip_path: true + paths: + - /mcp + plugins: + # Block access to /mcp by default + - name: request-termination + config: + status_code: 403 + message: "Access is forbidden." + # Enable local access (danger zone!) + # 1. Comment out the 'request-termination' section above + # 2. Uncomment the entire section below, including 'deny' + # 3. Add your local IPs to the 'allow' list + #- name: cors + #- name: ip-restriction + # config: + # allow: + # - 127.0.0.1 + # - ::1 + # deny: [] + ## Protected Dashboard - catch all remaining routes - name: dashboard _comment: 'Studio: /* -> http://studio:3000/*' diff --git a/supabase/code/volumes/logs/vector.yml b/supabase/code/volumes/logs/vector.yml index 1c438a8..f97f8e1 100644 --- a/supabase/code/volumes/logs/vector.yml +++ b/supabase/code/volumes/logs/vector.yml @@ -33,9 +33,9 @@ transforms: kong: '.appname == "supabase-kong"' auth: '.appname == "supabase-auth"' rest: '.appname == "supabase-rest"' - realtime: '.appname == "supabase-realtime"' + realtime: '.appname == "realtime-dev.supabase-realtime"' storage: '.appname == "supabase-storage"' - functions: '.appname == "supabase-functions"' + functions: '.appname == "supabase-edge-functions"' db: '.appname == "supabase-db"' # Ignores non nginx errors since they are related with kong booting up kong_logs: @@ -117,6 +117,13 @@ transforms: .event_message = parsed.msg .metadata.level = parsed.level } + # Function logs are unstructured messages on stderr + functions_logs: + type: remap + inputs: + - router.functions + source: |- + .metadata.project_ref = del(.project) # Storage logs may contain json objects so we parse them for completeness storage_logs: type: remap @@ -210,7 +217,7 @@ sinks: logflare_functions: type: 'http' inputs: - - router.functions + - functions_logs encoding: codec: 'json' method: 'post' diff --git a/supabase/update.js b/supabase/update.js index fdcaa71..8e8ae13 100644 --- a/supabase/update.js +++ b/supabase/update.js @@ -1,6 +1,6 @@ import utils from "../utils.js"; -await utils.cloneOrPullRepo({ repo: "https://github.com/supabase/supabase" }); +await utils.cloneOrPullRepo({ repo: "https://github.com/supabase/supabase", branch: "master" }); await utils.copyDir("./repo/docker", "./code"); await utils.removeContainerNames("./code/docker-compose.yml");