From e0e7cbeeef133fa09b36f087bc3f3d138ba80046 Mon Sep 17 00:00:00 2001 From: Andrei Canta Date: Thu, 3 Jul 2025 16:01:58 +0300 Subject: [PATCH] update --- appwrite/code/.env.example | 15 ++ appwrite/code/docker-compose.yml | 123 +++++++++------ dify/code/.env.example | 41 ++++- dify/code/docker-compose-template.yaml | 40 ++++- dify/code/docker-compose.middleware.yaml | 7 +- dify/code/docker-compose.yaml | 65 ++++++-- dify/code/middleware.env.example | 10 +- plane/code/README.md | 27 +++- plane/code/restore-airgapped.sh | 144 ++++++++++++++++++ supabase/code/.env.example | 15 +- supabase/code/dev/docker-compose.dev.yml | 2 +- supabase/code/docker-compose.yml | 22 +-- supabase/code/volumes/logs/vector.yml | 28 +++- twenty/code/.env.example | 1 - twenty/code/docker-compose.yml | 5 +- .../datasources/clickhouse-datasource.yaml | 15 ++ .../otel-collector/otel-collector-config.yaml | 24 +++ twenty/code/scripts/install.sh | 2 +- twenty/code/twenty-website/Dockerfile | 4 +- twenty/code/twenty/Dockerfile | 4 +- twenty/code/twenty/entrypoint.sh | 19 ++- 21 files changed, 511 insertions(+), 102 deletions(-) create mode 100755 plane/code/restore-airgapped.sh create mode 100644 twenty/code/grafana/provisioning/datasources/clickhouse-datasource.yaml create mode 100644 twenty/code/otel-collector/otel-collector-config.yaml diff --git a/appwrite/code/.env.example b/appwrite/code/.env.example index 110cf4d..ad44177 100644 --- a/appwrite/code/.env.example +++ b/appwrite/code/.env.example @@ -3,12 +3,17 @@ _APP_LOCALE=en _APP_OPTIONS_ABUSE=enabled _APP_OPTIONS_FORCE_HTTPS=disabled _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled +_APP_OPTIONS_ROUTER_FORCE_HTTPS=disabled _APP_OPTIONS_ROUTER_PROTECTION=disabled _APP_OPENSSL_KEY_V1=your-secret-key _APP_DOMAIN=localhost _APP_CUSTOM_DOMAIN_DENY_LIST=example.com,test.com,app.example.com _APP_DOMAIN_FUNCTIONS=functions.localhost +_APP_DOMAIN_SITES=sites.localhost _APP_DOMAIN_TARGET=localhost +_APP_DOMAIN_TARGET_CNAME=localhost +_APP_DOMAIN_TARGET_AAAA=::1 +_APP_DOMAIN_TARGET_A=127.0.0.1 _APP_CONSOLE_WHITELIST_ROOT=enabled _APP_CONSOLE_WHITELIST_EMAILS= _APP_CONSOLE_WHITELIST_IPS= @@ -79,12 +84,16 @@ _APP_STORAGE_WASABI_SECRET= _APP_STORAGE_WASABI_REGION=eu-central-1 _APP_STORAGE_WASABI_BUCKET= _APP_FUNCTIONS_SIZE_LIMIT=30000000 +_APP_COMPUTE_SIZE_LIMIT=30000000 _APP_FUNCTIONS_BUILD_SIZE_LIMIT=2000000000 _APP_FUNCTIONS_TIMEOUT=900 _APP_FUNCTIONS_BUILD_TIMEOUT=900 +_APP_COMPUTE_BUILD_TIMEOUT=900 _APP_FUNCTIONS_CONTAINERS=10 _APP_FUNCTIONS_CPUS=0 +_APP_COMPUTE_CPUS=0 _APP_FUNCTIONS_MEMORY=0 +_APP_COMPUTE_MEMORY=0 _APP_FUNCTIONS_MEMORY_SWAP=0 _APP_FUNCTIONS_RUNTIMES=node-16.0,php-8.0,python-3.9,ruby-3.0 _APP_EXECUTOR_SECRET=your-secret-key @@ -92,14 +101,19 @@ _APP_EXECUTOR_HOST=http://exc1/v1 _APP_EXECUTOR_RUNTIME_NETWORK=appwrite_runtimes _APP_FUNCTIONS_ENVS=node-16.0,php-7.4,python-3.9,ruby-3.0 _APP_FUNCTIONS_INACTIVE_THRESHOLD=60 +_APP_COMPUTE_INACTIVE_THRESHOLD=60 DOCKERHUB_PULL_USERNAME= DOCKERHUB_PULL_PASSWORD= DOCKERHUB_PULL_EMAIL= OPEN_RUNTIMES_NETWORK=appwrite_runtimes _APP_FUNCTIONS_RUNTIMES_NETWORK=runtimes +_APP_COMPUTE_RUNTIMES_NETWORK=runtimes _APP_DOCKER_HUB_USERNAME= _APP_DOCKER_HUB_PASSWORD= _APP_FUNCTIONS_MAINTENANCE_INTERVAL=3600 +_APP_COMPUTE_MAINTENANCE_INTERVAL=3600 +_APP_SITES_TIMEOUT=900 +_APP_SITES_RUNTIMES=static-1,node-22,flutter-3.29 _APP_VCS_GITHUB_APP_NAME= _APP_VCS_GITHUB_PRIVATE_KEY= _APP_VCS_GITHUB_APP_ID= @@ -108,6 +122,7 @@ _APP_VCS_GITHUB_CLIENT_SECRET= _APP_VCS_GITHUB_WEBHOOK_SECRET= _APP_MAINTENANCE_INTERVAL=86400 _APP_MAINTENANCE_DELAY=0 +_APP_MAINTENANCE_START_TIME=00:00 _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 diff --git a/appwrite/code/docker-compose.yml b/appwrite/code/docker-compose.yml index f3d746a..1d8bde4 100644 --- a/appwrite/code/docker-compose.yml +++ b/appwrite/code/docker-compose.yml @@ -28,7 +28,7 @@ services: - appwrite appwrite: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 <<: *x-logging restart: unless-stopped networks: @@ -49,10 +49,13 @@ services: - traefik.http.routers.appwrite_api_https.tls=true volumes: - appwrite-uploads:/storage/uploads:rw + - appwrite-imports:/storage/imports:rw - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw - appwrite-functions:/storage/functions:rw + - appwrite-sites:/storage/sites:rw + - appwrite-builds:/storage/builds:rw depends_on: - mariadb - redis @@ -74,10 +77,12 @@ services: - _APP_OPTIONS_ABUSE - _APP_OPTIONS_ROUTER_PROTECTION - _APP_OPTIONS_FORCE_HTTPS - - _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS + - _APP_OPTIONS_ROUTER_FORCE_HTTPS - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - - _APP_DOMAIN_TARGET + - _APP_DOMAIN_TARGET_CNAME + - _APP_DOMAIN_TARGET_AAAA + - _APP_DOMAIN_TARGET_A - _APP_DOMAIN_FUNCTIONS - _APP_REDIS_HOST - _APP_REDIS_PORT @@ -121,17 +126,21 @@ services: - _APP_STORAGE_WASABI_SECRET - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET - - _APP_FUNCTIONS_SIZE_LIMIT + - _APP_COMPUTE_SIZE_LIMIT - _APP_FUNCTIONS_TIMEOUT - - _APP_FUNCTIONS_BUILD_TIMEOUT - - _APP_FUNCTIONS_CPUS - - _APP_FUNCTIONS_MEMORY + - _APP_SITES_TIMEOUT + - _APP_COMPUTE_BUILD_TIMEOUT + - _APP_COMPUTE_CPUS + - _APP_COMPUTE_MEMORY - _APP_FUNCTIONS_RUNTIMES + - _APP_SITES_RUNTIMES + - _APP_DOMAIN_SITES - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - _APP_LOGGING_CONFIG - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_DELAY + - _APP_MAINTENANCE_START_TIME - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE @@ -153,9 +162,10 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET - _APP_ASSISTANT_OPENAI_API_KEY + appwrite-console: <<: *x-logging - image: appwrite/console:5.2.58 + image: appwrite/console:6.0.13 restart: unless-stopped networks: - appwrite @@ -175,7 +185,7 @@ services: - traefik.http.routers.appwrite_console_https.tls=true appwrite-realtime: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: realtime <<: *x-logging restart: unless-stopped @@ -217,7 +227,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-audits: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-audits <<: *x-logging restart: unless-stopped @@ -242,7 +252,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-webhooks: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-webhooks <<: *x-logging restart: unless-stopped @@ -269,7 +279,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-deletes: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-deletes <<: *x-logging restart: unless-stopped @@ -282,6 +292,7 @@ services: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw - appwrite-functions:/storage/functions:rw + - appwrite-sites:/storage/sites:rw - appwrite-builds:/storage/builds:rw - appwrite-certificates:/storage/certificates:rw environment: @@ -330,7 +341,7 @@ services: - _APP_EMAIL_CERTIFICATES appwrite-worker-databases: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-databases <<: *x-logging restart: unless-stopped @@ -355,7 +366,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-builds: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-builds <<: *x-logging restart: unless-stopped @@ -366,7 +377,9 @@ services: - mariadb volumes: - appwrite-functions:/storage/functions:rw + - appwrite-sites:/storage/sites:rw - appwrite-builds:/storage/builds:rw + - appwrite-uploads:/storage/uploads:rw environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -387,12 +400,13 @@ services: - _APP_VCS_GITHUB_PRIVATE_KEY - _APP_VCS_GITHUB_APP_ID - _APP_FUNCTIONS_TIMEOUT - - _APP_FUNCTIONS_BUILD_TIMEOUT - - _APP_FUNCTIONS_CPUS - - _APP_FUNCTIONS_MEMORY - - _APP_FUNCTIONS_SIZE_LIMIT + - _APP_SITES_TIMEOUT + - _APP_COMPUTE_BUILD_TIMEOUT + - _APP_COMPUTE_CPUS + - _APP_COMPUTE_MEMORY + - _APP_COMPUTE_SIZE_LIMIT - _APP_OPTIONS_FORCE_HTTPS - - _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS + - _APP_OPTIONS_ROUTER_FORCE_HTTPS - _APP_DOMAIN - _APP_STORAGE_DEVICE - _APP_STORAGE_S3_ACCESS_KEY @@ -416,9 +430,10 @@ services: - _APP_STORAGE_WASABI_SECRET - _APP_STORAGE_WASABI_REGION - _APP_STORAGE_WASABI_BUCKET + - _APP_DOMAIN_SITES appwrite-worker-certificates: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-certificates <<: *x-logging restart: unless-stopped @@ -435,7 +450,9 @@ services: - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - - _APP_DOMAIN_TARGET + - _APP_DOMAIN_TARGET_CNAME + - _APP_DOMAIN_TARGET_AAAA + - _APP_DOMAIN_TARGET_A - _APP_DOMAIN_FUNCTIONS - _APP_EMAIL_CERTIFICATES - _APP_REDIS_HOST @@ -450,7 +467,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-functions: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-functions <<: *x-logging restart: unless-stopped @@ -476,9 +493,10 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_FUNCTIONS_TIMEOUT - - _APP_FUNCTIONS_BUILD_TIMEOUT - - _APP_FUNCTIONS_CPUS - - _APP_FUNCTIONS_MEMORY + - _APP_SITES_TIMEOUT + - _APP_COMPUTE_BUILD_TIMEOUT + - _APP_COMPUTE_CPUS + - _APP_COMPUTE_MEMORY - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - _APP_USAGE_STATS @@ -487,7 +505,7 @@ services: - _APP_LOGGING_CONFIG appwrite-worker-mails: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-mails <<: *x-logging restart: unless-stopped @@ -520,7 +538,7 @@ services: - _APP_OPTIONS_FORCE_HTTPS appwrite-worker-messaging: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-messaging <<: *x-logging restart: unless-stopped @@ -570,12 +588,14 @@ services: - _APP_STORAGE_WASABI_BUCKET appwrite-worker-migrations: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-migrations <<: *x-logging restart: unless-stopped networks: - appwrite + volumes: + - appwrite-imports:/storage/imports:rw depends_on: - mariadb environment: @@ -583,7 +603,9 @@ services: - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - - _APP_DOMAIN_TARGET + - _APP_DOMAIN_TARGET_CNAME + - _APP_DOMAIN_TARGET_AAAA + - _APP_DOMAIN_TARGET_A - _APP_EMAIL_SECURITY - _APP_REDIS_HOST - _APP_REDIS_PORT @@ -599,7 +621,7 @@ services: - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET appwrite-task-maintenance: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: maintenance <<: *x-logging restart: unless-stopped @@ -611,7 +633,9 @@ services: - _APP_ENV - _APP_WORKER_PER_CORE - _APP_DOMAIN - - _APP_DOMAIN_TARGET + - _APP_DOMAIN_TARGET_CNAME + - _APP_DOMAIN_TARGET_AAAA + - _APP_DOMAIN_TARGET_A - _APP_DOMAIN_FUNCTIONS - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST @@ -633,14 +657,12 @@ services: - _APP_MAINTENANCE_RETENTION_SCHEDULES appwrite-task-stats-resources: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: stats-resources <<: *x-logging + restart: unless-stopped networks: - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src depends_on: - redis - mariadb @@ -663,7 +685,7 @@ services: - _APP_STATS_RESOURCES_INTERVAL appwrite-worker-stats-resources: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-stats-resources <<: *x-logging restart: unless-stopped @@ -690,7 +712,7 @@ services: - _APP_STATS_RESOURCES_INTERVAL appwrite-worker-stats-usage: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: worker-stats-usage <<: *x-logging restart: unless-stopped @@ -717,7 +739,7 @@ services: - _APP_USAGE_AGGREGATION_INTERVAL appwrite-task-scheduler-functions: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: schedule-functions <<: *x-logging restart: unless-stopped @@ -741,7 +763,7 @@ services: - _APP_DB_PASS appwrite-task-scheduler-executions: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: schedule-executions <<: *x-logging restart: unless-stopped @@ -765,7 +787,7 @@ services: - _APP_DB_PASS appwrite-task-scheduler-messages: - image: appwrite/appwrite:1.6.2 + image: appwrite/appwrite:1.7.4 entrypoint: schedule-messages <<: *x-logging restart: unless-stopped @@ -797,12 +819,19 @@ services: environment: - _APP_ASSISTANT_OPENAI_API_KEY + appwrite-browser: + image: appwrite/browser:0.2.4 + <<: *x-logging + restart: unless-stopped + networks: + - appwrite + openruntimes-executor: hostname: exc1 <<: *x-logging restart: unless-stopped stop_signal: SIGINT - image: openruntimes/executor:0.6.11 + image: openruntimes/executor:0.7.14 networks: - appwrite - runtimes @@ -810,18 +839,20 @@ services: - /var/run/docker.sock:/var/run/docker.sock - appwrite-builds:/storage/builds:rw - appwrite-functions:/storage/functions:rw + - appwrite-sites:/storage/sites:rw # Host mount nessessary to share files between executor and runtimes. # It's not possible to share mount file between 2 containers without host mount (copying is too slow) - /tmp:/tmp:rw environment: - - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD - - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL - - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK + - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_COMPUTE_INACTIVE_THRESHOLD + - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_COMPUTE_MAINTENANCE_INTERVAL + - OPR_EXECUTOR_NETWORK=$_APP_COMPUTE_RUNTIMES_NETWORK - OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_ENV=$_APP_ENV - - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES + - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES,$_APP_SITES_RUNTIMES - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET + - OPR_EXECUTOR_RUNTIME_VERSIONS=v5 - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG - OPR_EXECUTOR_STORAGE_DEVICE=$_APP_STORAGE_DEVICE - OPR_EXECUTOR_STORAGE_S3_ACCESS_KEY=$_APP_STORAGE_S3_ACCESS_KEY @@ -895,7 +926,9 @@ volumes: appwrite-redis: appwrite-cache: appwrite-uploads: + appwrite-imports: appwrite-certificates: appwrite-functions: + appwrite-sites: appwrite-builds: appwrite-config: diff --git a/dify/code/.env.example b/dify/code/.env.example index aacef0e..a024566 100644 --- a/dify/code/.env.example +++ b/dify/code/.env.example @@ -285,6 +285,7 @@ BROKER_USE_SSL=false # If you are using Redis Sentinel for high availability, configure the following settings. CELERY_USE_SENTINEL=false CELERY_SENTINEL_MASTER_NAME= +CELERY_SENTINEL_PASSWORD= CELERY_SENTINEL_SOCKET_TIMEOUT=0.1 # ------------------------------ @@ -399,7 +400,7 @@ SUPABASE_URL=your-server-url # ------------------------------ # The type of vector store to use. -# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`, `opengauss`, `tablestore`,`vastbase`,`tidb`,`tidb_on_qdrant`,`baidu`,`lindorm`,`huawei_cloud`,`upstash`. +# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`, `opengauss`, `tablestore`,`vastbase`,`tidb`,`tidb_on_qdrant`,`baidu`,`lindorm`,`huawei_cloud`,`upstash`, `matrixone`. VECTOR_STORE=weaviate # The Weaviate endpoint URL. Only available when VECTOR_STORE is `weaviate`. @@ -412,6 +413,7 @@ QDRANT_API_KEY=difyai123456 QDRANT_CLIENT_TIMEOUT=20 QDRANT_GRPC_ENABLED=false QDRANT_GRPC_PORT=6334 +QDRANT_REPLICATION_FACTOR=1 # Milvus configuration. Only available when VECTOR_STORE is `milvus`. # The milvus uri. @@ -489,6 +491,13 @@ TIDB_VECTOR_USER= TIDB_VECTOR_PASSWORD= TIDB_VECTOR_DATABASE=dify +# Matrixone vector configurations. +MATRIXONE_HOST=matrixone +MATRIXONE_PORT=6001 +MATRIXONE_USER=dump +MATRIXONE_PASSWORD=111 +MATRIXONE_DATABASE=dify + # Tidb on qdrant configuration, only available when VECTOR_STORE is `tidb_on_qdrant` TIDB_ON_QDRANT_URL=http://127.0.0.1 TIDB_ON_QDRANT_API_KEY=dify @@ -531,6 +540,7 @@ RELYT_DATABASE=postgres OPENSEARCH_HOST=opensearch OPENSEARCH_PORT=9200 OPENSEARCH_SECURE=true +OPENSEARCH_VERIFY_CERTS=true OPENSEARCH_AUTH_METHOD=basic OPENSEARCH_USER=admin OPENSEARCH_PASSWORD=admin @@ -717,10 +727,11 @@ NOTION_INTERNAL_SECRET= # Mail related configuration # ------------------------------ -# Mail type, support: resend, smtp +# Mail type, support: resend, smtp, sendgrid MAIL_TYPE=resend # Default send from email address, if not specified +# If using SendGrid, use the 'from' field for authentication if necessary. MAIL_DEFAULT_SEND_FROM= # API-Key for the Resend email provider, used when MAIL_TYPE is `resend`. @@ -736,6 +747,9 @@ SMTP_PASSWORD= SMTP_USE_TLS=true SMTP_OPPORTUNISTIC_TLS=false +# Sendgid configuration +SENDGRID_API_KEY= + # ------------------------------ # Others Configuration # ------------------------------ @@ -785,6 +799,9 @@ HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576 HTTP_REQUEST_NODE_SSL_VERIFY=True +# Respect X-* headers to redirect clients +RESPECT_XFORWARD_HEADERS_ENABLED=false + # SSRF Proxy server HTTP URL SSRF_PROXY_HTTP_URL=http://ssrf_proxy:3128 # SSRF Proxy server HTTPS URL @@ -800,7 +817,7 @@ MAX_TOOLS_NUM=10 MAX_PARALLEL_LIMIT=10 # The maximum number of iterations for agent setting -MAX_ITERATIONS_NUM=5 +MAX_ITERATIONS_NUM=99 # ------------------------------ # Environment Variables for web Service @@ -813,7 +830,8 @@ TEXT_GENERATION_TIMEOUT_MS=60000 # Environment Variables for db Service # ------------------------------ -PGUSER=${DB_USERNAME} +# The name of the default postgres user. +POSTGRES_USER=${DB_USERNAME} # The password for the default postgres user. POSTGRES_PASSWORD=${DB_PASSWORD} # The name of the default postgres database. @@ -1055,7 +1073,7 @@ PLUGIN_MAX_EXECUTION_TIMEOUT=600 PIP_MIRROR_URL= # https://github.com/langgenius/dify-plugin-daemon/blob/main/.env.example -# Plugin storage type, local aws_s3 tencent_cos azure_blob aliyun_oss +# Plugin storage type, local aws_s3 tencent_cos azure_blob aliyun_oss volcengine_tos PLUGIN_STORAGE_TYPE=local PLUGIN_STORAGE_LOCAL_ROOT=/app/storage PLUGIN_WORKING_PATH=/app/storage/cwd @@ -1065,6 +1083,7 @@ PLUGIN_MEDIA_CACHE_PATH=assets # Plugin oss bucket PLUGIN_STORAGE_OSS_BUCKET= # Plugin oss s3 credentials +PLUGIN_S3_USE_AWS=false PLUGIN_S3_USE_AWS_MANAGED_IAM=false PLUGIN_S3_ENDPOINT= PLUGIN_S3_USE_PATH_STYLE=false @@ -1085,6 +1104,11 @@ PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID= PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET= PLUGIN_ALIYUN_OSS_AUTH_VERSION=v4 PLUGIN_ALIYUN_OSS_PATH= +# Plugin oss volcengine tos +PLUGIN_VOLCENGINE_TOS_ENDPOINT= +PLUGIN_VOLCENGINE_TOS_ACCESS_KEY= +PLUGIN_VOLCENGINE_TOS_SECRET_KEY= +PLUGIN_VOLCENGINE_TOS_REGION= # ------------------------------ # OTLP Collector Configuration @@ -1104,3 +1128,10 @@ OTEL_METRIC_EXPORT_TIMEOUT=30000 # Prevent Clickjacking ALLOW_EMBED=false + +# Dataset queue monitor configuration +QUEUE_MONITOR_THRESHOLD=200 +# You can configure multiple ones, separated by commas. eg: test1@dify.ai,test2@dify.ai +QUEUE_MONITOR_ALERT_EMAILS= +# Monitor interval in minutes, default is 30 minutes +QUEUE_MONITOR_INTERVAL=30 diff --git a/dify/code/docker-compose-template.yaml b/dify/code/docker-compose-template.yaml index ceb32e4..d45f8f8 100644 --- a/dify/code/docker-compose-template.yaml +++ b/dify/code/docker-compose-template.yaml @@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:1.4.0 + image: langgenius/dify-api:1.5.1 restart: always environment: # Use the shared environment variables. @@ -31,7 +31,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:1.4.0 + image: langgenius/dify-api:1.5.1 restart: always environment: # Use the shared environment variables. @@ -57,7 +57,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.4.0 + image: langgenius/dify-web:1.5.1 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -75,7 +75,7 @@ services: LOOP_NODE_MAX_COUNT: ${LOOP_NODE_MAX_COUNT:-100} MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} - MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-5} + MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99} ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} @@ -84,7 +84,7 @@ services: image: postgres:15-alpine restart: always environment: - PGUSER: ${PGUSER:-postgres} + POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-difyai123456} POSTGRES_DB: ${POSTGRES_DB:-dify} PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata} @@ -142,7 +142,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.0.10-local + image: langgenius/dify-plugin-daemon:0.1.3-local restart: always environment: # Use the shared environment variables. @@ -168,6 +168,7 @@ services: PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} + S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false} S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} @@ -184,6 +185,10 @@ services: ALIYUN_OSS_ACCESS_KEY_SECRET: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET:-} ALIYUN_OSS_AUTH_VERSION: ${PLUGIN_ALIYUN_OSS_AUTH_VERSION:-v4} ALIYUN_OSS_PATH: ${PLUGIN_ALIYUN_OSS_PATH:-} + VOLCENGINE_TOS_ENDPOINT: ${PLUGIN_VOLCENGINE_TOS_ENDPOINT:-} + VOLCENGINE_TOS_ACCESS_KEY: ${PLUGIN_VOLCENGINE_TOS_ACCESS_KEY:-} + VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-} + VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-} ports: - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}" volumes: @@ -430,7 +435,7 @@ services: # OceanBase vector database oceanbase: - image: oceanbase/oceanbase-ce:4.3.5.1-101000042025031818 + image: oceanbase/oceanbase-ce:4.3.5-lts container_name: oceanbase profiles: - oceanbase @@ -444,9 +449,16 @@ services: OB_SYS_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} OB_TENANT_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} - MODE: MINI + OB_SERVER_IP: 127.0.0.1 + MODE: mini ports: - "${OCEANBASE_VECTOR_PORT:-2881}:2881" + healthcheck: + test: [ 'CMD-SHELL', 'obclient -h127.0.0.1 -P2881 -uroot@test -p$${OB_TENANT_PASSWORD} -e "SELECT 1;"' ] + interval: 10s + retries: 30 + start_period: 30s + timeout: 10s # Oracle vector database oracle: @@ -605,6 +617,18 @@ services: ports: - ${MYSCALE_PORT:-8123}:${MYSCALE_PORT:-8123} + # Matrixone vector store. + matrixone: + hostname: matrixone + image: matrixorigin/matrixone:2.1.1 + profiles: + - matrixone + restart: always + volumes: + - ./volumes/matrixone/data:/mo-data + ports: + - ${MATRIXONE_PORT:-6001}:${MATRIXONE_PORT:-6001} + # https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html # https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-prod-prerequisites elasticsearch: diff --git a/dify/code/docker-compose.middleware.yaml b/dify/code/docker-compose.middleware.yaml index 8690a50..0b18857 100644 --- a/dify/code/docker-compose.middleware.yaml +++ b/dify/code/docker-compose.middleware.yaml @@ -71,7 +71,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.0.10-local + image: langgenius/dify-plugin-daemon:0.1.3-local restart: always env_file: - ./middleware.env @@ -104,6 +104,7 @@ services: PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false} S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} @@ -121,6 +122,10 @@ services: ALIYUN_OSS_ACCESS_KEY_SECRET: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET:-} ALIYUN_OSS_AUTH_VERSION: ${PLUGIN_ALIYUN_OSS_AUTH_VERSION:-v4} ALIYUN_OSS_PATH: ${PLUGIN_ALIYUN_OSS_PATH:-} + VOLCENGINE_TOS_ENDPOINT: ${PLUGIN_VOLCENGINE_TOS_ENDPOINT:-} + VOLCENGINE_TOS_ACCESS_KEY: ${PLUGIN_VOLCENGINE_TOS_ACCESS_KEY:-} + VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-} + VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-} ports: - "${EXPOSE_PLUGIN_DAEMON_PORT:-5002}:${PLUGIN_DAEMON_PORT:-5002}" - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}" diff --git a/dify/code/docker-compose.yaml b/dify/code/docker-compose.yaml index 90dd29c..9b65af5 100644 --- a/dify/code/docker-compose.yaml +++ b/dify/code/docker-compose.yaml @@ -79,6 +79,7 @@ x-shared-env: &shared-api-worker-env BROKER_USE_SSL: ${BROKER_USE_SSL:-false} CELERY_USE_SENTINEL: ${CELERY_USE_SENTINEL:-false} CELERY_SENTINEL_MASTER_NAME: ${CELERY_SENTINEL_MASTER_NAME:-} + CELERY_SENTINEL_PASSWORD: ${CELERY_SENTINEL_PASSWORD:-} CELERY_SENTINEL_SOCKET_TIMEOUT: ${CELERY_SENTINEL_SOCKET_TIMEOUT:-0.1} WEB_API_CORS_ALLOW_ORIGINS: ${WEB_API_CORS_ALLOW_ORIGINS:-*} CONSOLE_CORS_ALLOW_ORIGINS: ${CONSOLE_CORS_ALLOW_ORIGINS:-*} @@ -138,6 +139,7 @@ x-shared-env: &shared-api-worker-env QDRANT_CLIENT_TIMEOUT: ${QDRANT_CLIENT_TIMEOUT:-20} QDRANT_GRPC_ENABLED: ${QDRANT_GRPC_ENABLED:-false} QDRANT_GRPC_PORT: ${QDRANT_GRPC_PORT:-6334} + QDRANT_REPLICATION_FACTOR: ${QDRANT_REPLICATION_FACTOR:-1} MILVUS_URI: ${MILVUS_URI:-http://host.docker.internal:19530} MILVUS_DATABASE: ${MILVUS_DATABASE:-} MILVUS_TOKEN: ${MILVUS_TOKEN:-} @@ -194,6 +196,11 @@ x-shared-env: &shared-api-worker-env TIDB_VECTOR_USER: ${TIDB_VECTOR_USER:-} TIDB_VECTOR_PASSWORD: ${TIDB_VECTOR_PASSWORD:-} TIDB_VECTOR_DATABASE: ${TIDB_VECTOR_DATABASE:-dify} + MATRIXONE_HOST: ${MATRIXONE_HOST:-matrixone} + MATRIXONE_PORT: ${MATRIXONE_PORT:-6001} + MATRIXONE_USER: ${MATRIXONE_USER:-dump} + MATRIXONE_PASSWORD: ${MATRIXONE_PASSWORD:-111} + MATRIXONE_DATABASE: ${MATRIXONE_DATABASE:-dify} TIDB_ON_QDRANT_URL: ${TIDB_ON_QDRANT_URL:-http://127.0.0.1} TIDB_ON_QDRANT_API_KEY: ${TIDB_ON_QDRANT_API_KEY:-dify} TIDB_ON_QDRANT_CLIENT_TIMEOUT: ${TIDB_ON_QDRANT_CLIENT_TIMEOUT:-20} @@ -227,6 +234,7 @@ x-shared-env: &shared-api-worker-env OPENSEARCH_HOST: ${OPENSEARCH_HOST:-opensearch} OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200} OPENSEARCH_SECURE: ${OPENSEARCH_SECURE:-true} + OPENSEARCH_VERIFY_CERTS: ${OPENSEARCH_VERIFY_CERTS:-true} OPENSEARCH_AUTH_METHOD: ${OPENSEARCH_AUTH_METHOD:-basic} OPENSEARCH_USER: ${OPENSEARCH_USER:-admin} OPENSEARCH_PASSWORD: ${OPENSEARCH_PASSWORD:-admin} @@ -320,6 +328,7 @@ x-shared-env: &shared-api-worker-env SMTP_PASSWORD: ${SMTP_PASSWORD:-} SMTP_USE_TLS: ${SMTP_USE_TLS:-true} SMTP_OPPORTUNISTIC_TLS: ${SMTP_OPPORTUNISTIC_TLS:-false} + SENDGRID_API_KEY: ${SENDGRID_API_KEY:-} INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000} INVITE_EXPIRY_HOURS: ${INVITE_EXPIRY_HOURS:-72} RESET_PASSWORD_TOKEN_EXPIRY_MINUTES: ${RESET_PASSWORD_TOKEN_EXPIRY_MINUTES:-5} @@ -347,14 +356,15 @@ x-shared-env: &shared-api-worker-env HTTP_REQUEST_NODE_MAX_BINARY_SIZE: ${HTTP_REQUEST_NODE_MAX_BINARY_SIZE:-10485760} HTTP_REQUEST_NODE_MAX_TEXT_SIZE: ${HTTP_REQUEST_NODE_MAX_TEXT_SIZE:-1048576} HTTP_REQUEST_NODE_SSL_VERIFY: ${HTTP_REQUEST_NODE_SSL_VERIFY:-True} + RESPECT_XFORWARD_HEADERS_ENABLED: ${RESPECT_XFORWARD_HEADERS_ENABLED:-false} SSRF_PROXY_HTTP_URL: ${SSRF_PROXY_HTTP_URL:-http://ssrf_proxy:3128} SSRF_PROXY_HTTPS_URL: ${SSRF_PROXY_HTTPS_URL:-http://ssrf_proxy:3128} LOOP_NODE_MAX_COUNT: ${LOOP_NODE_MAX_COUNT:-100} MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} - MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-5} + MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99} TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} - PGUSER: ${PGUSER:-${DB_USERNAME}} + POSTGRES_USER: ${POSTGRES_USER:-${DB_USERNAME}} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-${DB_PASSWORD}} POSTGRES_DB: ${POSTGRES_DB:-${DB_DATABASE}} PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata} @@ -465,6 +475,7 @@ x-shared-env: &shared-api-worker-env PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} + PLUGIN_S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false} PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} @@ -482,6 +493,10 @@ x-shared-env: &shared-api-worker-env PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET:-} PLUGIN_ALIYUN_OSS_AUTH_VERSION: ${PLUGIN_ALIYUN_OSS_AUTH_VERSION:-v4} PLUGIN_ALIYUN_OSS_PATH: ${PLUGIN_ALIYUN_OSS_PATH:-} + PLUGIN_VOLCENGINE_TOS_ENDPOINT: ${PLUGIN_VOLCENGINE_TOS_ENDPOINT:-} + PLUGIN_VOLCENGINE_TOS_ACCESS_KEY: ${PLUGIN_VOLCENGINE_TOS_ACCESS_KEY:-} + PLUGIN_VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-} + PLUGIN_VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-} ENABLE_OTEL: ${ENABLE_OTEL:-false} OTLP_BASE_ENDPOINT: ${OTLP_BASE_ENDPOINT:-http://localhost:4318} OTLP_API_KEY: ${OTLP_API_KEY:-} @@ -495,11 +510,14 @@ x-shared-env: &shared-api-worker-env OTEL_BATCH_EXPORT_TIMEOUT: ${OTEL_BATCH_EXPORT_TIMEOUT:-10000} OTEL_METRIC_EXPORT_TIMEOUT: ${OTEL_METRIC_EXPORT_TIMEOUT:-30000} ALLOW_EMBED: ${ALLOW_EMBED:-false} + QUEUE_MONITOR_THRESHOLD: ${QUEUE_MONITOR_THRESHOLD:-200} + QUEUE_MONITOR_ALERT_EMAILS: ${QUEUE_MONITOR_ALERT_EMAILS:-} + QUEUE_MONITOR_INTERVAL: ${QUEUE_MONITOR_INTERVAL:-30} services: # API service api: - image: langgenius/dify-api:1.4.0 + image: langgenius/dify-api:1.5.1 restart: always environment: # Use the shared environment variables. @@ -528,7 +546,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:1.4.0 + image: langgenius/dify-api:1.5.1 restart: always environment: # Use the shared environment variables. @@ -554,7 +572,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:1.4.0 + image: langgenius/dify-web:1.5.1 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -572,7 +590,7 @@ services: LOOP_NODE_MAX_COUNT: ${LOOP_NODE_MAX_COUNT:-100} MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} - MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-5} + MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99} ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} @@ -581,7 +599,7 @@ services: image: postgres:15-alpine restart: always environment: - PGUSER: ${PGUSER:-postgres} + POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-difyai123456} POSTGRES_DB: ${POSTGRES_DB:-dify} PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata} @@ -649,7 +667,7 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.0.10-local + image: langgenius/dify-plugin-daemon:0.1.3-local restart: always environment: # Use the shared environment variables. @@ -675,6 +693,7 @@ services: PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} + S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false} S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} @@ -691,6 +710,10 @@ services: ALIYUN_OSS_ACCESS_KEY_SECRET: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET:-} ALIYUN_OSS_AUTH_VERSION: ${PLUGIN_ALIYUN_OSS_AUTH_VERSION:-v4} ALIYUN_OSS_PATH: ${PLUGIN_ALIYUN_OSS_PATH:-} + VOLCENGINE_TOS_ENDPOINT: ${PLUGIN_VOLCENGINE_TOS_ENDPOINT:-} + VOLCENGINE_TOS_ACCESS_KEY: ${PLUGIN_VOLCENGINE_TOS_ACCESS_KEY:-} + VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-} + VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-} volumes: - ./volumes/plugin_daemon:/app/storage depends_on: @@ -949,7 +972,7 @@ services: # OceanBase vector database oceanbase: - image: oceanbase/oceanbase-ce:4.3.5.1-101000042025031818 + image: oceanbase/oceanbase-ce:4.3.5-lts profiles: - oceanbase restart: always @@ -962,7 +985,19 @@ services: OB_SYS_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} OB_TENANT_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} - MODE: MINI + OB_SERVER_IP: 127.0.0.1 + MODE: mini + healthcheck: + test: + [ + 'CMD-SHELL', + 'obclient -h127.0.0.1 -P2881 -uroot@test -p$${OB_TENANT_PASSWORD} -e + "SELECT 1;"' + ] + interval: 10s + retries: 30 + start_period: 30s + timeout: 10s # Oracle vector database oracle: @@ -1110,6 +1145,16 @@ services: - ./volumes/myscale/log:/var/log/clickhouse-server - ./volumes/myscale/config/users.d/custom_users_config.xml:/etc/clickhouse-server/users.d/custom_users_config.xml + # Matrixone vector store. + matrixone: + hostname: matrixone + image: matrixorigin/matrixone:2.1.1 + profiles: + - matrixone + restart: always + volumes: + - ./volumes/matrixone/data:/mo-data + # https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html # https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-prod-prerequisites elasticsearch: diff --git a/dify/code/middleware.env.example b/dify/code/middleware.env.example index 2437026..2eba62f 100644 --- a/dify/code/middleware.env.example +++ b/dify/code/middleware.env.example @@ -1,7 +1,7 @@ # ------------------------------ # Environment Variables for db Service # ------------------------------ -PGUSER=postgres +POSTGRES_USER=postgres # The password for the default postgres user. POSTGRES_PASSWORD=difyai123456 # The name of the default postgres database. @@ -109,7 +109,7 @@ EXPOSE_PLUGIN_DEBUGGING_HOST=localhost EXPOSE_PLUGIN_DEBUGGING_PORT=5003 PLUGIN_DIFY_INNER_API_KEY=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1 -PLUGIN_DIFY_INNER_API_URL=http://api:5001 +PLUGIN_DIFY_INNER_API_URL=http://host.docker.internal:5001 MARKETPLACE_ENABLED=true MARKETPLACE_API_URL=https://marketplace.dify.ai @@ -133,6 +133,7 @@ PLUGIN_MEDIA_CACHE_PATH=assets PLUGIN_STORAGE_OSS_BUCKET= # Plugin oss s3 credentials PLUGIN_S3_USE_AWS_MANAGED_IAM=false +PLUGIN_S3_USE_AWS=false PLUGIN_S3_ENDPOINT= PLUGIN_S3_USE_PATH_STYLE=false PLUGIN_AWS_ACCESS_KEY= @@ -152,3 +153,8 @@ PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID= PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET= PLUGIN_ALIYUN_OSS_AUTH_VERSION=v4 PLUGIN_ALIYUN_OSS_PATH= +# Plugin oss volcengine tos +PLUGIN_VOLCENGINE_TOS_ENDPOINT= +PLUGIN_VOLCENGINE_TOS_ACCESS_KEY= +PLUGIN_VOLCENGINE_TOS_SECRET_KEY= +PLUGIN_VOLCENGINE_TOS_REGION= diff --git a/plane/code/README.md b/plane/code/README.md index e5a8089..ba7af0e 100644 --- a/plane/code/README.md +++ b/plane/code/README.md @@ -486,7 +486,7 @@ When you want to restore the previously backed-up data, follow the instructions 1. Download the restore script using the command below. We suggest downloading it in the same folder as `setup.sh`. ```bash - curl -fsSL -o restore.sh https://raw.githubusercontent.com/makeplane/plane/master/deploy/selfhost/restore.sh + curl -fsSL -o restore.sh https://github.com/makeplane/plane/releases/latest/download/restore.sh chmod +x restore.sh ``` @@ -529,6 +529,31 @@ When you want to restore the previously backed-up data, follow the instructions --- +### Restore for Commercial Air-Gapped (Docker Compose) + +When you want to restore the previously backed-up data on Plane Commercial Air-Gapped version, follow the instructions below. + +1. Download the restore script using the command below + + ```bash + curl -fsSL -o restore-airgapped.sh https://github.com/makeplane/plane/releases/latest/download/restore-airgapped.sh + chmod +x restore-airgapped.sh + ``` + +1. Copy the backup folder and the `restore-airgapped.sh` to `Commercial Airgapped Edition` server + +1. Make sure that Plane Commercial (Airgapped) is extracted and ready to get started. In case it is running, you would need to stop that. + +1. Execute the command below to restore your data. + + ```bash + ./restore-airgapped.sh + ``` + +1. After restoration, you are ready to start Plane Commercial (Airgapped) will all your previously saved data. + +--- +

Upgrading from v0.13.2 to v0.14.x

 diff --git a/plane/code/restore-airgapped.sh b/plane/code/restore-airgapped.sh new file mode 100755 index 0000000..9da02fd --- /dev/null +++ b/plane/code/restore-airgapped.sh @@ -0,0 +1,144 @@ +#!/bin/bash ++set -euo pipefail + +function print_header() { +clear + +cat <<"EOF" +-------------------------------------------- + ____ _ ///////// +| _ \| | __ _ _ __ ___ ///////// +| |_) | |/ _` | '_ \ / _ \ ///// ///// +| __/| | (_| | | | | __/ ///// ///// +|_| |_|\__,_|_| |_|\___| //// + //// +-------------------------------------------- +Project management tool from the future +-------------------------------------------- +EOF +} + +function restoreData() { + + echo "" + echo "****************************************************" + echo "We are about to restore your data from the backup files." + echo "****************************************************" + echo "" + + # set the backup folder path + BACKUP_FOLDER=${1} + + if [ -z "$BACKUP_FOLDER" ]; then + BACKUP_FOLDER="$PWD/backup" + read -p "Enter the backup folder path [$BACKUP_FOLDER]: " BACKUP_FOLDER + if [ -z "$BACKUP_FOLDER" ]; then + BACKUP_FOLDER="$PWD/backup" + fi + fi + + # check if the backup folder exists + if [ ! -d "$BACKUP_FOLDER" ]; then + echo "Error: Backup folder not found at $BACKUP_FOLDER" + exit 1 + fi + + # check if there are any .tar.gz files in the backup folder + if ! ls "$BACKUP_FOLDER"/*.tar.gz 1> /dev/null 2>&1; then + echo "Error: Backup folder does not contain .tar.gz files" + exit 1 + fi + + echo "" + echo "Using backup folder: $BACKUP_FOLDER" + echo "" + + # ask for current install path + AIRGAPPED_INSTALL_PATH="$HOME/planeairgapped" + read -p "Enter the airgapped instance install path [$AIRGAPPED_INSTALL_PATH]: " AIRGAPPED_INSTALL_PATH + if [ -z "$AIRGAPPED_INSTALL_PATH" ]; then + AIRGAPPED_INSTALL_PATH="$HOME/planeairgapped" + fi + + # check if the airgapped instance install path exists + if [ ! -d "$AIRGAPPED_INSTALL_PATH" ]; then + echo "Error: Airgapped instance install path not found at $AIRGAPPED_INSTALL_PATH" + exit 1 + fi + + echo "" + echo "Using airgapped instance install path: $AIRGAPPED_INSTALL_PATH" + echo "" + + # check if the docker-compose.yaml exists + if [ ! -f "$AIRGAPPED_INSTALL_PATH/docker-compose.yml" ]; then + echo "Error: docker-compose.yml not found at $AIRGAPPED_INSTALL_PATH/docker-compose.yml" + exit 1 + fi + + local dockerServiceStatus + if command -v jq &> /dev/null; then + dockerServiceStatus=$($COMPOSE_CMD ls --filter name=plane-airgapped --format=json | jq -r .[0].Status) + else + dockerServiceStatus=$($COMPOSE_CMD ls --filter name=plane-airgapped | grep -o "running" | head -n 1) + fi + + if [[ $dockerServiceStatus == "running" ]]; then + echo "Plane Airgapped is running. Please STOP the Plane Airgapped before restoring data." + exit 1 + fi + + CURRENT_USER_ID=$(id -u) + CURRENT_GROUP_ID=$(id -g) + + # if the data folder not exists, create it + if [ ! -d "$AIRGAPPED_INSTALL_PATH/data" ]; then + mkdir -p "$AIRGAPPED_INSTALL_PATH/data" + chown -R $CURRENT_USER_ID:$CURRENT_GROUP_ID "$AIRGAPPED_INSTALL_PATH/data" + fi + + for BACKUP_FILE in "$BACKUP_FOLDER/*.tar.gz"; do + if [ -e "$BACKUP_FILE" ]; then + + # get the basefilename without the extension + BASE_FILE_NAME=$(basename "$BACKUP_FILE" ".tar.gz") + + # extract the restoreFile to the airgapped instance install path + echo "Restoring $BASE_FILE_NAME" + rm -rf "$AIRGAPPED_INSTALL_PATH/data/$BASE_FILE_NAME" || true + + tar -xvzf "$BACKUP_FILE" -C "$AIRGAPPED_INSTALL_PATH/data/" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $BACKUP_FILE" + exit 1 + fi + chown -R $CURRENT_USER_ID:$CURRENT_GROUP_ID "$AIRGAPPED_INSTALL_PATH/data/$BASE_FILE_NAME" + if [ $? -ne 0 ]; then + echo "Error: Failed to change ownership of $AIRGAPPED_INSTALL_PATH/data/$BASE_FILE_NAME" + exit 1 + fi + else + echo "No .tar.gz files found in the current directory." + echo "" + echo "Please provide the path to the backup file." + echo "" + echo "Usage: $0 /path/to/backup" + exit 1 + fi + done + + echo "" + echo "Restore completed successfully." + echo "" +} + +# if docker-compose is installed +if command -v docker-compose &> /dev/null +then + COMPOSE_CMD="docker-compose" +else + COMPOSE_CMD="docker compose" +fi + +print_header +restoreData "$@" diff --git a/supabase/code/.env.example b/supabase/code/.env.example index bb74500..8ee5f75 100644 --- a/supabase/code/.env.example +++ b/supabase/code/.env.example @@ -26,10 +26,17 @@ POSTGRES_PORT=5432 ############ # Supavisor -- Database pooler ############ +# Port Supavisor listens on for transaction pooling connections POOLER_PROXY_PORT_TRANSACTION=6543 +# Maximum number of PostgreSQL connections Supavisor opens per pool POOLER_DEFAULT_POOL_SIZE=20 +# Maximum number of client connections Supavisor accepts per pool POOLER_MAX_CLIENT_CONN=100 +# Unique tenant identifier POOLER_TENANT_ID=your-tenant-id +# Pool size for internal metadata storage used by Supavisor +# This is separate from client connections and used only by Supavisor itself +POOLER_DB_POOL_SIZE=5 ############ @@ -106,14 +113,14 @@ FUNCTIONS_VERIFY_JWT=false ############ -# Logs - Configuration for Logflare +# Logs - Configuration for Analytics # Please refer to https://supabase.com/docs/reference/self-hosting-analytics/introduction ############ -LOGFLARE_LOGGER_BACKEND_API_KEY=your-super-secret-and-long-logflare-key - # Change vector.toml sinks to reflect this change -LOGFLARE_API_KEY=your-super-secret-and-long-logflare-key +# these cannot be the same value +LOGFLARE_PUBLIC_ACCESS_TOKEN=your-super-secret-and-long-logflare-key-public +LOGFLARE_PRIVATE_ACCESS_TOKEN=your-super-secret-and-long-logflare-key-private # Docker socket location - this value will differ depending on your OS DOCKER_SOCKET_LOCATION=/var/run/docker.sock diff --git a/supabase/code/dev/docker-compose.dev.yml b/supabase/code/dev/docker-compose.dev.yml index ca19a0a..f8b3ba7 100644 --- a/supabase/code/dev/docker-compose.dev.yml +++ b/supabase/code/dev/docker-compose.dev.yml @@ -4,7 +4,7 @@ services: studio: build: context: .. - dockerfile: studio/Dockerfile + dockerfile: apps/studio/Dockerfile target: dev ports: - 8082:8082 diff --git a/supabase/code/docker-compose.yml b/supabase/code/docker-compose.yml index a8d8584..9a4e48e 100644 --- a/supabase/code/docker-compose.yml +++ b/supabase/code/docker-compose.yml @@ -10,7 +10,7 @@ name: supabase services: studio: - image: supabase/studio:2025.05.19-sha-3487831 + image: supabase/studio:2025.06.30-sha-6f5982d restart: unless-stopped healthcheck: test: @@ -41,7 +41,7 @@ services: SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} AUTH_JWT_SECRET: ${JWT_SECRET} - LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} + LOGFLARE_PRIVATE_ACCESS_TOKEN: ${LOGFLARE_PRIVATE_ACCESS_TOKEN} LOGFLARE_URL: http://analytics:4000 NEXT_PUBLIC_ENABLE_LOGS: true # Comment to use Big Query backend for analytics @@ -75,7 +75,7 @@ services: /docker-entrypoint.sh kong docker-start' auth: - image: supabase/gotrue:v2.172.1 + image: supabase/gotrue:v2.176.1 restart: unless-stopped healthcheck: test: @@ -221,7 +221,7 @@ services: # To use S3 backed storage: docker compose -f docker-compose.yml -f docker-compose.s3.yml up storage: - image: supabase/storage-api:v1.22.17 + image: supabase/storage-api:v1.24.7 restart: unless-stopped volumes: - ./volumes/storage:/var/lib/storage:z @@ -279,7 +279,7 @@ services: IMGPROXY_ENABLE_WEBP_DETECTION: ${IMGPROXY_ENABLE_WEBP_DETECTION} meta: - image: supabase/postgres-meta:v0.89.0 + image: supabase/postgres-meta:v0.89.3 restart: unless-stopped depends_on: db: @@ -314,7 +314,7 @@ services: command: [ "start", "--main-service", "/home/deno/functions/main" ] analytics: - image: supabase/logflare:1.12.0 + image: supabase/logflare:1.14.2 restart: unless-stopped # Uncomment to use Big Query backend for analytics # volumes: @@ -339,7 +339,8 @@ services: DB_PORT: ${POSTGRES_PORT} DB_PASSWORD: ${POSTGRES_PASSWORD} DB_SCHEMA: _analytics - LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} + LOGFLARE_PUBLIC_ACCESS_TOKEN: ${LOGFLARE_PUBLIC_ACCESS_TOKEN} + LOGFLARE_PRIVATE_ACCESS_TOKEN: ${LOGFLARE_PRIVATE_ACCESS_TOKEN} LOGFLARE_SINGLE_TENANT: true LOGFLARE_SUPABASE_MODE: true LOGFLARE_MIN_CLUSTER_SIZE: 1 @@ -421,14 +422,14 @@ services: interval: 5s retries: 3 environment: - LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} + LOGFLARE_PUBLIC_ACCESS_TOKEN: ${LOGFLARE_PUBLIC_ACCESS_TOKEN} command: [ "--config", "/etc/vector/vector.yml" ] security_opt: - "label=disable" # Update the DATABASE_URL if you are using an external Postgres database supavisor: - image: supabase/supavisor:2.5.1 + image: supabase/supavisor:2.5.6 restart: unless-stopped volumes: - ./volumes/pooler/pooler.exs:/etc/pooler/pooler.exs:ro,z @@ -456,7 +457,7 @@ services: POSTGRES_PORT: ${POSTGRES_PORT} POSTGRES_DB: ${POSTGRES_DB} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - DATABASE_URL: ecto://supabase_admin:${POSTGRES_PASSWORD}@db:${POSTGRES_PORT}/_supabase + DATABASE_URL: ecto://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase CLUSTER_POSTGRES: true SECRET_KEY_BASE: ${SECRET_KEY_BASE} VAULT_ENC_KEY: ${VAULT_ENC_KEY} @@ -468,6 +469,7 @@ services: POOLER_DEFAULT_POOL_SIZE: ${POOLER_DEFAULT_POOL_SIZE} POOLER_MAX_CLIENT_CONN: ${POOLER_MAX_CLIENT_CONN} POOLER_POOL_MODE: transaction + DB_POOL_SIZE: ${POOLER_DB_POOL_SIZE} command: [ "/bin/sh", diff --git a/supabase/code/volumes/logs/vector.yml b/supabase/code/volumes/logs/vector.yml index cce46df..1c438a8 100644 --- a/supabase/code/volumes/logs/vector.yml +++ b/supabase/code/volumes/logs/vector.yml @@ -165,7 +165,9 @@ sinks: method: 'post' request: retry_max_duration_secs: 10 - uri: 'http://analytics:4000/api/logs?source_name=gotrue.logs.prod&api_key=${LOGFLARE_API_KEY?LOGFLARE_API_KEY is required}' + headers: + x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required} + uri: 'http://analytics:4000/api/logs?source_name=gotrue.logs.prod' logflare_realtime: type: 'http' inputs: @@ -175,7 +177,9 @@ sinks: method: 'post' request: retry_max_duration_secs: 10 - uri: 'http://analytics:4000/api/logs?source_name=realtime.logs.prod&api_key=${LOGFLARE_API_KEY?LOGFLARE_API_KEY is required}' + headers: + x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required} + uri: 'http://analytics:4000/api/logs?source_name=realtime.logs.prod' logflare_rest: type: 'http' inputs: @@ -185,7 +189,9 @@ sinks: method: 'post' request: retry_max_duration_secs: 10 - uri: 'http://analytics:4000/api/logs?source_name=postgREST.logs.prod&api_key=${LOGFLARE_API_KEY?LOGFLARE_API_KEY is required}' + headers: + x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required} + uri: 'http://analytics:4000/api/logs?source_name=postgREST.logs.prod' logflare_db: type: 'http' inputs: @@ -195,10 +201,12 @@ sinks: method: 'post' request: retry_max_duration_secs: 10 + headers: + x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required} # We must route the sink through kong because ingesting logs before logflare is fully initialised will # lead to broken queries from studio. This works by the assumption that containers are started in the # following order: vector > db > logflare > kong - uri: 'http://kong:8000/analytics/v1/api/logs?source_name=postgres.logs&api_key=${LOGFLARE_API_KEY?LOGFLARE_API_KEY is required}' + uri: 'http://kong:8000/analytics/v1/api/logs?source_name=postgres.logs' logflare_functions: type: 'http' inputs: @@ -208,7 +216,9 @@ sinks: method: 'post' request: retry_max_duration_secs: 10 - uri: 'http://analytics:4000/api/logs?source_name=deno-relay-logs&api_key=${LOGFLARE_API_KEY?LOGFLARE_API_KEY is required}' + headers: + x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required} + uri: 'http://analytics:4000/api/logs?source_name=deno-relay-logs' logflare_storage: type: 'http' inputs: @@ -218,7 +228,9 @@ sinks: method: 'post' request: retry_max_duration_secs: 10 - uri: 'http://analytics:4000/api/logs?source_name=storage.logs.prod.2&api_key=${LOGFLARE_API_KEY?LOGFLARE_API_KEY is required}' + headers: + x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required} + uri: 'http://analytics:4000/api/logs?source_name=storage.logs.prod.2' logflare_kong: type: 'http' inputs: @@ -229,4 +241,6 @@ sinks: method: 'post' request: retry_max_duration_secs: 10 - uri: 'http://analytics:4000/api/logs?source_name=cloudflare.logs.prod&api_key=${LOGFLARE_API_KEY?LOGFLARE_API_KEY is required}' + headers: + x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required} + uri: 'http://analytics:4000/api/logs?source_name=cloudflare.logs.prod' diff --git a/twenty/code/.env.example b/twenty/code/.env.example index 941e7df..d538814 100644 --- a/twenty/code/.env.example +++ b/twenty/code/.env.example @@ -7,7 +7,6 @@ TAG=latest REDIS_URL=redis://redis:6379 SERVER_URL=https://$(PRIMARY_DOMAIN) -SIGN_IN_PREFILLED=false # Use openssl rand -base64 32 for each secret APP_SECRET=replace_me_with_a_random_string diff --git a/twenty/code/docker-compose.yml b/twenty/code/docker-compose.yml index 2c0e188..55dfc6d 100644 --- a/twenty/code/docker-compose.yml +++ b/twenty/code/docker-compose.yml @@ -10,6 +10,8 @@ services: PG_DATABASE_URL: postgres://${PG_DATABASE_USER:-postgres}:${PG_DATABASE_PASSWORD:-postgres}@${PG_DATABASE_HOST:-db}:${PG_DATABASE_PORT:-5432}/default SERVER_URL: ${SERVER_URL} REDIS_URL: ${REDIS_URL:-redis://redis:6379} + DISABLE_DB_MIGRATIONS: ${DISABLE_DB_MIGRATIONS} + DISABLE_CRON_JOBS_REGISTRATION: ${DISABLE_CRON_JOBS_REGISTRATION} STORAGE_TYPE: ${STORAGE_TYPE} STORAGE_S3_REGION: ${STORAGE_S3_REGION} @@ -54,13 +56,14 @@ services: worker: image: twentycrm/twenty:${TAG:-latest} volumes: - - server-local-data:/app/packages/twenty-server/${STORAGE_LOCAL_PATH:-.local-storage} + - server-local-data:/app/packages/twenty-server/.local-storage command: [ "yarn", "worker:prod" ] environment: PG_DATABASE_URL: postgres://${PG_DATABASE_USER:-postgres}:${PG_DATABASE_PASSWORD:-postgres}@${PG_DATABASE_HOST:-db}:${PG_DATABASE_PORT:-5432}/default SERVER_URL: ${SERVER_URL} REDIS_URL: ${REDIS_URL:-redis://redis:6379} DISABLE_DB_MIGRATIONS: "true" # it already runs on the server + DISABLE_CRON_JOBS_REGISTRATION: "true" # it already runs on the server STORAGE_TYPE: ${STORAGE_TYPE} STORAGE_S3_REGION: ${STORAGE_S3_REGION} diff --git a/twenty/code/grafana/provisioning/datasources/clickhouse-datasource.yaml b/twenty/code/grafana/provisioning/datasources/clickhouse-datasource.yaml new file mode 100644 index 0000000..bb4279e --- /dev/null +++ b/twenty/code/grafana/provisioning/datasources/clickhouse-datasource.yaml @@ -0,0 +1,15 @@ +apiVersion: 1 + +datasources: + - name: ClickHouse + type: grafana-clickhouse-datasource + uid: clickhouse_datasource + jsonData: + server: twenty_clickhouse + defaultDatabase: twenty_dev + port: 9000 + protocol: native + tlsSkipVerify: true + username: default + secureJsonData: + password: devPassword \ No newline at end of file diff --git a/twenty/code/otel-collector/otel-collector-config.yaml b/twenty/code/otel-collector/otel-collector-config.yaml new file mode 100644 index 0000000..1d6eb9d --- /dev/null +++ b/twenty/code/otel-collector/otel-collector-config.yaml @@ -0,0 +1,24 @@ +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + +exporters: + clickhouse: + endpoint: tcp://twenty_clickhouse:9000 + database: twenty_dev + username: default + password: devPassword + debug: + verbosity: detailed + +processors: + batch: + +service: + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [clickhouse, debug] \ No newline at end of file diff --git a/twenty/code/scripts/install.sh b/twenty/code/scripts/install.sh index 320696d..2ae7bb4 100755 --- a/twenty/code/scripts/install.sh +++ b/twenty/code/scripts/install.sh @@ -94,7 +94,7 @@ echo "# === Randomly generated secret ===" >> .env echo "APP_SECRET=$(openssl rand -base64 32)" >> .env echo "" >> .env -echo "PG_DATABASE_PASSWORD=$(openssl rand -hex 16)" >> .env +echo "PG_DATABASE_PASSWORD=$(openssl rand -hex 32)" >> .env echo -e "\t• .env configuration completed" diff --git a/twenty/code/twenty-website/Dockerfile b/twenty/code/twenty-website/Dockerfile index 3f64a07..c760216 100644 --- a/twenty/code/twenty-website/Dockerfile +++ b/twenty/code/twenty-website/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18.17.1-alpine as twenty-website-build +FROM node:22-alpine as twenty-website-build WORKDIR /app @@ -23,7 +23,7 @@ COPY ./packages/twenty-ui /app/packages/twenty-ui COPY ./packages/twenty-website /app/packages/twenty-website RUN npx nx build twenty-website -FROM node:18.17.1-alpine as twenty-website +FROM node:22-alpine as twenty-website WORKDIR /app/packages/twenty-website diff --git a/twenty/code/twenty/Dockerfile b/twenty/code/twenty/Dockerfile index 4100694..d5f7197 100644 --- a/twenty/code/twenty/Dockerfile +++ b/twenty/code/twenty/Dockerfile @@ -1,5 +1,5 @@ # Base image for common dependencies -FROM node:18.17.1-alpine as common-deps +FROM node:22-alpine as common-deps WORKDIR /app @@ -49,7 +49,7 @@ RUN npx nx build twenty-front # Final stage: Run the application -FROM node:18.17.1-alpine as twenty +FROM node:22-alpine as twenty # Used to run healthcheck in docker RUN apk add --no-cache curl jq diff --git a/twenty/code/twenty/entrypoint.sh b/twenty/code/twenty/entrypoint.sh index a7de30a..9990491 100755 --- a/twenty/code/twenty/entrypoint.sh +++ b/twenty/code/twenty/entrypoint.sh @@ -12,7 +12,7 @@ setup_and_migrate_db() { PGPASS=$(echo $PG_DATABASE_URL | awk -F ':' '{print $3}' | awk -F '@' '{print $1}') PGHOST=$(echo $PG_DATABASE_URL | awk -F '@' '{print $2}' | awk -F ':' '{print $1}') PGPORT=$(echo $PG_DATABASE_URL | awk -F ':' '{print $4}' | awk -F '/' '{print $1}') - PGDATABASE=$(echo $PG_DATABASE_URL | awk -F ':' '{print $4}' | awk -F '/' '{print $2}') + PGDATABASE=$(echo $PG_DATABASE_URL | awk -F '/' '{print $NF}' | cut -d'?' -f1) # Creating the database if it doesn't exist db_count=$(PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d postgres -tAc "SELECT COUNT(*) FROM pg_database WHERE datname = '${PGDATABASE}'") @@ -22,12 +22,29 @@ setup_and_migrate_db() { # Run setup and migration scripts NODE_OPTIONS="--max-old-space-size=1500" tsx ./scripts/setup-db.ts + yarn database:migrate:prod fi yarn command:prod upgrade echo "Successfully migrated DB!" } + +register_background_jobs() { + if [ "${DISABLE_CRON_JOBS_REGISTRATION}" = "true" ]; then + echo "Cron job registration is disabled, skipping..." + return + fi + + echo "Registering background sync jobs..." + if yarn command:prod cron:register:all; then + echo "Successfully registered all background sync jobs!" + else + echo "Warning: Failed to register background jobs, but continuing startup..." + fi +} + setup_and_migrate_db +register_background_jobs # Continue with the original Docker command exec "$@"