update templates 30.08.24

appwrite/code/.env.example

@@ -16,6 +16,8 @@ _APP_SYSTEM_EMAIL_NAME=Appwrite
 _APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io
 _APP_SYSTEM_RESPONSE_FORMAT=
 _APP_SYSTEM_SECURITY_EMAIL_ADDRESS=certs@appwrite.io
+_APP_EMAIL_SECURITY=
+_APP_EMAIL_CERTIFICATES=
 _APP_USAGE_STATS=enabled
 _APP_LOGGING_PROVIDER=
 _APP_LOGGING_CONFIG=

appwrite/code/docker-compose.yml

@@ -28,7 +28,7 @@ services:
       - appwrite
 
   appwrite:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     <<: *x-logging
     restart: unless-stopped
     networks:
@@ -43,10 +43,10 @@ services:
       - traefik.http.routers.appwrite_api_http.rule=PathPrefix(`/`)
       - traefik.http.routers.appwrite_api_http.service=appwrite_api
       # https
-      # - traefik.http.routers.appwrite_api_https.entrypoints=appwrite_websecure
+      - traefik.http.routers.appwrite_api_https.entrypoints=appwrite_websecure
-      # - traefik.http.routers.appwrite_api_https.rule=PathPrefix(`/`)
+      - traefik.http.routers.appwrite_api_https.rule=PathPrefix(`/`)
-      # - traefik.http.routers.appwrite_api_https.service=appwrite_api
+      - traefik.http.routers.appwrite_api_https.service=appwrite_api
-      # - traefik.http.routers.appwrite_api_https.tls=true
+      - traefik.http.routers.appwrite_api_https.tls=true
     volumes:
       - appwrite-uploads:/storage/uploads:rw
       - appwrite-cache:/storage/cache:rw
@@ -67,7 +67,7 @@ services:
       - _APP_CONSOLE_HOSTNAMES
       - _APP_SYSTEM_EMAIL_NAME
       - _APP_SYSTEM_EMAIL_ADDRESS
-      - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
+      - _APP_EMAIL_SECURITY
       - _APP_SYSTEM_RESPONSE_FORMAT
       - _APP_OPTIONS_ABUSE
       - _APP_OPTIONS_ROUTER_PROTECTION
@@ -152,7 +152,7 @@ services:
       - _APP_ASSISTANT_OPENAI_API_KEY
 
   appwrite-realtime:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: realtime
     <<: *x-logging
     restart: unless-stopped
@@ -166,10 +166,10 @@ services:
       - traefik.http.routers.appwrite_realtime_ws.rule=PathPrefix(`/v1/realtime`)
       - traefik.http.routers.appwrite_realtime_ws.service=appwrite_realtime
       # wss
-      # - traefik.http.routers.appwrite_realtime_wss.entrypoints=appwrite_websecure
+      - traefik.http.routers.appwrite_realtime_wss.entrypoints=appwrite_websecure
-      # - traefik.http.routers.appwrite_realtime_wss.rule=PathPrefix(`/v1/realtime`)
+      - traefik.http.routers.appwrite_realtime_wss.rule=PathPrefix(`/v1/realtime`)
-      # - traefik.http.routers.appwrite_realtime_wss.service=appwrite_realtime
+      - traefik.http.routers.appwrite_realtime_wss.service=appwrite_realtime
-      # - traefik.http.routers.appwrite_realtime_wss.tls=true
+      - traefik.http.routers.appwrite_realtime_wss.tls=true
     networks:
       - appwrite
     depends_on:
@@ -195,7 +195,7 @@ services:
       - _APP_LOGGING_CONFIG
 
   appwrite-worker-audits:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-audits
     <<: *x-logging
     restart: unless-stopped
@@ -221,7 +221,7 @@ services:
       - _APP_LOGGING_CONFIG
 
   appwrite-worker-webhooks:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-webhooks
     <<: *x-logging
     restart: unless-stopped
@@ -234,6 +234,7 @@ services:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
+      - _APP_EMAIL_SECURITY
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
       - _APP_DB_HOST
       - _APP_DB_PORT
@@ -248,7 +249,7 @@ services:
       - _APP_LOGGING_CONFIG
 
   appwrite-worker-deletes:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-deletes
     <<: *x-logging
     restart: unless-stopped
@@ -303,7 +304,7 @@ services:
       - _APP_EXECUTOR_HOST
 
   appwrite-worker-databases:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-databases
     <<: *x-logging
     restart: unless-stopped
@@ -329,7 +330,7 @@ services:
       - _APP_LOGGING_CONFIG
 
   appwrite-worker-builds:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-builds
     <<: *x-logging
     restart: unless-stopped
@@ -392,7 +393,7 @@ services:
       - _APP_STORAGE_WASABI_BUCKET
 
   appwrite-worker-certificates:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-certificates
     <<: *x-logging
     restart: unless-stopped
@@ -411,7 +412,7 @@ services:
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
       - _APP_DOMAIN_FUNCTIONS
-      - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
+      - _APP_EMAIL_CERTIFICATES
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -425,7 +426,7 @@ services:
       - _APP_LOGGING_CONFIG
 
   appwrite-worker-functions:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-functions
     <<: *x-logging
     restart: unless-stopped
@@ -461,7 +462,7 @@ services:
       - _APP_LOGGING_PROVIDER
 
   appwrite-worker-mails:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-mails
     <<: *x-logging
     restart: unless-stopped
@@ -493,7 +494,7 @@ services:
       - _APP_LOGGING_CONFIG
 
   appwrite-worker-messaging:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-messaging
     <<: *x-logging
     restart: unless-stopped
@@ -543,7 +544,7 @@ services:
       - _APP_STORAGE_WASABI_BUCKET
 
   appwrite-worker-migrations:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-migrations
     <<: *x-logging
     restart: unless-stopped
@@ -557,7 +558,7 @@ services:
       - _APP_OPENSSL_KEY_V1
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
-      - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
+      - _APP_EMAIL_SECURITY
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -573,7 +574,7 @@ services:
       - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET
 
   appwrite-task-maintenance:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: maintenance
     <<: *x-logging
     restart: unless-stopped
@@ -606,7 +607,7 @@ services:
       - _APP_MAINTENANCE_RETENTION_SCHEDULES
 
   appwrite-worker-usage:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-usage
     <<: *x-logging
     restart: unless-stopped
@@ -634,7 +635,7 @@ services:
       - _APP_USAGE_AGGREGATION_INTERVAL
 
   appwrite-worker-usage-dump:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: worker-usage-dump
     <<: *x-logging
     networks:
@@ -661,7 +662,7 @@ services:
       - _APP_USAGE_AGGREGATION_INTERVAL
 
   appwrite-task-scheduler-functions:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: schedule-functions
     <<: *x-logging
     restart: unless-stopped
@@ -685,7 +686,7 @@ services:
       - _APP_DB_PASS
 
   appwrite-task-scheduler-messages:
-    image: appwrite/appwrite:1.5.7
+    image: appwrite/appwrite:1.5.10
     entrypoint: schedule-messages
     <<: *x-logging
     restart: unless-stopped
@@ -722,7 +723,7 @@ services:
     <<: *x-logging
     restart: unless-stopped
     stop_signal: SIGINT
-    image: openruntimes/executor:0.5.5
+    image: openruntimes/executor:0.5.7
     networks:
       - appwrite
       - runtimes
@@ -798,6 +799,7 @@ services:
 
   # clamav:
   #   image: appwrite/clamav:1.2.0
+  #   container_name: appwrite-clamav
   #   restart: unless-stopped
   #   networks:
   #     - appwrite

dify/code/.env.example

@@ -124,10 +124,36 @@ GUNICORN_TIMEOUT=360
 # The number of Celery workers. The default is 1, and can be set as needed.
 CELERY_WORKER_AMOUNT=
 
+# Flag indicating whether to enable autoscaling of Celery workers.
+#
+# Autoscaling is useful when tasks are CPU intensive and can be dynamically
+# allocated and deallocated based on the workload.
+#
+# When autoscaling is enabled, the maximum and minimum number of workers can
+# be specified. The autoscaling algorithm will dynamically adjust the number
+# of workers within the specified range.
+#
+# Default is false (i.e., autoscaling is disabled).
+#
+# Example:
+# CELERY_AUTO_SCALE=true
+CELERY_AUTO_SCALE=false
+
+# The maximum number of Celery workers that can be autoscaled.
+# This is optional and only used when autoscaling is enabled.
+# Default is not set.
+CELERY_MAX_WORKERS=
+
+# The minimum number of Celery workers that can be autoscaled.
+# This is optional and only used when autoscaling is enabled.
+# Default is not set.
+CELERY_MIN_WORKERS=
+
 # API Tool configuration
 API_TOOL_DEFAULT_CONNECT_TIMEOUT=10
 API_TOOL_DEFAULT_READ_TIMEOUT=60
 
+
 # ------------------------------
 # Database Configuration
 # The database uses PostgreSQL. Please use the public schema.
@@ -147,6 +173,36 @@ SQLALCHEMY_POOL_RECYCLE=3600
 # Whether to print SQL, default is false.
 SQLALCHEMY_ECHO=false
 
+# Maximum number of connections to the database
+# Default is 100
+#
+# Reference: https://www.postgresql.org/docs/current/runtime-config-connection.html#GUC-MAX-CONNECTIONS
+POSTGRES_MAX_CONNECTIONS=100
+
+# Sets the amount of shared memory used for postgres's shared buffers.
+# Default is 128MB
+# Recommended value: 25% of available memory
+# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-SHARED-BUFFERS
+POSTGRES_SHARED_BUFFERS=128MB
+
+# Sets the amount of memory used by each database worker for working space.
+# Default is 4MB
+#
+# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-WORK-MEM
+POSTGRES_WORK_MEM=4MB
+
+# Sets the amount of memory reserved for maintenance activities.
+# Default is 64MB
+#
+# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-MAINTENANCE-WORK-MEM
+POSTGRES_MAINTENANCE_WORK_MEM=64MB
+
+# Sets the planner's assumption about the effective cache size.
+# Default is 4096MB
+#
+# Reference: https://www.postgresql.org/docs/current/runtime-config-query.html#GUC-EFFECTIVE-CACHE-SIZE
+POSTGRES_EFFECTIVE_CACHE_SIZE=4096MB
+
 # ------------------------------
 # Redis Configuration
 # This Redis configuration is used for caching and for pub/sub during conversation.
@@ -247,7 +303,7 @@ TENCENT_COS_SCHEME=your-scheme
 # ------------------------------
 
 # The type of vector store to use. 
-# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `chroma`, `opensearch`, `tidb_vector`, `oracle`, `tencent`.
+# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `chroma`, `opensearch`, `tidb_vector`, `oracle`, `tencent`, `elasticsearch`.
 VECTOR_STORE=weaviate
 
 # The Weaviate endpoint URL. Only available when VECTOR_STORE is `weaviate`.
@@ -340,6 +396,12 @@ TENCENT_VECTOR_DB_DATABASE=dify
 TENCENT_VECTOR_DB_SHARD=1
 TENCENT_VECTOR_DB_REPLICAS=2
 
+# ElasticSearch configuration, only available when VECTOR_STORE is `elasticsearch`
+ELASTICSEARCH_HOST=0.0.0.0
+ELASTICSEARCH_PORT=9200
+ELASTICSEARCH_USERNAME=elastic
+ELASTICSEARCH_PASSWORD=elastic
+
 # ------------------------------
 # Knowledge Configuration
 # ------------------------------
@@ -452,6 +514,8 @@ RESET_PASSWORD_TOKEN_EXPIRY_HOURS=24
 CODE_EXECUTION_ENDPOINT=http://sandbox:8194
 CODE_MAX_NUMBER=9223372036854775807
 CODE_MIN_NUMBER=-9223372036854775808
+CODE_MAX_DEPTH=5
+CODE_MAX_PRECISION=20
 CODE_MAX_STRING_LENGTH=80000
 TEMPLATE_TRANSFORM_MAX_LENGTH=80000
 CODE_MAX_STRING_ARRAY_LENGTH=30
@@ -601,6 +665,23 @@ NGINX_KEEPALIVE_TIMEOUT=65
 NGINX_PROXY_READ_TIMEOUT=3600s
 NGINX_PROXY_SEND_TIMEOUT=3600s
 
+# Set true to accept requests for /.well-known/acme-challenge/
+NGINX_ENABLE_CERTBOT_CHALLENGE=false
+
+# ------------------------------
+# Certbot Configuration
+# ------------------------------
+
+# Email address (required to get certificates from Let's Encrypt)
+CERTBOT_EMAIL=your_email@example.com
+
+# Domain name
+CERTBOT_DOMAIN=your_domain.com
+
+# certbot command options
+# i.e: --force-renewal --dry-run --test-cert --debug
+CERTBOT_OPTIONS=
+
 # ------------------------------
 # Environment Variables for SSRF Proxy
 # ------------------------------
@@ -611,8 +692,9 @@ SSRF_SANDBOX_HOST=sandbox
 
 # ------------------------------
 # docker env var for specifying vector db type at startup
-# (based on the vector db type, the corresponding docker 
+# (based on the vector db type, the corresponding docker
 # compose profile will be used)
+# if you want to use unstructured, add ',unstructured' to the end
 # ------------------------------
 COMPOSE_PROFILES=${VECTOR_STORE:-weaviate}
 
@@ -621,3 +703,22 @@ COMPOSE_PROFILES=${VECTOR_STORE:-weaviate}
 # ------------------------------
 EXPOSE_NGINX_PORT=80
 EXPOSE_NGINX_SSL_PORT=443
+
+# ----------------------------------------------------------------------------
+# ModelProvider & Tool Position Configuration
+# Used to specify the model providers and tools that can be used in the app.
+# ----------------------------------------------------------------------------
+
+# Pin, include, and exclude tools
+# Use comma-separated values with no spaces between items.
+# Example: POSITION_TOOL_PINS=bing,google
+POSITION_TOOL_PINS=
+POSITION_TOOL_INCLUDES=
+POSITION_TOOL_EXCLUDES=
+
+# Pin, include, and exclude model providers
+# Use comma-separated values with no spaces between items.
+# Example: POSITION_PROVIDER_PINS=openai,openllm
+POSITION_PROVIDER_PINS=
+POSITION_PROVIDER_INCLUDES=
+POSITION_PROVIDER_EXCLUDES=

dify/code/README.md

@@ -3,42 +3,52 @@
 Welcome to the new `docker` directory for deploying Dify using Docker Compose. This README outlines the updates, deployment instructions, and migration details for existing users.
 
 ### What's Updated
+
+- **Certbot Container**: `docker-compose.yaml` now contains `certbot` for managing SSL certificates. This container automatically renews certificates and ensures secure HTTPS connections.  
+  For more information, refer `docker/certbot/README.md`.
+
 - **Persistent Environment Variables**: Environment variables are now managed through a `.env` file, ensuring that your configurations persist across deployments.
 
-   > What is `.env`? </br> </br>
+  > What is `.env`? </br> </br>
-   > The `.env` file is a crucial component in Docker and Docker Compose environments, serving as a centralized configuration file where you can define environment variables that are accessible to the containers at runtime. This file simplifies the management of environment settings across different stages of development, testing, and production, providing consistency and ease of configuration to deployments.
+  > The `.env` file is a crucial component in Docker and Docker Compose environments, serving as a centralized configuration file where you can define environment variables that are accessible to the containers at runtime. This file simplifies the management of environment settings across different stages of development, testing, and production, providing consistency and ease of configuration to deployments.
 
 - **Unified Vector Database Services**: All vector database services are now managed from a single Docker Compose file `docker-compose.yaml`. You can switch between different vector databases by setting the `VECTOR_STORE` environment variable in your `.env` file.
 - **Mandatory .env File**: A `.env` file is now required to run `docker compose up`. This file is crucial for configuring your deployment and for any custom settings to persist through upgrades.
 - **Legacy Support**: Previous deployment files are now located in the `docker-legacy` directory and will no longer be maintained.
 
 ### How to Deploy Dify with `docker-compose.yaml`
+
 1. **Prerequisites**: Ensure Docker and Docker Compose are installed on your system.
 2. **Environment Setup**:
-   - Navigate to the `docker` directory.
+    - Navigate to the `docker` directory.
-   - Copy the `.env.example` file to a new file named `.env` by running `cp .env.example .env`.
+    - Copy the `.env.example` file to a new file named `.env` by running `cp .env.example .env`.
-   - Customize the `.env` file as needed. Refer to the `.env.example` file for detailed configuration options.
+    - Customize the `.env` file as needed. Refer to the `.env.example` file for detailed configuration options.
 3. **Running the Services**:
-   - Execute `docker compose up` from the `docker` directory to start the services.
+    - Execute `docker compose up` from the `docker` directory to start the services.
-   - To specify a vector database, set the `VECTOR_store` variable in your `.env` file to your desired vector database service, such as `milvus`, `weaviate`, or `opensearch`.
+    - To specify a vector database, set the `VECTOR_STORE` variable in your `.env` file to your desired vector database service, such as `milvus`, `weaviate`, or `opensearch`.
+4. **SSL Certificate Setup**:
+    - Rrefer `docker/certbot/README.md` to set up SSL certificates using Certbot.
 
 ### How to Deploy Middleware for Developing Dify
+
 1. **Middleware Setup**:
-   - Use the `docker-compose.middleware.yaml` for setting up essential middleware services like databases and caches.
+    - Use the `docker-compose.middleware.yaml` for setting up essential middleware services like databases and caches.
-   - Navigate to the `docker` directory.
+    - Navigate to the `docker` directory.
-   - Ensure the `middleware.env` file is created by running `cp middleware.env.example middleware.env` (refer to the `middleware.env.example` file).
+    - Ensure the `middleware.env` file is created by running `cp middleware.env.example middleware.env` (refer to the `middleware.env.example` file).
 2. **Running Middleware Services**:
-   - Execute `docker-compose -f docker-compose.middleware.yaml up -d` to start the middleware services.
+    - Execute `docker-compose -f docker-compose.middleware.yaml up -d` to start the middleware services.
 
 ### Migration for Existing Users
+
 For users migrating from the `docker-legacy` setup:
+
 1. **Review Changes**: Familiarize yourself with the new `.env` configuration and Docker Compose setup.
 2. **Transfer Customizations**:
-   - If you have customized configurations such as `docker-compose.yaml`, `ssrf_proxy/squid.conf`, or `nginx/conf.d/default.conf`, you will need to reflect these changes in the `.env` file you create.
+    - If you have customized configurations such as `docker-compose.yaml`, `ssrf_proxy/squid.conf`, or `nginx/conf.d/default.conf`, you will need to reflect these changes in the `.env` file you create.
 3. **Data Migration**:
-   - Ensure that data from services like databases and caches is backed up and migrated appropriately to the new structure if necessary.
+    - Ensure that data from services like databases and caches is backed up and migrated appropriately to the new structure if necessary.
 
-### Overview of `.env` 
+### Overview of `.env`
 
 #### Key Modules and Customization
 
@@ -47,42 +57,43 @@ For users migrating from the `docker-legacy` setup:
 - **API and Web Services**: Users can define URLs and other settings that affect how the API and web frontends operate.
 
 #### Other notable variables
+
 The `.env.example` file provided in the Docker setup is extensive and covers a wide range of configuration options. It is structured into several sections, each pertaining to different aspects of the application and its services. Here are some of the key sections and variables:
 
 1. **Common Variables**:
-   - `CONSOLE_API_URL`, `SERVICE_API_URL`: URLs for different API services.
+    - `CONSOLE_API_URL`, `SERVICE_API_URL`: URLs for different API services.
-   - `APP_WEB_URL`: Frontend application URL.
+    - `APP_WEB_URL`: Frontend application URL.
-   - `FILES_URL`: Base URL for file downloads and previews.
+    - `FILES_URL`: Base URL for file downloads and previews.
 
 2. **Server Configuration**:
-   - `LOG_LEVEL`, `DEBUG`, `FLASK_DEBUG`: Logging and debug settings.
+    - `LOG_LEVEL`, `DEBUG`, `FLASK_DEBUG`: Logging and debug settings.
-   - `SECRET_KEY`: A key for encrypting session cookies and other sensitive data.
+    - `SECRET_KEY`: A key for encrypting session cookies and other sensitive data.
 
 3. **Database Configuration**:
-   - `DB_USERNAME`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`, `DB_DATABASE`: PostgreSQL database credentials and connection details.
+    - `DB_USERNAME`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`, `DB_DATABASE`: PostgreSQL database credentials and connection details.
 
 4. **Redis Configuration**:
-   - `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`: Redis server connection settings.
+    - `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`: Redis server connection settings.
 
 5. **Celery Configuration**:
-   - `CELERY_BROKER_URL`: Configuration for Celery message broker.
+    - `CELERY_BROKER_URL`: Configuration for Celery message broker.
 
 6. **Storage Configuration**:
-   - `STORAGE_TYPE`, `S3_BUCKET_NAME`, `AZURE_BLOB_ACCOUNT_NAME`: Settings for file storage options like local, S3, Azure Blob, etc.
+    - `STORAGE_TYPE`, `S3_BUCKET_NAME`, `AZURE_BLOB_ACCOUNT_NAME`: Settings for file storage options like local, S3, Azure Blob, etc.
 
 7. **Vector Database Configuration**:
-   - `VECTOR_STORE`: Type of vector database (e.g., `weaviate`, `milvus`).
+    - `VECTOR_STORE`: Type of vector database (e.g., `weaviate`, `milvus`).
-   - Specific settings for each vector store like `WEAVIATE_ENDPOINT`, `MILVUS_HOST`.
+    - Specific settings for each vector store like `WEAVIATE_ENDPOINT`, `MILVUS_HOST`.
 
 8. **CORS Configuration**:
-   - `WEB_API_CORS_ALLOW_ORIGINS`, `CONSOLE_CORS_ALLOW_ORIGINS`: Settings for cross-origin resource sharing.
+    - `WEB_API_CORS_ALLOW_ORIGINS`, `CONSOLE_CORS_ALLOW_ORIGINS`: Settings for cross-origin resource sharing.
 
 9. **Other Service-Specific Environment Variables**:
-   - Each service like `nginx`, `redis`, `db`, and vector databases have specific environment variables that are directly referenced in the `docker-compose.yaml`.
+    - Each service like `nginx`, `redis`, `db`, and vector databases have specific environment variables that are directly referenced in the `docker-compose.yaml`.
-
 
 ### Additional Information
+
 - **Continuous Improvement Phase**: We are actively seeking feedback from the community to refine and enhance the deployment process. As more users adopt this new method, we will continue to make improvements based on your experiences and suggestions.
 - **Support**: For detailed configuration options and environment variable settings, refer to the `.env.example` file and the Docker Compose configuration files in the `docker` directory.
 
-This README aims to guide you through the deployment process using the new Docker Compose setup. For any issues or further assistance, please refer to the official documentation or contact support.
+This README aims to guide you through the deployment process using the new Docker Compose setup. For any issues or further assistance, please refer to the official documentation or contact support.

dify/code/certbot/README.md

@@ -0,0 +1,76 @@
+# Launching new servers with SSL certificates
+
+## Short description
+
+Docker-compose certbot configurations with Backward compatibility (without certbot container).  
+Use `docker-compose --profile certbot up` to use this features.
+
+## The simplest way for launching new servers with SSL certificates
+
+1. Get letsencrypt certs  
+   set `.env` values
+   ```properties
+   NGINX_SSL_CERT_FILENAME=fullchain.pem
+   NGINX_SSL_CERT_KEY_FILENAME=privkey.pem
+   NGINX_ENABLE_CERTBOT_CHALLENGE=true
+   CERTBOT_DOMAIN=your_domain.com
+   CERTBOT_EMAIL=example@your_domain.com
+   ```
+   execute command:
+   ```shell
+   sudo docker network prune
+   sudo docker-compose --profile certbot up --force-recreate -d
+   ```
+   then after the containers launched:
+   ```shell
+   sudo docker-compose exec -it certbot /bin/sh /update-cert.sh
+   ```
+2. Edit `.env` file and `sudo docker-compose --profile certbot up` again.  
+   set `.env` value additionally
+   ```properties
+   NGINX_HTTPS_ENABLED=true
+   ```
+   execute command:
+   ```shell
+   sudo docker-compose --profile certbot up -d --no-deps --force-recreate nginx
+   ```
+   Then you can access your serve with HTTPS.  
+   [https://your_domain.com](https://your_domain.com)
+
+## SSL certificates renewal
+
+For SSL certificates renewal, execute commands below:
+
+```shell
+sudo docker-compose exec -it certbot /bin/sh /update-cert.sh
+sudo docker-compose exec nginx nginx -s reload
+```
+
+## Options for certbot
+
+`CERTBOT_OPTIONS` key might be helpful for testing. i.e.,
+
+```properties
+CERTBOT_OPTIONS=--dry-run
+```
+
+To apply changes to `CERTBOT_OPTIONS`, regenerate the certbot container before updating the certificates.
+
+```shell
+sudo docker-compose --profile certbot up -d --no-deps --force-recreate certbot
+sudo docker-compose exec -it certbot /bin/sh /update-cert.sh
+```
+
+Then, reload the nginx container if necessary.
+
+```shell
+sudo docker-compose exec nginx nginx -s reload
+```
+
+## For legacy servers
+
+To use cert files dir `nginx/ssl` as before, simply launch containers WITHOUT `--profile certbot` option.
+
+```shell
+sudo docker-compose up -d
+```

dify/code/certbot/docker-entrypoint.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+set -e
+
+printf '%s\n' "Docker entrypoint script is running"
+
+printf '%s\n' "\nChecking specific environment variables:"
+printf '%s\n' "CERTBOT_EMAIL: ${CERTBOT_EMAIL:-Not set}"
+printf '%s\n' "CERTBOT_DOMAIN: ${CERTBOT_DOMAIN:-Not set}"
+printf '%s\n' "CERTBOT_OPTIONS: ${CERTBOT_OPTIONS:-Not set}"
+
+printf '%s\n' "\nChecking mounted directories:"
+for dir in "/etc/letsencrypt" "/var/www/html" "/var/log/letsencrypt"; do
+    if [ -d "$dir" ]; then
+        printf '%s\n' "$dir exists. Contents:"
+        ls -la "$dir"
+    else
+        printf '%s\n' "$dir does not exist."
+    fi
+done
+
+printf '%s\n' "\nGenerating update-cert.sh from template"
+sed -e "s|\${CERTBOT_EMAIL}|$CERTBOT_EMAIL|g" \
+    -e "s|\${CERTBOT_DOMAIN}|$CERTBOT_DOMAIN|g" \
+    -e "s|\${CERTBOT_OPTIONS}|$CERTBOT_OPTIONS|g" \
+    /update-cert.template.txt > /update-cert.sh
+
+chmod +x /update-cert.sh
+
+printf '%s\n' "\nExecuting command:" "$@"
+exec "$@"

dify/code/certbot/update-cert.template.txt

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+DOMAIN="${CERTBOT_DOMAIN}"
+EMAIL="${CERTBOT_EMAIL}"
+OPTIONS="${CERTBOT_OPTIONS}"
+CERT_NAME="${DOMAIN}" # 証明書名をドメイン名と同じにする
+
+# Check if the certificate already exists
+if [ -f "/etc/letsencrypt/renewal/${CERT_NAME}.conf" ]; then
+  echo "Certificate exists. Attempting to renew..."
+  certbot renew --noninteractive --cert-name ${CERT_NAME} --webroot --webroot-path=/var/www/html --email ${EMAIL} --agree-tos --no-eff-email ${OPTIONS}
+else
+  echo "Certificate does not exist. Obtaining a new certificate..."
+  certbot certonly --noninteractive --webroot --webroot-path=/var/www/html --email ${EMAIL} --agree-tos --no-eff-email -d ${DOMAIN} ${OPTIONS}
+fi
+echo "Certificate operation successful"
+# Note: Nginx reload should be handled outside this container
+echo "Please ensure to reload Nginx to apply any certificate changes."

dify/code/docker-compose.middleware.yaml

@@ -9,6 +9,12 @@ services:
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-difyai123456}
       POSTGRES_DB: ${POSTGRES_DB:-dify}
       PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata}
+    command: >
+      postgres -c 'max_connections=${POSTGRES_MAX_CONNECTIONS:-100}'
+               -c 'shared_buffers=${POSTGRES_SHARED_BUFFERS:-128MB}'
+               -c 'work_mem=${POSTGRES_WORK_MEM:-4MB}'
+               -c 'maintenance_work_mem=${POSTGRES_MAINTENANCE_WORK_MEM:-64MB}'
+               -c 'effective_cache_size=${POSTGRES_EFFECTIVE_CACHE_SIZE:-4096MB}'
     volumes:
       - ./volumes/db/data:/var/lib/postgresql/data
     ports:
@@ -28,7 +34,7 @@ services:
 
   # The DifySandbox
   sandbox:
-    image: langgenius/dify-sandbox:0.2.1
+    image: langgenius/dify-sandbox:0.2.6
     restart: always
     environment:
       # The DifySandbox configurations

dify/code/docker-compose.yaml

@@ -1,5 +1,6 @@
 x-shared-env: &shared-api-worker-env
   LOG_LEVEL: ${LOG_LEVEL:-INFO}
+  LOG_FILE: ${LOG_FILE:-}
   DEBUG: ${DEBUG:-false}
   FLASK_DEBUG: ${FLASK_DEBUG:-false}
   SECRET_KEY: ${SECRET_KEY:-sk-9f73s3ljTXVcMT3Blb3ljTqtsKiGHXVcMT3BlbkFJLK7U}
@@ -22,6 +23,9 @@ x-shared-env: &shared-api-worker-env
   CELERY_WORKER_CLASS: ${CELERY_WORKER_CLASS:-}
   GUNICORN_TIMEOUT: ${GUNICORN_TIMEOUT:-360}
   CELERY_WORKER_AMOUNT: ${CELERY_WORKER_AMOUNT:-}
+  CELERY_AUTO_SCALE: ${CELERY_AUTO_SCALE:-false}
+  CELERY_MAX_WORKERS: ${CELERY_MAX_WORKERS:-}
+  CELERY_MIN_WORKERS: ${CELERY_MIN_WORKERS:-}
   API_TOOL_DEFAULT_CONNECT_TIMEOUT: ${API_TOOL_DEFAULT_CONNECT_TIMEOUT:-10}
   API_TOOL_DEFAULT_READ_TIMEOUT: ${API_TOOL_DEFAULT_READ_TIMEOUT:-60}
   DB_USERNAME: ${DB_USERNAME:-postgres}
@@ -117,6 +121,11 @@ x-shared-env: &shared-api-worker-env
   CHROMA_DATABASE: ${CHROMA_DATABASE:-default_database}
   CHROMA_AUTH_PROVIDER: ${CHROMA_AUTH_PROVIDER:-chromadb.auth.token_authn.TokenAuthClientProvider}
   CHROMA_AUTH_CREDENTIALS: ${CHROMA_AUTH_CREDENTIALS:-}
+  ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-0.0.0.0}
+  ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
+  ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_USERNAME:-elastic}
+  ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-elastic}
+  KIBANA_PORT: ${KIBANA_PORT:-5601}
   # AnalyticDB configuration
   ANALYTICDB_KEY_ID: ${ANALYTICDB_KEY_ID:-}
   ANALYTICDB_KEY_SECRET: ${ANALYTICDB_KEY_SECRET:-}
@@ -168,6 +177,8 @@ x-shared-env: &shared-api-worker-env
   CODE_EXECUTION_API_KEY: ${SANDBOX_API_KEY:-dify-sandbox}
   CODE_MAX_NUMBER: ${CODE_MAX_NUMBER:-9223372036854775807}
   CODE_MIN_NUMBER: ${CODE_MIN_NUMBER:--9223372036854775808}
+  CODE_MAX_DEPTH: ${CODE_MAX_DEPTH:-5}
+  CODE_MAX_PRECISION: ${CODE_MAX_PRECISION:-20}
   CODE_MAX_STRING_LENGTH: ${CODE_MAX_STRING_LENGTH:-80000}
   TEMPLATE_TRANSFORM_MAX_LENGTH: ${TEMPLATE_TRANSFORM_MAX_LENGTH:-80000}
   CODE_MAX_STRING_ARRAY_LENGTH: ${CODE_MAX_STRING_ARRAY_LENGTH:-30}
@@ -179,7 +190,7 @@ x-shared-env: &shared-api-worker-env
 services:
   # API service
   api:
-    image: langgenius/dify-api:0.7.1
+    image: langgenius/dify-api:0.7.2
     restart: always
     environment:
       # Use the shared environment variables.
@@ -199,7 +210,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:0.7.1
+    image: langgenius/dify-api:0.7.2
     restart: always
     environment:
       # Use the shared environment variables.
@@ -218,12 +229,13 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:0.7.1
+    image: langgenius/dify-web:0.7.2
     restart: always
     environment:
       CONSOLE_API_URL: ${CONSOLE_API_URL:-}
       APP_API_URL: ${APP_API_URL:-}
       SENTRY_DSN: ${WEB_SENTRY_DSN:-}
+      NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-0}
 
   # The postgres database.
   db:
@@ -234,6 +246,12 @@ services:
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-difyai123456}
       POSTGRES_DB: ${POSTGRES_DB:-dify}
       PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata}
+    command: >
+      postgres -c 'max_connections=${POSTGRES_MAX_CONNECTIONS:-100}'
+               -c 'shared_buffers=${POSTGRES_SHARED_BUFFERS:-128MB}'
+               -c 'work_mem=${POSTGRES_WORK_MEM:-4MB}'
+               -c 'maintenance_work_mem=${POSTGRES_MAINTENANCE_WORK_MEM:-64MB}'
+               -c 'effective_cache_size=${POSTGRES_EFFECTIVE_CACHE_SIZE:-4096MB}'
     volumes:
       - ./volumes/db/data:/var/lib/postgresql/data
     healthcheck:
@@ -276,7 +294,7 @@ services:
 
   # ssrf_proxy server
   # for more information, please refer to
-  # https://docs.dify.ai/getting-started/install-self-hosted/install-faq#id-16.-why-is-ssrf_proxy-needed
+  # https://docs.dify.ai/learn-more/faq/install-faq#id-18.-why-is-ssrf_proxy-needed
   ssrf_proxy:
     image: ubuntu/squid:latest
     restart: always
@@ -300,6 +318,26 @@ services:
       - ssrf_proxy_network
       - default
 
+  # Certbot service
+  # use `docker-compose --profile certbot up` to start the certbot service.
+  certbot:
+    image: certbot/certbot
+    profiles:
+      - certbot
+    volumes:
+      - ./volumes/certbot/conf:/etc/letsencrypt
+      - ./volumes/certbot/www:/var/www/html
+      - ./volumes/certbot/logs:/var/log/letsencrypt
+      - ./volumes/certbot/conf/live:/etc/letsencrypt/live
+      - ./certbot/update-cert.template.txt:/update-cert.template.txt
+      - ./certbot/docker-entrypoint.sh:/docker-entrypoint.sh
+    environment:
+      - CERTBOT_EMAIL=${CERTBOT_EMAIL}
+      - CERTBOT_DOMAIN=${CERTBOT_DOMAIN}
+      - CERTBOT_OPTIONS=${CERTBOT_OPTIONS:-}
+    entrypoint: ["/docker-entrypoint.sh"]
+    command: ["tail", "-f", "/dev/null"]
+
   # The nginx reverse proxy.
   # used for reverse proxying the API service and Web service.
   nginx:
@@ -311,7 +349,10 @@ services:
       - ./nginx/https.conf.template:/etc/nginx/https.conf.template
       - ./nginx/conf.d:/etc/nginx/conf.d
       - ./nginx/docker-entrypoint.sh:/docker-entrypoint-mount.sh
-      - ./nginx/ssl:/etc/ssl
+      - ./nginx/ssl:/etc/ssl # cert dir (legacy)
+      - ./volumes/certbot/conf/live:/etc/letsencrypt/live # cert dir (with certbot container)
+      - ./volumes/certbot/conf:/etc/letsencrypt
+      - ./volumes/certbot/www:/var/www/html
     entrypoint:
       [
         "sh",
@@ -333,6 +374,8 @@ services:
       NGINX_KEEPALIVE_TIMEOUT: ${NGINX_KEEPALIVE_TIMEOUT:-65}
       NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s}
       NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s}
+      NGINX_ENABLE_CERTBOT_CHALLENGE: ${NGINX_ENABLE_CERTBOT_CHALLENGE:-false}
+      CERTBOT_DOMAIN: ${CERTBOT_DOMAIN:-}
     depends_on:
       - api
       - web
@@ -397,7 +440,7 @@ services:
 
   # pgvecto-rs vector store
   pgvecto-rs:
-    image: tensorchord/pgvecto-rs:pg16-v0.2.0
+    image: tensorchord/pgvecto-rs:pg16-v0.3.0
     profiles:
       - pgvecto-rs
     restart: always
@@ -544,7 +587,7 @@ services:
 
   # MyScale vector database
   myscale:
-    image: myscale/myscaledb:1.6
+    image: myscale/myscaledb:1.6.4
     profiles:
       - myscale
     restart: always
@@ -554,6 +597,66 @@ services:
       - ./volumes/myscale/log:/var/log/clickhouse-server
       - ./volumes/myscale/config/users.d/custom_users_config.xml:/etc/clickhouse-server/users.d/custom_users_config.xml
 
+  # https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html
+  # https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-prod-prerequisites
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.14.3
+    profiles:
+      - elasticsearch
+    restart: always
+    volumes:
+      - dify_es01_data:/usr/share/elasticsearch/data
+    environment:
+      - ELASTIC_PASSWORD=${ELASTICSEARCH_PASSWORD:-elastic}
+      - cluster.name=dify-es-cluster
+      - node.name=dify-es0
+      - discovery.type=single-node
+      - xpack.license.self_generated.type=trial
+      - xpack.security.enabled=true
+      - xpack.security.enrollment.enabled=false
+      - xpack.security.http.ssl.enabled=false
+    healthcheck:
+      test:
+        ["CMD", "curl", "-s", "http://localhost:9200/_cluster/health?pretty"]
+      interval: 30s
+      timeout: 10s
+      retries: 50
+
+  # https://www.elastic.co/guide/en/kibana/current/docker.html
+  # https://www.elastic.co/guide/en/kibana/current/settings.html
+  kibana:
+    image: docker.elastic.co/kibana/kibana:8.14.3
+    profiles:
+      - elasticsearch
+    depends_on:
+      - elasticsearch
+    restart: always
+    environment:
+      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=d1a66dfd-c4d3-4a0a-8290-2abcb83ab3aa
+      - NO_PROXY=localhost,127.0.0.1,elasticsearch,kibana
+      - XPACK_SECURITY_ENABLED=true
+      - XPACK_SECURITY_ENROLLMENT_ENABLED=false
+      - XPACK_SECURITY_HTTP_SSL_ENABLED=false
+      - XPACK_FLEET_ISAIRGAPPED=true
+      - I18N_LOCALE=zh-CN
+      - SERVER_PORT=5601
+      - ELASTICSEARCH_HOSTS="http://elasticsearch:9200"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s http://localhost:5601 >/dev/null || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  # unstructured .
+  # (if used, you need to set ETL_TYPE to Unstructured in the api & worker service.)
+  unstructured:
+    image: downloads.unstructured.io/unstructured-io/unstructured-api:latest
+    profiles:
+      - unstructured
+    restart: always
+    volumes:
+      - ./volumes/unstructured:/app/data
+
 networks:
   # create a network between sandbox, api and ssrf_proxy, and can not access outside.
   ssrf_proxy_network:
@@ -567,3 +670,4 @@ networks:
 
 volumes:
   oradata:
+  dify_es01_data:

dify/code/middleware.env.example

@@ -9,6 +9,35 @@ POSTGRES_DB=dify
 # postgres data directory
 PGDATA=/var/lib/postgresql/data/pgdata
 
+# Maximum number of connections to the database
+# Default is 100
+#
+# Reference: https://www.postgresql.org/docs/current/runtime-config-connection.html#GUC-MAX-CONNECTIONS
+POSTGRES_MAX_CONNECTIONS=100
+
+# Sets the amount of shared memory used for postgres's shared buffers.
+# Default is 128MB
+# Recommended value: 25% of available memory
+# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-SHARED-BUFFERS
+POSTGRES_SHARED_BUFFERS=128MB
+
+# Sets the amount of memory used by each database worker for working space.
+# Default is 4MB
+#
+# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-WORK-MEM
+POSTGRES_WORK_MEM=4MB
+
+# Sets the amount of memory reserved for maintenance activities.
+# Default is 64MB
+#
+# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-MAINTENANCE-WORK-MEM
+POSTGRES_MAINTENANCE_WORK_MEM=64MB
+
+# Sets the planner's assumption about the effective cache size.
+# Default is 4096MB
+#
+# Reference: https://www.postgresql.org/docs/current/runtime-config-query.html#GUC-EFFECTIVE-CACHE-SIZE
+POSTGRES_EFFECTIVE_CACHE_SIZE=4096MB
 
 # ------------------------------
 # Environment Variables for sandbox Service

dify/code/nginx/conf.d/default.conf.template

@@ -29,6 +29,9 @@ server {
       include proxy.conf;
     }
 
+    # placeholder for acme challenge location
+    ${ACME_CHALLENGE_LOCATION}
+
     # placeholder for https config defined in https.conf.template
     ${HTTPS_CONFIG}
 }

dify/code/nginx/docker-entrypoint.sh

@@ -1,6 +1,19 @@
 #!/bin/bash
 
 if [ "${NGINX_HTTPS_ENABLED}" = "true" ]; then
+    # Check if the certificate and key files for the specified domain exist
+    if [ -n "${CERTBOT_DOMAIN}" ] && \
+       [ -f "/etc/letsencrypt/live/${CERTBOT_DOMAIN}/${NGINX_SSL_CERT_FILENAME}" ] && \
+       [ -f "/etc/letsencrypt/live/${CERTBOT_DOMAIN}/${NGINX_SSL_CERT_KEY_FILENAME}" ]; then
+        SSL_CERTIFICATE_PATH="/etc/letsencrypt/live/${CERTBOT_DOMAIN}/${NGINX_SSL_CERT_FILENAME}"
+        SSL_CERTIFICATE_KEY_PATH="/etc/letsencrypt/live/${CERTBOT_DOMAIN}/${NGINX_SSL_CERT_KEY_FILENAME}"
+    else
+        SSL_CERTIFICATE_PATH="/etc/ssl/${NGINX_SSL_CERT_FILENAME}"
+        SSL_CERTIFICATE_KEY_PATH="/etc/ssl/${NGINX_SSL_CERT_KEY_FILENAME}"
+    fi
+    export SSL_CERTIFICATE_PATH
+    export SSL_CERTIFICATE_KEY_PATH
+
     # set the HTTPS_CONFIG environment variable to the content of the https.conf.template
     HTTPS_CONFIG=$(envsubst < /etc/nginx/https.conf.template)
     export HTTPS_CONFIG
@@ -8,6 +21,13 @@ if [ "${NGINX_HTTPS_ENABLED}" = "true" ]; then
     envsubst '${HTTPS_CONFIG}' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf
 fi
 
+if [ "${NGINX_ENABLE_CERTBOT_CHALLENGE}" = "true" ]; then
+    ACME_CHALLENGE_LOCATION='location /.well-known/acme-challenge/ { root /var/www/html; }'
+else
+    ACME_CHALLENGE_LOCATION=''
+fi
+export ACME_CHALLENGE_LOCATION
+
 env_vars=$(printenv | cut -d= -f1 | sed 's/^/$/g' | paste -sd, -)
 
 envsubst "$env_vars" < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf

dify/code/nginx/https.conf.template

@@ -1,8 +1,8 @@
 # Please do not directly edit this file. Instead, modify the .env variables related to NGINX configuration.
 
 listen ${NGINX_SSL_PORT} ssl;
-ssl_certificate ./../ssl/${NGINX_SSL_CERT_FILENAME};
+ssl_certificate ${SSL_CERTIFICATE_PATH};
-ssl_certificate_key ./../ssl/${NGINX_SSL_CERT_KEY_FILENAME};
+ssl_certificate_key ${SSL_CERTIFICATE_KEY_PATH};
 ssl_protocols ${NGINX_SSL_PROTOCOLS};
 ssl_prefer_server_ciphers on;
 ssl_session_cache shared:SSL:10m;

plane/code/docker-compose.yml

@@ -138,7 +138,7 @@ services:
 
   plane-db:
     <<: *app-env
-    image: postgres:15.5-alpine
+    image: postgres:15.7-alpine
     pull_policy: if_not_present
     restart: unless-stopped
     command: postgres -c 'max_connections=1000'

plane/code/install.sh

@@ -9,11 +9,20 @@ export DOCKERHUB_USER=makeplane
 export PULL_POLICY=${PULL_POLICY:-if_not_present}
 
 CPU_ARCH=$(uname -m)
+OS_NAME=$(uname)
+UPPER_CPU_ARCH=$(tr '[:lower:]' '[:upper:]' <<< "$CPU_ARCH")
 
 mkdir -p $PLANE_INSTALL_DIR/archive
 DOCKER_FILE_PATH=$PLANE_INSTALL_DIR/docker-compose.yaml
 DOCKER_ENV_PATH=$PLANE_INSTALL_DIR/plane.env
 
+SED_PREFIX=()
+if [ "$OS_NAME" == "Darwin" ]; then
+  SED_PREFIX=("-i" "")
+else
+  SED_PREFIX=("-i")
+fi
+
 function print_header() {
 clear
 
@@ -51,12 +60,12 @@ function spinner() {
 }
 
 function initialize(){
-    printf "Please wait while we check the availability of Docker images for the selected release ($APP_RELEASE) with ${CPU_ARCH^^} support." >&2
+    printf "Please wait while we check the availability of Docker images for the selected release ($APP_RELEASE) with ${UPPER_CPU_ARCH} support." >&2
 
     if [ "$CUSTOM_BUILD" == "true" ]; then
         echo "" >&2
         echo "" >&2
-        echo "${CPU_ARCH^^} images are not available for selected release ($APP_RELEASE)." >&2
+        echo "${UPPER_CPU_ARCH} images are not available for selected release ($APP_RELEASE)." >&2
         echo "build"
         return 1
     fi
@@ -78,7 +87,7 @@ function initialize(){
     else
         echo "" >&2
         echo "" >&2
-        echo "${CPU_ARCH^^} images are not available for selected release ($APP_RELEASE)." >&2
+        echo "${UPPER_CPU_ARCH} images are not available for selected release ($APP_RELEASE)." >&2
         echo "" >&2
         echo "build"
         return 1
@@ -122,7 +131,7 @@ function updateEnvFile() {
             return
         else 
             # if key exists, update the value
-            sed -i "s/^$key=.*/$key=$value/g" "$file"
+            sed "${SED_PREFIX[@]}" "s/^$key=.*/$key=$value/g" "$file"
         fi
     else
         echo "File not found: $file"

supabase/code/docker-compose.yml

@@ -9,7 +9,7 @@ version: "3.8"
 
 services:
   studio:
-    image: supabase/studio:20240701-05dfbec
+    image: supabase/studio:20240729-ce42139
     restart: unless-stopped
     healthcheck:
       test:
@@ -17,7 +17,7 @@ services:
           "CMD",
           "node",
           "-e",
-          "require('http').get('http://localhost:3000/api/profile', (r) => {if (r.statusCode !== 200) throw new Error(r.statusCode)})"
+          "require('http').get('http://localhost:3000/api/profile', (r) => {if (r.statusCode !== 200) throw new Error(r.statusCode)})",
         ]
       timeout: 5s
       interval: 5s
@@ -71,7 +71,7 @@ services:
       - ./volumes/api/kong.yml:/home/kong/temp.yml:ro
 
   auth:
-    image: supabase/gotrue:v2.151.0
+    image: supabase/gotrue:v2.158.1
     depends_on:
       db:
         # Disable this if you are using an external Postgres database
@@ -86,7 +86,7 @@ services:
           "--no-verbose",
           "--tries=1",
           "--spider",
-          "http://localhost:9999/health"
+          "http://localhost:9999/health",
         ]
       timeout: 5s
       interval: 5s
@@ -139,9 +139,6 @@ services:
       # GOTRUE_HOOK_PASSWORD_VERIFICATION_ATTEMPT_ENABLED="true"
       # GOTRUE_HOOK_PASSWORD_VERIFICATION_ATTEMPT_URI="pg-functions://postgres/public/password_verification_attempt"
 
-
-
-
   rest:
     image: postgrest/postgrest:v12.2.0
     depends_on:
@@ -163,7 +160,7 @@ services:
 
   realtime:
     # This container name looks inconsistent but is correct because realtime constructs tenant id by parsing the subdomain
-    image: supabase/realtime:v2.29.15
+    image: supabase/realtime:v2.30.23
     depends_on:
       db:
         # Disable this if you are using an external Postgres database
@@ -181,7 +178,7 @@ services:
           "/dev/null",
           "-H",
           "Authorization: Bearer ${ANON_KEY}",
-          "http://localhost:4000/api/tenants/realtime-dev/health"
+          "http://localhost:4000/api/tenants/realtime-dev/health",
         ]
       timeout: 5s
       interval: 5s
@@ -194,7 +191,7 @@ services:
       DB_USER: supabase_admin
       DB_PASSWORD: ${POSTGRES_PASSWORD}
       DB_NAME: ${POSTGRES_DB}
-      DB_AFTER_CONNECT_QUERY: 'SET search_path TO _realtime'
+      DB_AFTER_CONNECT_QUERY: "SET search_path TO _realtime"
       DB_ENC_KEY: supabaserealtime
       API_JWT_SECRET: ${JWT_SECRET}
       SECRET_KEY_BASE: UpNVntn3cDxHJpq99YMc1T1AQgQpc8kfYTuRgBiYa15BLrx8etQoXz3gZv1/u2oq
@@ -206,7 +203,7 @@ services:
 
   # To use S3 backed storage: docker compose -f docker-compose.yml -f docker-compose.s3.yml up
   storage:
-    image: supabase/storage-api:v1.0.6
+    image: supabase/storage-api:v1.10.1
     depends_on:
       db:
         # Disable this if you are using an external Postgres database
@@ -223,7 +220,7 @@ services:
           "--no-verbose",
           "--tries=1",
           "--spider",
-          "http://localhost:5000/status"
+          "http://localhost:5000/status",
         ]
       timeout: 5s
       interval: 5s
@@ -250,7 +247,7 @@ services:
   imgproxy:
     image: darthsim/imgproxy:v3.8.0
     healthcheck:
-      test: [ "CMD", "imgproxy", "health" ]
+      test: ["CMD", "imgproxy", "health"]
       timeout: 5s
       interval: 5s
       retries: 3
@@ -280,7 +277,7 @@ services:
       PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD}
 
   functions:
-    image: supabase/edge-runtime:v1.55.0
+    image: supabase/edge-runtime:v1.56.1
     restart: unless-stopped
     depends_on:
       analytics:
@@ -303,7 +300,7 @@ services:
   analytics:
     image: supabase/logflare:1.4.0
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:4000/health" ]
+      test: ["CMD", "curl", "http://localhost:4000/health"]
       timeout: 5s
       interval: 5s
       retries: 10
@@ -341,7 +338,7 @@ services:
 
   # Comment out everything below this point if you are using an external Postgres database
   db:
-    image: supabase/postgres:15.1.1.61
+    image: supabase/postgres:15.1.1.78
     healthcheck:
       test: pg_isready -U postgres -h localhost
       interval: 5s
@@ -387,13 +384,12 @@ services:
     healthcheck:
       test:
         [
-
           "CMD",
           "wget",
           "--no-verbose",
           "--tries=1",
           "--spider",
-          "http://vector:9001/health"
+          "http://vector:9001/health",
         ]
       timeout: 5s
       interval: 5s
@@ -403,7 +399,7 @@ services:
       - ${DOCKER_SOCKET_LOCATION}:/var/run/docker.sock:ro
     environment:
       LOGFLARE_API_KEY: ${LOGFLARE_API_KEY}
-    command: [ "--config", "etc/vector/vector.yml" ]
+    command: ["--config", "etc/vector/vector.yml"]
 
 volumes:
   db-config:

twenty/code/k8s/manifests/deployment-db.yaml

@@ -37,8 +37,8 @@ spec:
         ports:
         - containerPort: 5432
           name: tcp
-          protocol: TCP    
+          protocol: TCP
-        resources: 
+        resources:
           requests:
             memory: "256Mi"
             cpu: "250m"

twenty/code/k8s/manifests/deployment-server.yaml

@@ -31,6 +31,8 @@ spec:
           value: 3000
         - name: SERVER_URL
           value: "https://crm.example.com:443"
+        - name: FRONT_BASE_URL
+          value: "https://crm.example.com:443"
         - name: PG_DATABASE_URL
           value: "postgres://twenty:twenty@twenty-db.twentycrm.svc.cluster.local/default"
         - name: ENABLE_DB_MIGRATIONS
@@ -39,6 +41,8 @@ spec:
           value: "true"
         - name: STORAGE_TYPE
           value: "local"
+        - name: "MESSAGE_QUEUE_TYPE"
+          value: "pg-boss"
         - name: ACCESS_TOKEN_SECRET
           valueFrom:
             secretKeyRef:
@@ -65,8 +69,8 @@ spec:
         ports:
         - containerPort: 3000
           name: http-tcp
-          protocol: TCP    
+          protocol: TCP
-        resources: 
+        resources:
           requests:
             memory: "256Mi"
             cpu: "250m"
@@ -76,6 +80,8 @@ spec:
         stdin: true
         tty: true
         volumeMounts:
+        - mountPath: /app/docker-data
+          name: twentycrm-server-data
         - mountPath: /app/.local-storage
           name: twentycrm-server-data
       dnsPolicy: ClusterFirst

twenty/code/k8s/manifests/deployment-worker.yaml

@@ -0,0 +1,78 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: twentycrm-worker
+  name: twentycrm-worker
+  namespace: twentycrm
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: twentycrm-worker
+  template:
+    metadata:
+      labels:
+        app: twentycrm-worker
+    spec:
+      volumes:
+      - name: twentycrm-worker-data
+        persistentVolumeClaim:
+          claimName: twentycrm-worker-pvc
+      containers:
+      - env:
+        - name: SERVER_URL
+          value: "https://crm.example.com:443"
+        - name: FRONT_BASE_URL
+          value: "https://crm.example.com:443"
+        - name: PG_DATABASE_URL
+          value: "postgres://twenty:twenty@twenty-db.twentycrm.svc.cluster.local/default"
+        - name: ENABLE_DB_MIGRATIONS
+          value: "false" # it already runs on the server
+        - name: STORAGE_TYPE
+          value: "local"
+        - name: "MESSAGE_QUEUE_TYPE"
+          value: "pg-boss"
+        - name: ACCESS_TOKEN_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: tokens
+              key: accessToken
+        - name: LOGIN_TOKEN_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: tokens
+              key: loginToken
+        - name: REFRESH_TOKEN_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: tokens
+              key: refreshToken
+        - name: FILE_TOKEN_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: tokens
+              key: fileToken
+      - image: twentycrm/twenty:latest
+        imagePullPolicy: Always
+        name: twentycrm
+        command:
+        - yarn
+        - worker:prod
+        resources:
+          requests:
+            memory: "256Mi"
+            cpu: "250m"
+          limits:
+            memory: "1024Mi"
+            cpu: "1000m"
+        stdin: true
+        tty: true
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always

twenty/code/k8s/manifests/ingress.yaml

@@ -20,5 +20,5 @@ spec:
         backend:
           service:
             name: twentycrm-server
-            port: 
+            port:
               name: http-tcp

twenty/code/k8s/terraform/.terraform-docs.yml

@@ -0,0 +1,48 @@
+formatter: "markdown table" # this is required
+
+version: ""
+
+header-from: main.tf
+
+recursive:
+  enabled: false
+  path: modules
+
+output:
+  file: "README.md"
+  mode: inject
+  template: |-
+    <!-- BEGIN_TF_DOCS -->
+    # TwentyCRM Terraform Docs
+
+    This file was generated by [terraform-docs](https://terraform-docs.io/), for more information on how to install, configure and use visit their website.
+
+    To update this `README.md` after changes to the Terraform code in this folder, run: `terraform-docs .`
+
+    To make configuration changes to how this doc is generated, see `./.terraform-docs.yml`
+    
+    {{ .Content }}
+    <!-- END_TF_DOCS -->
+
+output-values:
+  enabled: false
+  from: "outputs.tf"
+
+sort:
+  enabled: true
+  by: required
+
+settings:
+  anchor: true
+  color: true
+  default: true
+  description: true
+  escape: true
+  hide-empty: true
+  html: true
+  indent: 2
+  lockfile: true
+  read-comments: true
+  required: true
+  sensitive: true
+  type: true

twenty/code/k8s/terraform/README.md

@@ -0,0 +1,64 @@
+<!-- BEGIN_TF_DOCS -->
+# TwentyCRM Terraform Docs
+
+This file was generated by [terraform-docs](https://terraform-docs.io/), for more information on how to install, configure and use visit their website.
+
+To update this `README.md` after changes to the Terraform code in this folder, run: `terraform-docs .`
+
+To make configuration changes to how this doc is generated, see `./.terraform-docs.yml`
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.9.2 |
+| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.31.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | >= 2.31.0 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [kubernetes_deployment.twentycrm_db](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/deployment) | resource |
+| [kubernetes_deployment.twentycrm_server](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/deployment) | resource |
+| [kubernetes_deployment.twentycrm_worker](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/deployment) | resource |
+| [kubernetes_ingress.twentycrm](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/ingress) | resource |
+| [kubernetes_namespace.twentycrm](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource |
+| [kubernetes_persistent_volume.db](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume) | resource |
+| [kubernetes_persistent_volume.server](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume) | resource |
+| [kubernetes_persistent_volume_claim.db](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume_claim) | resource |
+| [kubernetes_persistent_volume_claim.server](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume_claim) | resource |
+| [kubernetes_secret.twentycrm_tokens](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource |
+| [kubernetes_service.twentycrm_db](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service) | resource |
+| [kubernetes_service.twentycrm_server](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_twentycrm_app_hostname"></a> [twentycrm\_app\_hostname](#input\_twentycrm\_app\_hostname) | The protocol, DNS fully qualified hostname, and port used to access TwentyCRM in your environment. Ex: https://crm.example.com:443 | `string` | n/a | yes |
+| <a name="input_twentycrm_pgdb_admin_password"></a> [twentycrm\_pgdb\_admin\_password](#input\_twentycrm\_pgdb\_admin\_password) | TwentyCRM password for postgres database. | `string` | n/a | yes |
+| <a name="input_twentycrm_token_accessToken"></a> [twentycrm\_token\_accessToken](#input\_twentycrm\_token\_accessToken) | TwentyCRM access Token | `string` | n/a | yes |
+| <a name="input_twentycrm_token_fileToken"></a> [twentycrm\_token\_fileToken](#input\_twentycrm\_token\_fileToken) | TwentyCRM file Token | `string` | n/a | yes |
+| <a name="input_twentycrm_token_loginToken"></a> [twentycrm\_token\_loginToken](#input\_twentycrm\_token\_loginToken) | TwentyCRM login Token | `string` | n/a | yes |
+| <a name="input_twentycrm_token_refreshToken"></a> [twentycrm\_token\_refreshToken](#input\_twentycrm\_token\_refreshToken) | TwentyCRM refresh Token | `string` | n/a | yes |
+| <a name="input_twentycrm_app_name"></a> [twentycrm\_app\_name](#input\_twentycrm\_app\_name) | A friendly name prefix to use for every component deployed. | `string` | `"twentycrm"` | no |
+| <a name="input_twentycrm_db_image"></a> [twentycrm\_db\_image](#input\_twentycrm\_db\_image) | TwentyCRM image for database deployment. This defaults to latest. | `string` | `"twentycrm/twenty-postgres:latest"` | no |
+| <a name="input_twentycrm_db_pv_capacity"></a> [twentycrm\_db\_pv\_capacity](#input\_twentycrm\_db\_pv\_capacity) | Storage capacity provisioned for database persistent volume. | `string` | `"10Gi"` | no |
+| <a name="input_twentycrm_db_pv_path"></a> [twentycrm\_db\_pv\_path](#input\_twentycrm\_db\_pv\_path) | Local path to use to store the physical volume if using local storage on nodes. | `string` | `""` | no |
+| <a name="input_twentycrm_db_pvc_requests"></a> [twentycrm\_db\_pvc\_requests](#input\_twentycrm\_db\_pvc\_requests) | Storage capacity reservation for database persistent volume claim. | `string` | `"10Gi"` | no |
+| <a name="input_twentycrm_db_replicas"></a> [twentycrm\_db\_replicas](#input\_twentycrm\_db\_replicas) | Number of replicas for the TwentyCRM database deployment. This defaults to 1. | `number` | `1` | no |
+| <a name="input_twentycrm_namespace"></a> [twentycrm\_namespace](#input\_twentycrm\_namespace) | Namespace for all TwentyCRM resources | `string` | `"twentycrm"` | no |
+| <a name="input_twentycrm_server_data_mount_path"></a> [twentycrm\_server\_data\_mount\_path](#input\_twentycrm\_server\_data\_mount\_path) | TwentyCRM mount path for servers application data. Defaults to '/app/docker-data'. | `string` | `"/app/docker-data"` | no |
+| <a name="input_twentycrm_server_image"></a> [twentycrm\_server\_image](#input\_twentycrm\_server\_image) | TwentyCRM server image for the server deployment. This defaults to latest. This value is also used for the workers image. | `string` | `"twentycrm/twenty:latest"` | no |
+| <a name="input_twentycrm_server_pv_capacity"></a> [twentycrm\_server\_pv\_capacity](#input\_twentycrm\_server\_pv\_capacity) | Storage capacity provisioned for server persistent volume. | `string` | `"10Gi"` | no |
+| <a name="input_twentycrm_server_pv_path"></a> [twentycrm\_server\_pv\_path](#input\_twentycrm\_server\_pv\_path) | Local path to use to store the physical volume if using local storage on nodes. | `string` | `""` | no |
+| <a name="input_twentycrm_server_pvc_requests"></a> [twentycrm\_server\_pvc\_requests](#input\_twentycrm\_server\_pvc\_requests) | Storage capacity reservation for server persistent volume claim. | `string` | `"10Gi"` | no |
+| <a name="input_twentycrm_server_replicas"></a> [twentycrm\_server\_replicas](#input\_twentycrm\_server\_replicas) | Number of replicas for the TwentyCRM server deployment. This defaults to 1. | `number` | `1` | no |
+| <a name="input_twentycrm_worker_replicas"></a> [twentycrm\_worker\_replicas](#input\_twentycrm\_worker\_replicas) | Number of replicas for the TwentyCRM worker deployment. This defaults to 1. | `number` | `1` | no |
+<!-- END_TF_DOCS -->

twenty/code/k8s/terraform/deployment-db.tf

@@ -1,17 +1,17 @@
 resource "kubernetes_deployment" "twentycrm_db" {
   metadata {
-    name      = "${local.twentycrm_app_name}-db"
+    name      = "${var.twentycrm_app_name}-db"
     namespace = kubernetes_namespace.twentycrm.metadata.0.name
     labels = {
-      app = "${local.twentycrm_app_name}-db"
+      app = "${var.twentycrm_app_name}-db"
     }
   }
 
   spec {
-    replicas = 1
+    replicas = var.twentycrm_db_replicas
     selector {
       match_labels = {
-        app = "${local.twentycrm_app_name}-db"
+        app = "${var.twentycrm_app_name}-db"
       }
     }
 
@@ -26,17 +26,14 @@ resource "kubernetes_deployment" "twentycrm_db" {
     template {
       metadata {
         labels = {
-          app = "${local.twentycrm_app_name}-db"
+          app = "${var.twentycrm_app_name}-db"
         }
       }
 
       spec {
-        # security_context {
-        #   fs_group = 0
-        # }
         container {
-          image = local.twentycrm_db_image
+          image = var.twentycrm_db_image
-          name  = local.twentycrm_app_name
+          name  = var.twentycrm_app_name
           stdin = true
           tty   = true
           security_context {
@@ -45,7 +42,7 @@ resource "kubernetes_deployment" "twentycrm_db" {
 
           env {
             name  = "POSTGRES_PASSWORD"
-            value = "twenty"
+            value = var.twentycrm_pgdb_admin_password
           }
           env {
             name  = "BITNAMI_DEBUG"
@@ -69,16 +66,16 @@ resource "kubernetes_deployment" "twentycrm_db" {
           }
 
           volume_mount {
-            name       = "nfs-twentycrm-db-data"
+            name       = "db-data"
             mount_path = "/bitnami/postgresql"
           }
         }
 
         volume {
-          name = "nfs-twentycrm-db-data"
+          name = "db-data"
 
           persistent_volume_claim {
-            claim_name = "nfs-twentycrm-db-data-pvc"
+            claim_name = kubernetes_persistent_volume_claim.db.metadata.0.name
           }
         }
 

twenty/code/k8s/terraform/deployment-server.tf

@@ -1,17 +1,17 @@
 resource "kubernetes_deployment" "twentycrm_server" {
   metadata {
-    name      = "${local.twentycrm_app_name}-server"
+    name      = "${var.twentycrm_app_name}-server"
     namespace = kubernetes_namespace.twentycrm.metadata.0.name
     labels = {
-      app = "${local.twentycrm_app_name}-server"
+      app = "${var.twentycrm_app_name}-server"
     }
   }
 
   spec {
-    replicas = 1
+    replicas = var.twentycrm_server_replicas
     selector {
       match_labels = {
-        app = "${local.twentycrm_app_name}-server"
+        app = "${var.twentycrm_app_name}-server"
       }
     }
 
@@ -26,14 +26,14 @@ resource "kubernetes_deployment" "twentycrm_server" {
     template {
       metadata {
         labels = {
-          app = "${local.twentycrm_app_name}-server"
+          app = "${var.twentycrm_app_name}-server"
         }
       }
 
       spec {
         container {
-          image = local.twentycrm_server_image
+          image = var.twentycrm_server_image
-          name  = local.twentycrm_app_name
+          name  = var.twentycrm_app_name
           stdin = true
           tty   = true
 
@@ -54,22 +54,17 @@ resource "kubernetes_deployment" "twentycrm_server" {
 
           env {
             name  = "SERVER_URL"
-            value = "https://crm.example.com:443"
+            value = var.twentycrm_app_hostname
           }
 
           env {
             name  = "FRONT_BASE_URL"
-            value = "https://crm.example.com:443"
+            value = var.twentycrm_app_hostname
-          }
-
-          env {
-            name  = "BACKEND_SERVER_URL"
-            value = "https://crm.example.com:443"
           }
 
           env {
             name  = "PG_DATABASE_URL"
-            value = "postgres://twenty:twenty@twentycrm-db.twentycrm.svc.cluster.local/default"
+            value = "postgres://twenty:${var.twentycrm_pgdb_admin_password}@${var.twentycrm_app_name}-db.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local/default"
           }
 
           env {
@@ -86,7 +81,10 @@ resource "kubernetes_deployment" "twentycrm_server" {
             name  = "STORAGE_TYPE"
             value = "local"
           }
-
+          env {
+            name  = "MESSAGE_QUEUE_TYPE"
+            value = "pg-boss"
+          }
           env {
             name = "ACCESS_TOKEN_SECRET"
             value_from {
@@ -144,16 +142,16 @@ resource "kubernetes_deployment" "twentycrm_server" {
           }
 
           volume_mount {
-            name       = "nfs-twentycrm-server-data"
+            name       = "server-data"
-            mount_path = "/app/.local-storage"
+            mount_path = var.twentycrm_server_data_mount_path
           }
         }
 
         volume {
-          name = "nfs-twentycrm-server-data"
+          name = "server-data"
 
           persistent_volume_claim {
-            claim_name = "nfs-twentycrm-server-data-pvc"
+            claim_name = kubernetes_persistent_volume_claim.server.metadata.0.name
           }
         }
 

twenty/code/k8s/terraform/deployment-worker.tf

@@ -0,0 +1,131 @@
+resource "kubernetes_deployment" "twentycrm_worker" {
+  metadata {
+    name      = "${var.twentycrm_app_name}-worker"
+    namespace = kubernetes_namespace.twentycrm.metadata.0.name
+    labels = {
+      app = "${var.twentycrm_app_name}-worker"
+    }
+  }
+
+  spec {
+    replicas = var.twentycrm_worker_replicas
+    selector {
+      match_labels = {
+        app = "${var.twentycrm_app_name}-worker"
+      }
+    }
+
+    strategy {
+      type = "RollingUpdate"
+      rolling_update {
+        max_surge       = "1"
+        max_unavailable = "1"
+      }
+    }
+
+    template {
+      metadata {
+        labels = {
+          app = "${var.twentycrm_app_name}-worker"
+        }
+      }
+
+      spec {
+        container {
+          image   = var.twentycrm_server_image
+          name    = var.twentycrm_app_name
+          stdin   = true
+          tty     = true
+          command = ["yarn", "worker:prod"]
+
+          env {
+            name  = "SERVER_URL"
+            value = var.twentycrm_app_hostname
+          }
+
+          env {
+            name  = "FRONT_BASE_URL"
+            value = var.twentycrm_app_hostname
+          }
+
+          env {
+            name  = "PG_DATABASE_URL"
+            value = "postgres://twenty:${var.twentycrm_pgdb_admin_password}@${var.twentycrm_app_name}-db.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local/default"
+          }
+
+          env {
+            name  = "ENABLE_DB_MIGRATIONS"
+            value = "false" #it already runs on the server
+          }
+
+          env {
+            name  = "STORAGE_TYPE"
+            value = "local"
+          }
+          env {
+            name  = "MESSAGE_QUEUE_TYPE"
+            value = "pg-boss"
+          }
+
+          env {
+            name = "ACCESS_TOKEN_SECRET"
+            value_from {
+              secret_key_ref {
+                name = "tokens"
+                key  = "accessToken"
+              }
+            }
+          }
+
+          env {
+            name = "LOGIN_TOKEN_SECRET"
+            value_from {
+              secret_key_ref {
+                name = "tokens"
+                key  = "loginToken"
+              }
+            }
+          }
+
+          env {
+            name = "REFRESH_TOKEN_SECRET"
+            value_from {
+              secret_key_ref {
+                name = "tokens"
+                key  = "refreshToken"
+              }
+            }
+          }
+
+          env {
+            name = "FILE_TOKEN_SECRET"
+            value_from {
+              secret_key_ref {
+                name = "tokens"
+                key  = "fileToken"
+              }
+            }
+          }
+
+          resources {
+            requests = {
+              cpu    = "250m"
+              memory = "256Mi"
+            }
+            limits = {
+              cpu    = "1000m"
+              memory = "1024Mi"
+            }
+          }
+        }
+
+        dns_policy     = "ClusterFirst"
+        restart_policy = "Always"
+      }
+    }
+  }
+  depends_on = [
+    kubernetes_deployment.twentycrm_db,
+    kubernetes_secret.twentycrm_tokens
+  ]
+}

twenty/code/k8s/terraform/ingress.tf

@@ -1,7 +1,7 @@
 resource "kubernetes_ingress" "twentycrm" {
   wait_for_load_balancer = true
   metadata {
-    name      = "${local.twentycrm_app_name}-ingress"
+    name      = "${var.twentycrm_app_name}-ingress"
     namespace = kubernetes_namespace.twentycrm.metadata.0.name
     annotations = {
       "kubernetes.io/ingress.class"                       = "nginx"
@@ -15,7 +15,7 @@ resource "kubernetes_ingress" "twentycrm" {
   spec {
     ingress_class_name = "nginx"
     rule {
-      host = local.twentycrm_app_hostname
+      host = var.twentycrm_app_hostname
       http {
         path {
           path = "/*"

twenty/code/k8s/terraform/main.tf

@@ -5,32 +5,15 @@ provider "kubernetes" {
   config_path = "~/.kube/config"
 }
 
-#################
-# Global Locals #
-#################
-locals {
-  twentycrm_app_name            = "twentycrm"
-  twentycrm_app_hostname        = "crm.example.com"
-  twentycrm_server_image        = "twentycrm/twenty:v0.10.4"
-  twentycrm_db_image            = "twentycrm/twenty-postgres:v0.10.4"
-  twentycrm_db_pv_path          = "/path/to/mystorage"
-  twentycrm_db_pv_capacity      = "10Gi"
-  twentycrm_db_pvc_requests     = "10Gi"
-  twentycrm_server_pv_path      = "/path/to/mystorage"
-  twentycrm_server_pv_capacity  = "10Gi"
-  twentycrm_server_pvc_requests = "10Gi"
-}
-
 ####################
 # Terraform Config #
 ####################
 terraform {
-  required_version = ">= 1.7.4"
+  required_version = ">= 1.9.2"
   required_providers {
     kubernetes = {
       source  = "hashicorp/kubernetes"
-      version = ">= 2.23.0"
+      version = ">= 2.31.0"
     }
   }
-
 }

twenty/code/k8s/terraform/namespace.tf

@@ -1,9 +1,9 @@
 resource "kubernetes_namespace" "twentycrm" {
   metadata {
     annotations = {
-      name = "twentycrm"
+      name = var.twentycrm_namespace
     }
 
-    name = "twentycrm"
+    name = var.twentycrm_namespace
   }
 }

twenty/code/k8s/terraform/pv-db.tf

@@ -1,18 +1,18 @@
 resource "kubernetes_persistent_volume" "db" {
   metadata {
-    name = "${local.twentycrm_app_name}-db-pv"
+    name = "${var.twentycrm_app_name}-db-pv"
   }
   spec {
     storage_class_name = "default"
     capacity = {
-      storage = local.twentycrm_db_pv_capacity
+      storage = var.twentycrm_db_pv_capacity
     }
     access_modes = ["ReadWriteOnce"]
-    # refer to Terraform Docs for your specific implementation requirements 
+    # refer to Terraform Docs for your specific implementation requirements
     # https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume
     persistent_volume_source {
       local {
-        path = local.twentycrm_db_pv_path
+        path = var.twentycrm_db_pv_path
       }
     }
   }

twenty/code/k8s/terraform/pv-server.tf

@@ -1,18 +1,18 @@
 resource "kubernetes_persistent_volume" "server" {
   metadata {
-    name = "${local.twentycrm_app_name}-server-pv"
+    name = "${var.twentycrm_app_name}-server-pv"
   }
   spec {
     storage_class_name = "default"
     capacity = {
-      storage = local.twentycrm_server_pv_capacity
+      storage = var.twentycrm_server_pv_capacity
     }
     access_modes = ["ReadWriteOnce"]
-    # refer to Terraform Docs for your specific implementation requirements 
+    # refer to Terraform Docs for your specific implementation requirements
     # https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume
     persistent_volume_source {
       local {
-        path = local.twentycrm_server_pv_path
+        path = var.twentycrm_server_pv_path
       }
     }
   }

twenty/code/k8s/terraform/pvc-db.tf

@@ -1,13 +1,13 @@
 resource "kubernetes_persistent_volume_claim" "db" {
   metadata {
-    name      = "${local.twentycrm_app_name}-db-pvc"
+    name      = "${var.twentycrm_app_name}-db-pvc"
     namespace = kubernetes_namespace.twentycrm.metadata.0.name
   }
   spec {
     access_modes = ["ReadWriteOnce"]
     resources {
       requests = {
-        storage = local.twentycrm_db_pvc_requests
+        storage = var.twentycrm_db_pvc_requests
       }
     }
     volume_name = kubernetes_persistent_volume.db.metadata.0.name

twenty/code/k8s/terraform/pvc-server.tf

@@ -1,13 +1,13 @@
 resource "kubernetes_persistent_volume_claim" "server" {
   metadata {
-    name      = "${local.twentycrm_app_name}-server-pvc"
+    name      = "${var.twentycrm_app_name}-server-pvc"
     namespace = kubernetes_namespace.twentycrm.metadata.0.name
   }
   spec {
     access_modes = ["ReadWriteOnce"]
     resources {
       requests = {
-        storage = local.twentycrm_server_pvc_requests
+        storage = var.twentycrm_server_pvc_requests
       }
     }
     volume_name = kubernetes_persistent_volume.server.metadata.0.name

twenty/code/k8s/terraform/service-db.tf

@@ -1,11 +1,11 @@
 resource "kubernetes_service" "twentycrm_db" {
   metadata {
-    name      = "${local.twentycrm_app_name}-db"
+    name      = "${var.twentycrm_app_name}-db"
     namespace = kubernetes_namespace.twentycrm.metadata.0.name
   }
   spec {
     selector = {
-      app = "${local.twentycrm_app_name}-db"
+      app = "${var.twentycrm_app_name}-db"
     }
     session_affinity = "ClientIP"
     port {

twenty/code/k8s/terraform/service-server.tf

@@ -1,11 +1,11 @@
 resource "kubernetes_service" "twentycrm_server" {
   metadata {
-    name      = "${local.twentycrm_app_name}-server"
+    name      = "${var.twentycrm_app_name}-server"
     namespace = kubernetes_namespace.twentycrm.metadata.0.name
   }
   spec {
     selector = {
-      app = "${local.twentycrm_app_name}-server"
+      app = "${var.twentycrm_app_name}-server"
     }
     session_affinity = "ClientIP"
     port {

twenty/code/k8s/terraform/variables.tf

@@ -1,24 +1,124 @@
+######################
+# Required Variables #
+######################
 variable "twentycrm_token_accessToken" {
   type        = string
   description = "TwentyCRM access Token"
+  sensitive   = true
 }
 
 variable "twentycrm_token_loginToken" {
   type        = string
   description = "TwentyCRM login Token"
+  sensitive   = true
 }
 
 variable "twentycrm_token_refreshToken" {
   type        = string
   description = "TwentyCRM refresh Token"
+  sensitive   = true
 }
 
 variable "twentycrm_token_fileToken" {
   type        = string
   description = "TwentyCRM file Token"
+  sensitive   = true
 }
 
 variable "twentycrm_pgdb_admin_password" {
   type        = string
-  description = "TwentyCRM password for postgres database"
+  description = "TwentyCRM password for postgres database."
+  sensitive   = true
+}
+
+variable "twentycrm_app_hostname" {
+  type        = string
+  description = "The protocol, DNS fully qualified hostname, and port used to access TwentyCRM in your environment. Ex: https://crm.example.com:443"
+}
+
+######################
+# Optional Variables #
+######################
+variable "twentycrm_app_name" {
+  type        = string
+  default     = "twentycrm"
+  description = "A friendly name prefix to use for every component deployed."
+}
+
+variable "twentycrm_server_image" {
+  type        = string
+  default     = "twentycrm/twenty:latest"
+  description = "TwentyCRM server image for the server deployment. This defaults to latest. This value is also used for the workers image."
+}
+
+variable "twentycrm_db_image" {
+  type        = string
+  default     = "twentycrm/twenty-postgres:latest"
+  description = "TwentyCRM image for database deployment. This defaults to latest."
+}
+
+variable "twentycrm_server_replicas" {
+  type        = number
+  default     = 1
+  description = "Number of replicas for the TwentyCRM server deployment. This defaults to 1."
+}
+
+variable "twentycrm_worker_replicas" {
+  type        = number
+  default     = 1
+  description = "Number of replicas for the TwentyCRM worker deployment. This defaults to 1."
+}
+
+variable "twentycrm_db_replicas" {
+  type        = number
+  default     = 1
+  description = "Number of replicas for the TwentyCRM database deployment. This defaults to 1."
+}
+
+variable "twentycrm_server_data_mount_path" {
+  type        = string
+  default     = "/app/docker-data"
+  description = "TwentyCRM mount path for servers application data. Defaults to '/app/docker-data'."
+}
+
+variable "twentycrm_db_pv_path" {
+  type        = string
+  default     = ""
+  description = "Local path to use to store the physical volume if using local storage on nodes."
+}
+
+variable "twentycrm_server_pv_path" {
+  type        = string
+  default     = ""
+  description = "Local path to use to store the physical volume if using local storage on nodes."
+}
+
+variable "twentycrm_db_pv_capacity" {
+  type        = string
+  default     = "10Gi"
+  description = "Storage capacity provisioned for database persistent volume."
+}
+
+variable "twentycrm_db_pvc_requests" {
+  type        = string
+  default     = "10Gi"
+  description = "Storage capacity reservation for database persistent volume claim."
+}
+
+variable "twentycrm_server_pv_capacity" {
+  type        = string
+  default     = "10Gi"
+  description = "Storage capacity provisioned for server persistent volume."
+}
+
+variable "twentycrm_server_pvc_requests" {
+  type        = string
+  default     = "10Gi"
+  description = "Storage capacity reservation for server persistent volume claim."
+}
+
+variable "twentycrm_namespace" {
+  type        = string
+  default     = "twentycrm"
+  description = "Namespace for all TwentyCRM resources"
 }