From aa05d8edecbb8595fbd4a2a69b5e60aaf85e3a20 Mon Sep 17 00:00:00 2001 From: Andrei Canta Date: Wed, 4 Dec 2024 15:21:36 +0200 Subject: [PATCH] update all --- appwrite/code/docker-compose.yml | 38 +-- dify/code/.env.example | 101 ++++++- dify/code/README.md | 2 +- dify/code/couchbase-server/Dockerfile | 4 + dify/code/couchbase-server/init-cbserver.sh | 44 +++ dify/code/docker-compose.middleware.yaml | 13 +- dify/code/docker-compose.yaml | 251 ++++++++++++++---- dify/code/middleware.env.example | 10 +- .../volumes/oceanbase/init.d/vec_memory.sql | 1 + dify/code/volumes/sandbox/conf/config.yaml | 14 + .../volumes/sandbox/conf/config.yaml.example | 35 +++ plane/code/.env.example | 23 +- plane/code/docker-compose.yml | 72 ++--- plane/code/install.sh | 4 +- supabase/code/.env.example | 12 + supabase/code/docker-compose.s3.yml | 4 +- supabase/code/docker-compose.yml | 99 ++++--- supabase/code/volumes/db/_supabase.sql | 3 + supabase/code/volumes/db/logs.sql | 2 + supabase/code/volumes/db/pooler.sql | 6 + supabase/code/volumes/pooler/pooler.exs | 30 +++ twenty/code/.env.example | 24 +- twenty/code/Makefile | 36 --- twenty/code/docker-compose.yml | 46 ++-- twenty/code/k8s/manifests/deployment-db.yaml | 54 ++-- .../code/k8s/manifests/deployment-redis.yaml | 44 +++ .../code/k8s/manifests/deployment-server.yaml | 116 ++++---- .../code/k8s/manifests/deployment-worker.yaml | 89 +++---- twenty/code/k8s/manifests/ingress.yaml | 28 +- twenty/code/k8s/manifests/pv-docker-data.yaml | 11 + .../code/k8s/manifests/pvc-docker-data.yaml | 13 + twenty/code/k8s/manifests/service-db.yaml | 6 +- twenty/code/k8s/manifests/service-redis.yaml | 18 ++ twenty/code/k8s/manifests/service-server.yaml | 8 +- twenty/code/k8s/terraform/.terraform-docs.yml | 8 +- twenty/code/k8s/terraform/README.md | 31 ++- twenty/code/k8s/terraform/deployment-redis.tf | 60 +++++ .../code/k8s/terraform/deployment-server.tf | 77 +++--- .../code/k8s/terraform/deployment-worker.tf | 54 ++-- twenty/code/k8s/terraform/main.tf | 6 +- twenty/code/k8s/terraform/pv-docker-data.tf | 19 ++ twenty/code/k8s/terraform/pvc-docker-data.tf | 15 ++ twenty/code/k8s/terraform/secret.tf | 25 +- twenty/code/k8s/terraform/service-redis.tf | 18 ++ twenty/code/k8s/terraform/variables.tf | 66 +++-- twenty/code/scripts/1-click.sh | 22 ++ twenty/code/scripts/install.sh | 166 ++++++++++++ twenty/code/twenty-postgres-spilo/Dockerfile | 22 +- twenty/code/twenty-postgres/Dockerfile | 54 ---- twenty/code/twenty-postgres/init.sql | 4 - twenty/code/twenty/Dockerfile | 3 +- twenty/code/twenty/entrypoint.sh | 9 + 52 files changed, 1279 insertions(+), 641 deletions(-) create mode 100644 dify/code/couchbase-server/Dockerfile create mode 100755 dify/code/couchbase-server/init-cbserver.sh create mode 100644 dify/code/volumes/oceanbase/init.d/vec_memory.sql create mode 100644 dify/code/volumes/sandbox/conf/config.yaml create mode 100644 dify/code/volumes/sandbox/conf/config.yaml.example create mode 100644 supabase/code/volumes/db/_supabase.sql create mode 100644 supabase/code/volumes/db/pooler.sql create mode 100644 supabase/code/volumes/pooler/pooler.exs create mode 100644 twenty/code/k8s/manifests/deployment-redis.yaml create mode 100644 twenty/code/k8s/manifests/pv-docker-data.yaml create mode 100644 twenty/code/k8s/manifests/pvc-docker-data.yaml create mode 100644 twenty/code/k8s/manifests/service-redis.yaml create mode 100644 twenty/code/k8s/terraform/deployment-redis.tf create mode 100644 twenty/code/k8s/terraform/pv-docker-data.tf create mode 100644 twenty/code/k8s/terraform/pvc-docker-data.tf create mode 100644 twenty/code/k8s/terraform/service-redis.tf create mode 100644 twenty/code/scripts/1-click.sh create mode 100755 twenty/code/scripts/install.sh delete mode 100644 twenty/code/twenty-postgres/Dockerfile delete mode 100644 twenty/code/twenty-postgres/init.sql diff --git a/appwrite/code/docker-compose.yml b/appwrite/code/docker-compose.yml index ea853f6..cca2c8d 100644 --- a/appwrite/code/docker-compose.yml +++ b/appwrite/code/docker-compose.yml @@ -7,7 +7,6 @@ x-logging: &x-logging services: traefik: image: traefik:2.11 - # container_name: appwrite-traefik <<: *x-logging command: - --providers.file.directory=/storage/config @@ -18,9 +17,6 @@ services: - --entrypoints.appwrite_web.address=:80 - --entrypoints.appwrite_websecure.address=:443 restart: unless-stopped - # ports: - # - 80:80 - # - 443:443 volumes: - /var/run/docker.sock:/var/run/docker.sock - appwrite-config:/storage/config:ro @@ -33,7 +29,6 @@ services: appwrite: image: appwrite/appwrite:1.6.0 - # container_name: appwrite <<: *x-logging restart: unless-stopped networks: @@ -61,7 +56,7 @@ services: depends_on: - mariadb - redis -# - clamav + # - clamav environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -158,7 +153,6 @@ services: appwrite-console: <<: *x-logging - # container_name: appwrite-console image: appwrite/console:5.0.12 restart: unless-stopped networks: @@ -181,7 +175,6 @@ services: appwrite-realtime: image: appwrite/appwrite:1.6.0 entrypoint: realtime - # container_name: appwrite-realtime <<: *x-logging restart: unless-stopped labels: @@ -225,7 +218,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: worker-audits <<: *x-logging - # container_name: appwrite-worker-audits restart: unless-stopped networks: - appwrite @@ -251,7 +243,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: worker-webhooks <<: *x-logging - # container_name: appwrite-worker-webhooks restart: unless-stopped networks: - appwrite @@ -279,7 +270,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: worker-deletes <<: *x-logging - # container_name: appwrite-worker-deletes restart: unless-stopped networks: - appwrite @@ -337,7 +327,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: worker-databases <<: *x-logging - # container_name: appwrite-worker-databases restart: unless-stopped networks: - appwrite @@ -363,7 +352,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: worker-builds <<: *x-logging - # container_name: appwrite-worker-builds restart: unless-stopped networks: - appwrite @@ -426,7 +414,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: worker-certificates <<: *x-logging - # container_name: appwrite-worker-certificates restart: unless-stopped networks: - appwrite @@ -459,7 +446,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: worker-functions <<: *x-logging - # container_name: appwrite-worker-functions restart: unless-stopped networks: - appwrite @@ -497,7 +483,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: worker-mails <<: *x-logging - # container_name: appwrite-worker-mails restart: unless-stopped networks: - appwrite @@ -529,7 +514,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: worker-messaging <<: *x-logging - # container_name: appwrite-worker-messaging restart: unless-stopped networks: - appwrite @@ -579,7 +563,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: worker-migrations <<: *x-logging - # container_name: appwrite-worker-migrations restart: unless-stopped networks: - appwrite @@ -609,7 +592,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: maintenance <<: *x-logging - # container_name: appwrite-task-maintenance restart: unless-stopped networks: - appwrite @@ -642,7 +624,6 @@ services: appwrite-worker-usage: image: appwrite/appwrite:1.6.0 entrypoint: worker-usage - # container_name: appwrite-worker-usage <<: *x-logging restart: unless-stopped networks: @@ -671,7 +652,6 @@ services: image: appwrite/appwrite:1.6.0 entrypoint: worker-usage-dump <<: *x-logging - # container_name: appwrite-worker-usage-dump networks: - appwrite depends_on: @@ -697,7 +677,6 @@ services: appwrite-task-scheduler-functions: image: appwrite/appwrite:1.6.0 entrypoint: schedule-functions - # container_name: appwrite-task-scheduler-functions <<: *x-logging restart: unless-stopped networks: @@ -722,7 +701,6 @@ services: appwrite-task-scheduler-executions: image: appwrite/appwrite:1.6.0 entrypoint: schedule-executions - # container_name: appwrite-task-scheduler-executions <<: *x-logging restart: unless-stopped networks: @@ -747,7 +725,6 @@ services: appwrite-task-scheduler-messages: image: appwrite/appwrite:1.6.0 entrypoint: schedule-messages - # container_name: appwrite-task-scheduler-messages <<: *x-logging restart: unless-stopped networks: @@ -771,7 +748,6 @@ services: appwrite-assistant: image: appwrite/assistant:0.4.0 - # container_name: appwrite-assistant <<: *x-logging restart: unless-stopped networks: @@ -780,7 +756,6 @@ services: - _APP_ASSISTANT_OPENAI_API_KEY openruntimes-executor: - # container_name: openruntimes-executor hostname: exc1 <<: *x-logging restart: unless-stopped @@ -830,7 +805,6 @@ services: mariadb: image: mariadb:10.11 # fix issues when upgrading using: mysql_upgrade -u root -p - # container_name: appwrite-mariadb <<: *x-logging restart: unless-stopped networks: @@ -847,22 +821,18 @@ services: redis: image: redis:7.2.4-alpine - # container_name: appwrite-redis <<: *x-logging restart: unless-stopped command: > - redis-server - --maxmemory 512mb - --maxmemory-policy allkeys-lru - --maxmemory-samples 5 + redis-server --maxmemory 512mb + --maxmemory-policy allkeys-lru --maxmemory-samples 5 networks: - appwrite volumes: - appwrite-redis:/data:rw - # clamav: # image: appwrite/clamav:1.2.0 - container_name: appwrite-clamav + # container_name: appwrite-clamav # restart: unless-stopped # networks: # - appwrite diff --git a/dify/code/.env.example b/dify/code/.env.example index 0a9e081..719a025 100644 --- a/dify/code/.env.example +++ b/dify/code/.env.example @@ -49,11 +49,15 @@ FILES_URL= # Supported values are `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` LOG_LEVEL=INFO # Log file path -LOG_FILE= +LOG_FILE=/app/logs/server.log # Log file max size, the unit is MB LOG_FILE_MAX_SIZE=20 # Log file max backup count LOG_FILE_BACKUP_COUNT=5 +# Log dateformat +LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S +# Log Timezone +LOG_TZ=UTC # Debug mode, default is false. # It is recommended to turn on this configuration for local development @@ -71,7 +75,8 @@ SECRET_KEY=sk-9f73s3ljTXVcMT3Blb3ljTqtsKiGHXVcMT3BlbkFJLK7U # Password for admin user initialization. # If left unset, admin user will not be prompted for a password -# when creating the initial admin account. +# when creating the initial admin account. +# The length of the password cannot exceed 30 charactors. INIT_PASSWORD= # Deployment environment. @@ -235,6 +240,12 @@ REDIS_SENTINEL_USERNAME= REDIS_SENTINEL_PASSWORD= REDIS_SENTINEL_SOCKET_TIMEOUT=0.1 +# List of Redis Cluster nodes. If Cluster mode is enabled, provide at least one Cluster IP and port. +# Format: `:,:,:` +REDIS_USE_CLUSTERS=false +REDIS_CLUSTERS= +REDIS_CLUSTERS_PASSWORD= + # ------------------------------ # Celery Configuration # ------------------------------ @@ -273,6 +284,7 @@ CONSOLE_CORS_ALLOW_ORIGINS=* # Supported values are `local` , `s3` , `azure-blob` , `google-storage`, `tencent-cos`, `huawei-obs`, `volcengine-tos`, `baidu-obs`, `supabase` # Default: `local` STORAGE_TYPE=local +STORAGE_LOCAL_PATH=storage # S3 Configuration # Whether to use AWS managed IAM roles for authenticating with the S3 service. @@ -373,7 +385,7 @@ SUPABASE_URL=your-server-url # ------------------------------ # The type of vector store to use. -# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `tidb_vector`, `oracle`, `tencent`, `elasticsearch`, `analyticdb`, `vikingdb`. +# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `tidb_vector`, `oracle`, `tencent`, `elasticsearch`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`. VECTOR_STORE=weaviate # The Weaviate endpoint URL. Only available when VECTOR_STORE is `weaviate`. @@ -412,6 +424,14 @@ MYSCALE_PASSWORD= MYSCALE_DATABASE=dify MYSCALE_FTS_PARAMS= +# Couchbase configurations, only available when VECTOR_STORE is `couchbase` +# The connection string must include hostname defined in the docker-compose file (couchbase-server in this case) +COUCHBASE_CONNECTION_STRING=couchbase://couchbase-server +COUCHBASE_USER=Administrator +COUCHBASE_PASSWORD=password +COUCHBASE_BUCKET_NAME=Embeddings +COUCHBASE_SCOPE_NAME=_default + # pgvector configurations, only available when VECTOR_STORE is `pgvector` PGVECTOR_HOST=pgvector PGVECTOR_PORT=5432 @@ -437,6 +457,10 @@ ANALYTICDB_ACCOUNT=testaccount ANALYTICDB_PASSWORD=testpassword ANALYTICDB_NAMESPACE=dify ANALYTICDB_NAMESPACE_PASSWORD=difypassword +ANALYTICDB_HOST=gp-test.aliyuncs.com +ANALYTICDB_PORT=5432 +ANALYTICDB_MIN_CONNECTION=1 +ANALYTICDB_MAX_CONNECTION=5 # TiDB vector configurations, only available when VECTOR_STORE is `tidb` TIDB_VECTOR_HOST=tidb @@ -445,6 +469,20 @@ TIDB_VECTOR_USER=xxx.root TIDB_VECTOR_PASSWORD=xxxxxx TIDB_VECTOR_DATABASE=dify +# Tidb on qdrant configuration, only available when VECTOR_STORE is `tidb_on_qdrant` +TIDB_ON_QDRANT_URL=http://127.0.0.1 +TIDB_ON_QDRANT_API_KEY=dify +TIDB_ON_QDRANT_CLIENT_TIMEOUT=20 +TIDB_ON_QDRANT_GRPC_ENABLED=false +TIDB_ON_QDRANT_GRPC_PORT=6334 +TIDB_PUBLIC_KEY=dify +TIDB_PRIVATE_KEY=dify +TIDB_API_URL=http://127.0.0.1 +TIDB_IAM_API_URL=http://127.0.0.1 +TIDB_REGION=regions/aws-us-east-1 +TIDB_PROJECT_ID=dify +TIDB_SPEND_LIMIT=100 + # Chroma configuration, only available when VECTOR_STORE is `chroma` CHROMA_HOST=127.0.0.1 CHROMA_PORT=8000 @@ -507,6 +545,20 @@ VIKINGDB_SCHEMA=http VIKINGDB_CONNECTION_TIMEOUT=30 VIKINGDB_SOCKET_TIMEOUT=30 + +# Lindorm configuration, only available when VECTOR_STORE is `lindorm` +LINDORM_URL=http://ld-***************-proxy-search-pub.lindorm.aliyuncs.com:30070 +LINDORM_USERNAME=username +LINDORM_PASSWORD=password + +# OceanBase Vector configuration, only available when VECTOR_STORE is `oceanbase` +OCEANBASE_VECTOR_HOST=oceanbase +OCEANBASE_VECTOR_PORT=2881 +OCEANBASE_VECTOR_USER=root@test +OCEANBASE_VECTOR_PASSWORD=difyai123456 +OCEANBASE_VECTOR_DATABASE=test +OCEANBASE_MEMORY_LIMIT=6G + # ------------------------------ # Knowledge Configuration # ------------------------------ @@ -517,29 +569,55 @@ UPLOAD_FILE_SIZE_LIMIT=15 # The maximum number of files that can be uploaded at a time, default 5. UPLOAD_FILE_BATCH_LIMIT=5 -# ETl type, support: `dify`, `Unstructured` +# ETL type, support: `dify`, `Unstructured` # `dify` Dify's proprietary file extraction scheme # `Unstructured` Unstructured.io file extraction scheme ETL_TYPE=dify -# Unstructured API path, needs to be configured when ETL_TYPE is Unstructured. +# Unstructured API path and API key, needs to be configured when ETL_TYPE is Unstructured +# Or using Unstructured for document extractor node for pptx. # For example: http://unstructured:8000/general/v0/general UNSTRUCTURED_API_URL= +UNSTRUCTURED_API_KEY= +SCARF_NO_ANALYTICS=true + +# ------------------------------ +# Model Configuration +# ------------------------------ + +# The maximum number of tokens allowed for prompt generation. +# This setting controls the upper limit of tokens that can be used by the LLM +# when generating a prompt in the prompt generation tool. +# Default: 512 tokens. +PROMPT_GENERATION_MAX_TOKENS=512 + +# The maximum number of tokens allowed for code generation. +# This setting controls the upper limit of tokens that can be used by the LLM +# when generating code in the code generation tool. +# Default: 1024 tokens. +CODE_GENERATION_MAX_TOKENS=1024 # ------------------------------ # Multi-modal Configuration # ------------------------------ -# The format of the image sent when the multi-modal model is input, +# The format of the image/video sent when the multi-modal model is input, # the default is base64, optional url. # The delay of the call in url mode will be lower than that in base64 mode. # It is generally recommended to use the more compatible base64 mode. -# If configured as url, you need to configure FILES_URL as an externally accessible address so that the multi-modal model can access the image. +# If configured as url, you need to configure FILES_URL as an externally accessible address so that the multi-modal model can access the image/video. MULTIMODAL_SEND_IMAGE_FORMAT=base64 +MULTIMODAL_SEND_VIDEO_FORMAT=base64 # Upload image file size limit, default 10M. UPLOAD_IMAGE_FILE_SIZE_LIMIT=10 +# Upload video file size limit, default 100M. +UPLOAD_VIDEO_FILE_SIZE_LIMIT=100 + +# Upload audio file size limit, default 50M. +UPLOAD_AUDIO_FILE_SIZE_LIMIT=50 + # ------------------------------ # Sentry Configuration # Used for application monitoring and error log tracking. @@ -605,7 +683,7 @@ SMTP_OPPORTUNISTIC_TLS=false # ------------------------------ # Maximum length of segmentation tokens for indexing -INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000 +INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 # Member invitation link valid time (hours), # Default: 72. @@ -625,12 +703,16 @@ TEMPLATE_TRANSFORM_MAX_LENGTH=80000 CODE_MAX_STRING_ARRAY_LENGTH=30 CODE_MAX_OBJECT_ARRAY_LENGTH=30 CODE_MAX_NUMBER_ARRAY_LENGTH=1000 +CODE_EXECUTION_CONNECT_TIMEOUT=10 +CODE_EXECUTION_READ_TIMEOUT=60 +CODE_EXECUTION_WRITE_TIMEOUT=10 # Workflow runtime configuration WORKFLOW_MAX_EXECUTION_STEPS=500 WORKFLOW_MAX_EXECUTION_TIME=1200 WORKFLOW_CALL_MAX_DEPTH=5 MAX_VARIABLE_SIZE=204800 +WORKFLOW_FILE_UPLOAD_LIMIT=10 # HTTP request node in workflow configuration HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 @@ -846,3 +928,6 @@ POSITION_PROVIDER_EXCLUDES= # CSP https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP CSP_WHITELIST= + +# Enable or disable create tidb service job +CREATE_TIDB_SERVICE_JOB_ENABLED=false diff --git a/dify/code/README.md b/dify/code/README.md index 7ce3f9b..c3cd1f9 100644 --- a/dify/code/README.md +++ b/dify/code/README.md @@ -36,7 +36,7 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T - Navigate to the `docker` directory. - Ensure the `middleware.env` file is created by running `cp middleware.env.example middleware.env` (refer to the `middleware.env.example` file). 2. **Running Middleware Services**: - - Execute `docker-compose -f docker-compose.middleware.yaml up -d` to start the middleware services. + - Execute `docker-compose -f docker-compose.middleware.yaml up --env-file middleware.env -d` to start the middleware services. ### Migration for Existing Users diff --git a/dify/code/couchbase-server/Dockerfile b/dify/code/couchbase-server/Dockerfile new file mode 100644 index 0000000..bd8af64 --- /dev/null +++ b/dify/code/couchbase-server/Dockerfile @@ -0,0 +1,4 @@ +FROM couchbase/server:latest AS stage_base +# FROM couchbase:latest AS stage_base +COPY init-cbserver.sh /opt/couchbase/init/ +RUN chmod +x /opt/couchbase/init/init-cbserver.sh \ No newline at end of file diff --git a/dify/code/couchbase-server/init-cbserver.sh b/dify/code/couchbase-server/init-cbserver.sh new file mode 100755 index 0000000..e66bc18 --- /dev/null +++ b/dify/code/couchbase-server/init-cbserver.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# used to start couchbase server - can't get around this as docker compose only allows you to start one command - so we have to start couchbase like the standard couchbase Dockerfile would +# https://github.com/couchbase/docker/blob/master/enterprise/couchbase-server/7.2.0/Dockerfile#L88 + +/entrypoint.sh couchbase-server & + +# track if setup is complete so we don't try to setup again +FILE=/opt/couchbase/init/setupComplete.txt + +if ! [ -f "$FILE" ]; then + # used to automatically create the cluster based on environment variables + # https://docs.couchbase.com/server/current/cli/cbcli/couchbase-cli-cluster-init.html + + echo $COUCHBASE_ADMINISTRATOR_USERNAME ":" $COUCHBASE_ADMINISTRATOR_PASSWORD + + sleep 20s + /opt/couchbase/bin/couchbase-cli cluster-init -c 127.0.0.1 \ + --cluster-username $COUCHBASE_ADMINISTRATOR_USERNAME \ + --cluster-password $COUCHBASE_ADMINISTRATOR_PASSWORD \ + --services data,index,query,fts \ + --cluster-ramsize $COUCHBASE_RAM_SIZE \ + --cluster-index-ramsize $COUCHBASE_INDEX_RAM_SIZE \ + --cluster-eventing-ramsize $COUCHBASE_EVENTING_RAM_SIZE \ + --cluster-fts-ramsize $COUCHBASE_FTS_RAM_SIZE \ + --index-storage-setting default + + sleep 2s + + # used to auto create the bucket based on environment variables + # https://docs.couchbase.com/server/current/cli/cbcli/couchbase-cli-bucket-create.html + + /opt/couchbase/bin/couchbase-cli bucket-create -c localhost:8091 \ + --username $COUCHBASE_ADMINISTRATOR_USERNAME \ + --password $COUCHBASE_ADMINISTRATOR_PASSWORD \ + --bucket $COUCHBASE_BUCKET \ + --bucket-ramsize $COUCHBASE_BUCKET_RAMSIZE \ + --bucket-type couchbase + + # create file so we know that the cluster is setup and don't run the setup again + touch $FILE +fi + # docker compose will stop the container from running unless we do this + # known issue and workaround + tail -f /dev/null diff --git a/dify/code/docker-compose.middleware.yaml b/dify/code/docker-compose.middleware.yaml index d7900de..11f5302 100644 --- a/dify/code/docker-compose.middleware.yaml +++ b/dify/code/docker-compose.middleware.yaml @@ -16,7 +16,7 @@ services: -c 'maintenance_work_mem=${POSTGRES_MAINTENANCE_WORK_MEM:-64MB}' -c 'effective_cache_size=${POSTGRES_EFFECTIVE_CACHE_SIZE:-4096MB}' volumes: - - ./volumes/db/data:/var/lib/postgresql/data + - ${PGDATA_HOST_VOLUME:-./volumes/db/data}:/var/lib/postgresql/data ports: - "${EXPOSE_POSTGRES_PORT:-5432}:5432" healthcheck: @@ -29,11 +29,13 @@ services: redis: image: redis:6-alpine restart: always + environment: + REDISCLI_AUTH: ${REDIS_PASSWORD:-difyai123456} volumes: # Mount the redis data directory to the container. - - ./volumes/redis/data:/data + - ${REDIS_HOST_VOLUME:-./volumes/redis/data}:/data # Set the redis password when startup redis server. - command: redis-server --requirepass difyai123456 + command: redis-server --requirepass ${REDIS_PASSWORD:-difyai123456} ports: - "${EXPOSE_REDIS_PORT:-6379}:6379" healthcheck: @@ -41,7 +43,7 @@ services: # The DifySandbox sandbox: - image: langgenius/dify-sandbox:0.2.9 + image: langgenius/dify-sandbox:0.2.10 restart: always environment: # The DifySandbox configurations @@ -56,6 +58,7 @@ services: SANDBOX_PORT: ${SANDBOX_PORT:-8194} volumes: - ./volumes/sandbox/dependencies:/dependencies + - ./volumes/sandbox/conf:/conf healthcheck: test: [ "CMD", "curl", "-f", "http://localhost:8194/health" ] networks: @@ -94,7 +97,7 @@ services: restart: always volumes: # Mount the Weaviate data directory to the container. - - ./volumes/weaviate:/var/lib/weaviate + - ${WEAVIATE_HOST_VOLUME:-./volumes/weaviate}:/var/lib/weaviate env_file: - ./middleware.env environment: diff --git a/dify/code/docker-compose.yaml b/dify/code/docker-compose.yaml index 3d2039f..102ef58 100644 --- a/dify/code/docker-compose.yaml +++ b/dify/code/docker-compose.yaml @@ -1,6 +1,13 @@ x-shared-env: &shared-api-worker-env + WORKFLOW_FILE_UPLOAD_LIMIT: ${WORKFLOW_FILE_UPLOAD_LIMIT:-10} LOG_LEVEL: ${LOG_LEVEL:-INFO} LOG_FILE: ${LOG_FILE:-} + LOG_FILE_MAX_SIZE: ${LOG_FILE_MAX_SIZE:-20} + LOG_FILE_BACKUP_COUNT: ${LOG_FILE_BACKUP_COUNT:-5} + # Log dateformat + LOG_DATEFORMAT: ${LOG_DATEFORMAT:-%Y-%m-%d %H:%M:%S} + # Log Timezone + LOG_TZ: ${LOG_TZ:-UTC} DEBUG: ${DEBUG:-false} FLASK_DEBUG: ${FLASK_DEBUG:-false} SECRET_KEY: ${SECRET_KEY:-sk-9f73s3ljTXVcMT3Blb3ljTqtsKiGHXVcMT3BlbkFJLK7U} @@ -41,13 +48,17 @@ x-shared-env: &shared-api-worker-env REDIS_USERNAME: ${REDIS_USERNAME:-} REDIS_PASSWORD: ${REDIS_PASSWORD:-difyai123456} REDIS_USE_SSL: ${REDIS_USE_SSL:-false} - REDIS_DB: 0 + REDIS_DB: ${REDIS_DB:-0} REDIS_USE_SENTINEL: ${REDIS_USE_SENTINEL:-false} REDIS_SENTINELS: ${REDIS_SENTINELS:-} REDIS_SENTINEL_SERVICE_NAME: ${REDIS_SENTINEL_SERVICE_NAME:-} REDIS_SENTINEL_USERNAME: ${REDIS_SENTINEL_USERNAME:-} REDIS_SENTINEL_PASSWORD: ${REDIS_SENTINEL_PASSWORD:-} REDIS_SENTINEL_SOCKET_TIMEOUT: ${REDIS_SENTINEL_SOCKET_TIMEOUT:-0.1} + REDIS_CLUSTERS: ${REDIS_CLUSTERS:-} + REDIS_USE_CLUSTERS: ${REDIS_USE_CLUSTERS:-false} + REDIS_CLUSTERS_PASSWORD: ${REDIS_CLUSTERS_PASSWORD:-} + ACCESS_TOKEN_EXPIRE_MINUTES: ${ACCESS_TOKEN_EXPIRE_MINUTES:-60} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://:difyai123456@redis:6379/1} BROKER_USE_SSL: ${BROKER_USE_SSL:-false} CELERY_USE_SENTINEL: ${CELERY_USE_SENTINEL:-false} @@ -56,7 +67,7 @@ x-shared-env: &shared-api-worker-env WEB_API_CORS_ALLOW_ORIGINS: ${WEB_API_CORS_ALLOW_ORIGINS:-*} CONSOLE_CORS_ALLOW_ORIGINS: ${CONSOLE_CORS_ALLOW_ORIGINS:-*} STORAGE_TYPE: ${STORAGE_TYPE:-local} - STORAGE_LOCAL_PATH: storage + STORAGE_LOCAL_PATH: ${STORAGE_LOCAL_PATH:-storage} S3_USE_AWS_MANAGED_IAM: ${S3_USE_AWS_MANAGED_IAM:-false} S3_ENDPOINT: ${S3_ENDPOINT:-} S3_BUCKET_NAME: ${S3_BUCKET_NAME:-} @@ -95,6 +106,10 @@ x-shared-env: &shared-api-worker-env VOLCENGINE_TOS_ACCESS_KEY: ${VOLCENGINE_TOS_ACCESS_KEY:-} VOLCENGINE_TOS_ENDPOINT: ${VOLCENGINE_TOS_ENDPOINT:-} VOLCENGINE_TOS_REGION: ${VOLCENGINE_TOS_REGION:-} + BAIDU_OBS_BUCKET_NAME: ${BAIDU_OBS_BUCKET_NAME:-} + BAIDU_OBS_SECRET_KEY: ${BAIDU_OBS_SECRET_KEY:-} + BAIDU_OBS_ACCESS_KEY: ${BAIDU_OBS_ACCESS_KEY:-} + BAIDU_OBS_ENDPOINT: ${BAIDU_OBS_ENDPOINT:-} VECTOR_STORE: ${VECTOR_STORE:-weaviate} WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080} WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} @@ -103,6 +118,11 @@ x-shared-env: &shared-api-worker-env QDRANT_CLIENT_TIMEOUT: ${QDRANT_CLIENT_TIMEOUT:-20} QDRANT_GRPC_ENABLED: ${QDRANT_GRPC_ENABLED:-false} QDRANT_GRPC_PORT: ${QDRANT_GRPC_PORT:-6334} + COUCHBASE_CONNECTION_STRING: ${COUCHBASE_CONNECTION_STRING:-'couchbase-server'} + COUCHBASE_USER: ${COUCHBASE_USER:-Administrator} + COUCHBASE_PASSWORD: ${COUCHBASE_PASSWORD:-password} + COUCHBASE_BUCKET_NAME: ${COUCHBASE_BUCKET_NAME:-Embeddings} + COUCHBASE_SCOPE_NAME: ${COUCHBASE_SCOPE_NAME:-_default} MILVUS_URI: ${MILVUS_URI:-http://127.0.0.1:19530} MILVUS_TOKEN: ${MILVUS_TOKEN:-} MILVUS_USER: ${MILVUS_USER:-root} @@ -128,6 +148,18 @@ x-shared-env: &shared-api-worker-env TIDB_VECTOR_USER: ${TIDB_VECTOR_USER:-} TIDB_VECTOR_PASSWORD: ${TIDB_VECTOR_PASSWORD:-} TIDB_VECTOR_DATABASE: ${TIDB_VECTOR_DATABASE:-dify} + TIDB_ON_QDRANT_URL: ${TIDB_ON_QDRANT_URL:-http://127.0.0.1} + TIDB_ON_QDRANT_API_KEY: ${TIDB_ON_QDRANT_API_KEY:-dify} + TIDB_ON_QDRANT_CLIENT_TIMEOUT: ${TIDB_ON_QDRANT_CLIENT_TIMEOUT:-20} + TIDB_ON_QDRANT_GRPC_ENABLED: ${TIDB_ON_QDRANT_GRPC_ENABLED:-false} + TIDB_ON_QDRANT_GRPC_PORT: ${TIDB_ON_QDRANT_GRPC_PORT:-6334} + TIDB_PUBLIC_KEY: ${TIDB_PUBLIC_KEY:-dify} + TIDB_PRIVATE_KEY: ${TIDB_PRIVATE_KEY:-dify} + TIDB_API_URL: ${TIDB_API_URL:-http://127.0.0.1} + TIDB_IAM_API_URL: ${TIDB_IAM_API_URL:-http://127.0.0.1} + TIDB_REGION: ${TIDB_REGION:-regions/aws-us-east-1} + TIDB_PROJECT_ID: ${TIDB_PROJECT_ID:-dify} + TIDB_SPEND_LIMIT: ${TIDB_SPEND_LIMIT:-100} ORACLE_HOST: ${ORACLE_HOST:-oracle} ORACLE_PORT: ${ORACLE_PORT:-1521} ORACLE_USER: ${ORACLE_USER:-dify} @@ -143,6 +175,9 @@ x-shared-env: &shared-api-worker-env ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_USERNAME:-elastic} ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-elastic} + LINDORM_URL: ${LINDORM_URL:-http://lindorm:30070} + LINDORM_USERNAME: ${LINDORM_USERNAME:-lindorm} + LINDORM_PASSWORD: ${LINDORM_PASSWORD:-lindorm } KIBANA_PORT: ${KIBANA_PORT:-5601} # AnalyticDB configuration ANALYTICDB_KEY_ID: ${ANALYTICDB_KEY_ID:-} @@ -153,6 +188,10 @@ x-shared-env: &shared-api-worker-env ANALYTICDB_PASSWORD: ${ANALYTICDB_PASSWORD:-} ANALYTICDB_NAMESPACE: ${ANALYTICDB_NAMESPACE:-dify} ANALYTICDB_NAMESPACE_PASSWORD: ${ANALYTICDB_NAMESPACE_PASSWORD:-} + ANALYTICDB_HOST: ${ANALYTICDB_HOST:-} + ANALYTICDB_PORT: ${ANALYTICDB_PORT:-5432} + ANALYTICDB_MIN_CONNECTION: ${ANALYTICDB_MIN_CONNECTION:-1} + ANALYTICDB_MAX_CONNECTION: ${ANALYTICDB_MAX_CONNECTION:-5} OPENSEARCH_HOST: ${OPENSEARCH_HOST:-opensearch} OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200} OPENSEARCH_USER: ${OPENSEARCH_USER:-admin} @@ -165,12 +204,32 @@ x-shared-env: &shared-api-worker-env TENCENT_VECTOR_DB_DATABASE: ${TENCENT_VECTOR_DB_DATABASE:-dify} TENCENT_VECTOR_DB_SHARD: ${TENCENT_VECTOR_DB_SHARD:-1} TENCENT_VECTOR_DB_REPLICAS: ${TENCENT_VECTOR_DB_REPLICAS:-2} + BAIDU_VECTOR_DB_ENDPOINT: ${BAIDU_VECTOR_DB_ENDPOINT:-http://127.0.0.1:5287} + BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS: ${BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS:-30000} + BAIDU_VECTOR_DB_ACCOUNT: ${BAIDU_VECTOR_DB_ACCOUNT:-root} + BAIDU_VECTOR_DB_API_KEY: ${BAIDU_VECTOR_DB_API_KEY:-dify} + BAIDU_VECTOR_DB_DATABASE: ${BAIDU_VECTOR_DB_DATABASE:-dify} + BAIDU_VECTOR_DB_SHARD: ${BAIDU_VECTOR_DB_SHARD:-1} + BAIDU_VECTOR_DB_REPLICAS: ${BAIDU_VECTOR_DB_REPLICAS:-3} + VIKINGDB_ACCESS_KEY: ${VIKINGDB_ACCESS_KEY:-dify} + VIKINGDB_SECRET_KEY: ${VIKINGDB_SECRET_KEY:-dify} + VIKINGDB_REGION: ${VIKINGDB_REGION:-cn-shanghai} + VIKINGDB_HOST: ${VIKINGDB_HOST:-api-vikingdb.xxx.volces.com} + VIKINGDB_SCHEMA: ${VIKINGDB_SCHEMA:-http} + UPSTASH_VECTOR_URL: ${UPSTASH_VECTOR_URL:-https://xxx-vector.upstash.io} + UPSTASH_VECTOR_TOKEN: ${UPSTASH_VECTOR_TOKEN:-dify} UPLOAD_FILE_SIZE_LIMIT: ${UPLOAD_FILE_SIZE_LIMIT:-15} UPLOAD_FILE_BATCH_LIMIT: ${UPLOAD_FILE_BATCH_LIMIT:-5} ETL_TYPE: ${ETL_TYPE:-dify} UNSTRUCTURED_API_URL: ${UNSTRUCTURED_API_URL:-} + UNSTRUCTURED_API_KEY: ${UNSTRUCTURED_API_KEY:-} + PROMPT_GENERATION_MAX_TOKENS: ${PROMPT_GENERATION_MAX_TOKENS:-512} + CODE_GENERATION_MAX_TOKENS: ${CODE_GENERATION_MAX_TOKENS:-1024} MULTIMODAL_SEND_IMAGE_FORMAT: ${MULTIMODAL_SEND_IMAGE_FORMAT:-base64} + MULTIMODAL_SEND_VIDEO_FORMAT: ${MULTIMODAL_SEND_VIDEO_FORMAT:-base64} UPLOAD_IMAGE_FILE_SIZE_LIMIT: ${UPLOAD_IMAGE_FILE_SIZE_LIMIT:-10} + UPLOAD_VIDEO_FILE_SIZE_LIMIT: ${UPLOAD_VIDEO_FILE_SIZE_LIMIT:-100} + UPLOAD_AUDIO_FILE_SIZE_LIMIT: ${UPLOAD_AUDIO_FILE_SIZE_LIMIT:-50} SENTRY_DSN: ${API_SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} @@ -187,12 +246,15 @@ x-shared-env: &shared-api-worker-env SMTP_USE_TLS: ${SMTP_USE_TLS:-true} SMTP_OPPORTUNISTIC_TLS: ${SMTP_OPPORTUNISTIC_TLS:-false} RESEND_API_KEY: ${RESEND_API_KEY:-your-resend-api-key} - RESEND_API_URL: https://api.resend.com - INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-1000} + RESEND_API_URL: ${RESEND_API_URL:-https://api.resend.com} + INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000} INVITE_EXPIRY_HOURS: ${INVITE_EXPIRY_HOURS:-72} - RESET_PASSWORD_TOKEN_EXPIRY_HOURS: ${RESET_PASSWORD_TOKEN_EXPIRY_HOURS:-24} + RESET_PASSWORD_TOKEN_EXPIRY_MINUTES: ${RESET_PASSWORD_TOKEN_EXPIRY_MINUTES:-5} CODE_EXECUTION_ENDPOINT: ${CODE_EXECUTION_ENDPOINT:-http://sandbox:8194} CODE_EXECUTION_API_KEY: ${SANDBOX_API_KEY:-dify-sandbox} + CODE_EXECUTION_CONNECT_TIMEOUT: ${CODE_EXECUTION_CONNECT_TIMEOUT:-10} + CODE_EXECUTION_READ_TIMEOUT: ${CODE_EXECUTION_READ_TIMEOUT:-60} + CODE_EXECUTION_WRITE_TIMEOUT: ${CODE_EXECUTION_WRITE_TIMEOUT:-10} CODE_MAX_NUMBER: ${CODE_MAX_NUMBER:-9223372036854775807} CODE_MIN_NUMBER: ${CODE_MIN_NUMBER:--9223372036854775808} CODE_MAX_DEPTH: ${CODE_MAX_DEPTH:-5} @@ -204,16 +266,33 @@ x-shared-env: &shared-api-worker-env CODE_MAX_NUMBER_ARRAY_LENGTH: ${CODE_MAX_NUMBER_ARRAY_LENGTH:-1000} WORKFLOW_MAX_EXECUTION_STEPS: ${WORKFLOW_MAX_EXECUTION_STEPS:-500} WORKFLOW_MAX_EXECUTION_TIME: ${WORKFLOW_MAX_EXECUTION_TIME:-1200} - WORKFLOW_CALL_MAX_DEPTH: ${WORKFLOW_MAX_EXECUTION_TIME:-5} + WORKFLOW_CALL_MAX_DEPTH: ${WORKFLOW_CALL_MAX_DEPTH:-5} SSRF_PROXY_HTTP_URL: ${SSRF_PROXY_HTTP_URL:-http://ssrf_proxy:3128} SSRF_PROXY_HTTPS_URL: ${SSRF_PROXY_HTTPS_URL:-http://ssrf_proxy:3128} HTTP_REQUEST_NODE_MAX_BINARY_SIZE: ${HTTP_REQUEST_NODE_MAX_BINARY_SIZE:-10485760} HTTP_REQUEST_NODE_MAX_TEXT_SIZE: ${HTTP_REQUEST_NODE_MAX_TEXT_SIZE:-1048576} + APP_MAX_EXECUTION_TIME: ${APP_MAX_EXECUTION_TIME:-12000} + POSITION_TOOL_PINS: ${POSITION_TOOL_PINS:-} + POSITION_TOOL_INCLUDES: ${POSITION_TOOL_INCLUDES:-} + POSITION_TOOL_EXCLUDES: ${POSITION_TOOL_EXCLUDES:-} + POSITION_PROVIDER_PINS: ${POSITION_PROVIDER_PINS:-} + POSITION_PROVIDER_INCLUDES: ${POSITION_PROVIDER_INCLUDES:-} + POSITION_PROVIDER_EXCLUDES: ${POSITION_PROVIDER_EXCLUDES:-} + MAX_VARIABLE_SIZE: ${MAX_VARIABLE_SIZE:-204800} + OCEANBASE_VECTOR_HOST: ${OCEANBASE_VECTOR_HOST:-http://oceanbase-vector} + OCEANBASE_VECTOR_PORT: ${OCEANBASE_VECTOR_PORT:-2881} + OCEANBASE_VECTOR_USER: ${OCEANBASE_VECTOR_USER:-root@test} + OCEANBASE_VECTOR_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} + OCEANBASE_VECTOR_DATABASE: ${OCEANBASE_VECTOR_DATABASE:-test} + OCEANBASE_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} + OCEANBASE_MEMORY_LIMIT: ${OCEANBASE_MEMORY_LIMIT:-6G} + CREATE_TIDB_SERVICE_JOB_ENABLED: ${CREATE_TIDB_SERVICE_JOB_ENABLED:-false} + RETRIEVAL_TOP_N: ${RETRIEVAL_TOP_N:-0} services: # API service api: - image: langgenius/dify-api:0.11.0 + image: langgenius/dify-api:0.13.0 restart: always environment: # Use the shared environment variables. @@ -233,7 +312,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.11.0 + image: langgenius/dify-api:0.13.0 restart: always environment: # Use the shared environment variables. @@ -252,7 +331,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.11.0 + image: langgenius/dify-web:0.13.0 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -260,6 +339,7 @@ services: SENTRY_DSN: ${WEB_SENTRY_DSN:-} NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-0} TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} + CSP_WHITELIST: ${CSP_WHITELIST:-} # The postgres database. db: @@ -279,7 +359,7 @@ services: volumes: - ./volumes/db/data:/var/lib/postgresql/data healthcheck: - test: [ "CMD", "pg_isready" ] + test: [ 'CMD', 'pg_isready' ] interval: 1s timeout: 3s retries: 30 @@ -288,17 +368,19 @@ services: redis: image: redis:6-alpine restart: always + environment: + REDISCLI_AUTH: ${REDIS_PASSWORD:-difyai123456} volumes: # Mount the redis data directory to the container. - ./volumes/redis/data:/data # Set the redis password when startup redis server. command: redis-server --requirepass ${REDIS_PASSWORD:-difyai123456} healthcheck: - test: [ "CMD", "redis-cli", "ping" ] + test: [ 'CMD', 'redis-cli', 'ping' ] # The DifySandbox sandbox: - image: langgenius/dify-sandbox:0.2.9 + image: langgenius/dify-sandbox:0.2.10 restart: always environment: # The DifySandbox configurations @@ -314,7 +396,7 @@ services: volumes: - ./volumes/sandbox/dependencies:/dependencies healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:8194/health" ] + test: [ 'CMD', 'curl', '-f', 'http://localhost:8194/health' ] networks: - ssrf_proxy_network @@ -327,7 +409,14 @@ services: volumes: - ./ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template - ./ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint-mount.sh - entrypoint: [ "sh", "-c", "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" ] + entrypoint: + [ + 'sh', + '-c', + "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i + 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && + /docker-entrypoint.sh" + ] environment: # pls clearly modify the squid env vars to fit your network environment. HTTP_PORT: ${SSRF_HTTP_PORT:-3128} @@ -356,8 +445,8 @@ services: - CERTBOT_EMAIL=${CERTBOT_EMAIL} - CERTBOT_DOMAIN=${CERTBOT_DOMAIN} - CERTBOT_OPTIONS=${CERTBOT_OPTIONS:-} - entrypoint: [ "/docker-entrypoint.sh" ] - command: [ "tail", "-f", "/dev/null" ] + entrypoint: [ '/docker-entrypoint.sh' ] + command: [ 'tail', '-f', '/dev/null' ] # The nginx reverse proxy. # used for reverse proxying the API service and Web service. @@ -374,7 +463,14 @@ services: - ./volumes/certbot/conf/live:/etc/letsencrypt/live # cert dir (with certbot container) - ./volumes/certbot/conf:/etc/letsencrypt - ./volumes/certbot/www:/var/www/html - entrypoint: [ "sh", "-c", "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" ] + entrypoint: + [ + 'sh', + '-c', + "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i + 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && + /docker-entrypoint.sh" + ] environment: NGINX_SERVER_NAME: ${NGINX_SERVER_NAME:-_} NGINX_HTTPS_ENABLED: ${NGINX_HTTPS_ENABLED:-false} @@ -395,15 +491,12 @@ services: depends_on: - api - web - # ports: - # - "${EXPOSE_NGINX_PORT:-80}:${NGINX_PORT:-80}" - # - "${EXPOSE_NGINX_SSL_PORT:-443}:${NGINX_SSL_PORT:-443}" # The Weaviate vector store. weaviate: image: semitechnologies/weaviate:1.19.0 profiles: - - "" + - '' - weaviate restart: always volumes: @@ -435,6 +528,44 @@ services: environment: QDRANT_API_KEY: ${QDRANT_API_KEY:-difyai123456} + # The Couchbase vector store. + couchbase-server: + build: ./couchbase-server + profiles: + - couchbase + restart: always + environment: + - CLUSTER_NAME=dify_search + - COUCHBASE_ADMINISTRATOR_USERNAME=${COUCHBASE_USER:-Administrator} + - COUCHBASE_ADMINISTRATOR_PASSWORD=${COUCHBASE_PASSWORD:-password} + - COUCHBASE_BUCKET=${COUCHBASE_BUCKET_NAME:-Embeddings} + - COUCHBASE_BUCKET_RAMSIZE=512 + - COUCHBASE_RAM_SIZE=2048 + - COUCHBASE_EVENTING_RAM_SIZE=512 + - COUCHBASE_INDEX_RAM_SIZE=512 + - COUCHBASE_FTS_RAM_SIZE=1024 + hostname: couchbase-server + working_dir: /opt/couchbase + stdin_open: true + tty: true + entrypoint: [ "" ] + command: sh -c "/opt/couchbase/init/init-cbserver.sh" + volumes: + - ./volumes/couchbase/data:/opt/couchbase/var/lib/couchbase/data + healthcheck: + # ensure bucket was created before proceeding + test: + [ + "CMD-SHELL", + "curl -s -f -u Administrator:password + http://localhost:8091/pools/default/buckets | grep -q '\\[{' || exit + 1" + ] + interval: 10s + retries: 10 + start_period: 30s + timeout: 10s + # The pgvector vector database. pgvector: image: pgvector/pgvector:pg16 @@ -452,7 +583,7 @@ services: volumes: - ./volumes/pgvector/data:/var/lib/postgresql/data healthcheck: - test: [ "CMD", "pg_isready" ] + test: [ 'CMD', 'pg_isready' ] interval: 1s timeout: 3s retries: 30 @@ -474,14 +605,14 @@ services: volumes: - ./volumes/pgvecto_rs/data:/var/lib/postgresql/data healthcheck: - test: [ "CMD", "pg_isready" ] + test: [ 'CMD', 'pg_isready' ] interval: 1s timeout: 3s retries: 30 # Chroma vector database chroma: - image: ghcr.io/chroma-core/chroma:0.5.1 + image: ghcr.io/chroma-core/chroma:0.5.20 profiles: - chroma restart: always @@ -492,6 +623,23 @@ services: CHROMA_SERVER_AUTHN_PROVIDER: ${CHROMA_SERVER_AUTHN_PROVIDER:-chromadb.auth.token_authn.TokenAuthenticationServerProvider} IS_PERSISTENT: ${CHROMA_IS_PERSISTENT:-TRUE} + # OceanBase vector database + oceanbase: + image: quay.io/oceanbase/oceanbase-ce:4.3.3.0-100000142024101215 + profiles: + - oceanbase + restart: always + volumes: + - ./volumes/oceanbase/data:/root/ob + - ./volumes/oceanbase/conf:/root/.obd/cluster + - ./volumes/oceanbase/init.d:/root/boot/init.d + environment: + OB_MEMORY_LIMIT: ${OCEANBASE_MEMORY_LIMIT:-6G} + OB_SYS_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} + OB_TENANT_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} + OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} + OB_SERVER_IP: '127.0.0.1' + # Oracle vector database oracle: image: container-registry.oracle.com/database/free:latest @@ -509,7 +657,6 @@ services: # Milvus vector database services etcd: - # container_name: milvus-etcd image: quay.io/coreos/etcd:v3.5.5 profiles: - milvus @@ -520,9 +667,10 @@ services: ETCD_SNAPSHOT_COUNT: ${ETCD_SNAPSHOT_COUNT:-50000} volumes: - ./volumes/milvus/etcd:/etcd - command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls + http://0.0.0.0:2379 --data-dir /etcd healthcheck: - test: [ "CMD", "etcdctl", "endpoint", "health" ] + test: [ 'CMD', 'etcdctl', 'endpoint', 'health' ] interval: 30s timeout: 20s retries: 3 @@ -530,7 +678,6 @@ services: - milvus minio: - # container_name: milvus-minio image: minio/minio:RELEASE.2023-03-20T20-16-18Z profiles: - milvus @@ -541,7 +688,7 @@ services: - ./volumes/milvus/minio:/minio_data command: minio server /minio_data --console-address ":9001" healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] + test: [ 'CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live' ] interval: 30s timeout: 20s retries: 3 @@ -549,11 +696,10 @@ services: - milvus milvus-standalone: - # container_name: milvus-standalone image: milvusdb/milvus:v2.3.1 profiles: - milvus - command: [ "milvus", "run", "standalone" ] + command: [ 'milvus', 'run', 'standalone' ] environment: ETCD_ENDPOINTS: ${ETCD_ENDPOINTS:-etcd:2379} MINIO_ADDRESS: ${MINIO_ADDRESS:-minio:9000} @@ -561,7 +707,7 @@ services: volumes: - ./volumes/milvus/milvus:/var/lib/milvus healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:9091/healthz" ] + test: [ 'CMD', 'curl', '-f', 'http://localhost:9091/healthz' ] interval: 30s start_period: 90s timeout: 20s @@ -569,22 +715,19 @@ services: depends_on: - etcd - minio - # ports: - # - 19530:19530 - # - 9091:9091 networks: - milvus # Opensearch vector database opensearch: - # container_name: opensearch image: opensearchproject/opensearch:latest profiles: - opensearch environment: discovery.type: ${OPENSEARCH_DISCOVERY_TYPE:-single-node} bootstrap.memory_lock: ${OPENSEARCH_BOOTSTRAP_MEMORY_LOCK:-true} - OPENSEARCH_JAVA_OPTS: -Xms${OPENSEARCH_JAVA_OPTS_MIN:-512m} -Xmx${OPENSEARCH_JAVA_OPTS_MAX:-1024m} + OPENSEARCH_JAVA_OPTS: -Xms${OPENSEARCH_JAVA_OPTS_MIN:-512m} + -Xmx${OPENSEARCH_JAVA_OPTS_MAX:-1024m} OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-Qazwsxedc!@#123} ulimits: memlock: @@ -599,7 +742,6 @@ services: - opensearch-net opensearch-dashboards: - # container_name: opensearch-dashboards image: opensearchproject/opensearch-dashboards:latest profiles: - opensearch @@ -614,7 +756,6 @@ services: # MyScale vector database myscale: - # container_name: myscale image: myscale/myscaledb:1.6.4 profiles: - myscale @@ -624,14 +765,11 @@ services: - ./volumes/myscale/data:/var/lib/clickhouse - ./volumes/myscale/log:/var/log/clickhouse-server - ./volumes/myscale/config/users.d/custom_users_config.xml:/etc/clickhouse-server/users.d/custom_users_config.xml - # ports: - # - ${MYSCALE_PORT:-8123}:${MYSCALE_PORT:-8123} # https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html # https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-prod-prerequisites elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.15.1 - # container_name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.3 profiles: - elasticsearch restart: always @@ -643,13 +781,11 @@ services: node.name: dify-es0 discovery.type: single-node xpack.license.self_generated.type: trial - xpack.security.enabled: "true" - xpack.security.enrollment.enabled: "false" - xpack.security.http.ssl.enabled: "false" - # ports: - # - ${ELASTICSEARCH_PORT:-9200}:9200 + xpack.security.enabled: 'true' + xpack.security.enrollment.enabled: 'false' + xpack.security.http.ssl.enabled: 'false' healthcheck: - test: [ "CMD", "curl", "-s", "http://localhost:9200/_cluster/health?pretty" ] + test: [ 'CMD', 'curl', '-s', 'http://localhost:9200/_cluster/health?pretty' ] interval: 30s timeout: 10s retries: 50 @@ -657,8 +793,7 @@ services: # https://www.elastic.co/guide/en/kibana/current/docker.html # https://www.elastic.co/guide/en/kibana/current/settings.html kibana: - image: docker.elastic.co/kibana/kibana:8.15.1 - # container_name: kibana + image: docker.elastic.co/kibana/kibana:8.14.3 profiles: - elasticsearch depends_on: @@ -667,17 +802,15 @@ services: environment: XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: d1a66dfd-c4d3-4a0a-8290-2abcb83ab3aa NO_PROXY: localhost,127.0.0.1,elasticsearch,kibana - XPACK_SECURITY_ENABLED: "true" - XPACK_SECURITY_ENROLLMENT_ENABLED: "false" - XPACK_SECURITY_HTTP_SSL_ENABLED: "false" - XPACK_FLEET_ISAIRGAPPED: "true" + XPACK_SECURITY_ENABLED: 'true' + XPACK_SECURITY_ENROLLMENT_ENABLED: 'false' + XPACK_SECURITY_HTTP_SSL_ENABLED: 'false' + XPACK_FLEET_ISAIRGAPPED: 'true' I18N_LOCALE: zh-CN - SERVER_PORT: "5601" + SERVER_PORT: '5601' ELASTICSEARCH_HOSTS: http://elasticsearch:9200 - # ports: - # - ${KIBANA_PORT:-5601}:5601 healthcheck: - test: [ "CMD-SHELL", "curl -s http://localhost:5601 >/dev/null || exit 1" ] + test: [ 'CMD-SHELL', 'curl -s http://localhost:5601 >/dev/null || exit 1' ] interval: 30s timeout: 10s retries: 3 diff --git a/dify/code/middleware.env.example b/dify/code/middleware.env.example index 04d0fb5..c4ce9f0 100644 --- a/dify/code/middleware.env.example +++ b/dify/code/middleware.env.example @@ -8,6 +8,7 @@ POSTGRES_PASSWORD=difyai123456 POSTGRES_DB=dify # postgres data directory PGDATA=/var/lib/postgresql/data/pgdata +PGDATA_HOST_VOLUME=./volumes/db/data # Maximum number of connections to the database # Default is 100 @@ -39,8 +40,15 @@ POSTGRES_MAINTENANCE_WORK_MEM=64MB # Reference: https://www.postgresql.org/docs/current/runtime-config-query.html#GUC-EFFECTIVE-CACHE-SIZE POSTGRES_EFFECTIVE_CACHE_SIZE=4096MB +# ----------------------------- +# Environment Variables for redis Service +# ----------------------------- +REDIS_HOST_VOLUME=./volumes/redis/data +REDIS_PASSWORD=difyai123456 + # ------------------------------ # Environment Variables for sandbox Service +# ------------------------------ SANDBOX_API_KEY=dify-sandbox SANDBOX_GIN_MODE=release SANDBOX_WORKER_TIMEOUT=15 @@ -48,7 +56,6 @@ SANDBOX_ENABLE_NETWORK=true SANDBOX_HTTP_PROXY=http://ssrf_proxy:3128 SANDBOX_HTTPS_PROXY=http://ssrf_proxy:3128 SANDBOX_PORT=8194 -# ------------------------------ # ------------------------------ # Environment Variables for ssrf_proxy Service @@ -70,6 +77,7 @@ WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih WEAVIATE_AUTHENTICATION_APIKEY_USERS=hello@dify.ai WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED=true WEAVIATE_AUTHORIZATION_ADMINLIST_USERS=hello@dify.ai +WEAVIATE_HOST_VOLUME=./volumes/weaviate # ------------------------------ # Docker Compose Service Expose Host Port Configurations diff --git a/dify/code/volumes/oceanbase/init.d/vec_memory.sql b/dify/code/volumes/oceanbase/init.d/vec_memory.sql new file mode 100644 index 0000000..f4c283f --- /dev/null +++ b/dify/code/volumes/oceanbase/init.d/vec_memory.sql @@ -0,0 +1 @@ +ALTER SYSTEM SET ob_vector_memory_limit_percentage = 30; \ No newline at end of file diff --git a/dify/code/volumes/sandbox/conf/config.yaml b/dify/code/volumes/sandbox/conf/config.yaml new file mode 100644 index 0000000..8c1a1de --- /dev/null +++ b/dify/code/volumes/sandbox/conf/config.yaml @@ -0,0 +1,14 @@ +app: + port: 8194 + debug: True + key: dify-sandbox +max_workers: 4 +max_requests: 50 +worker_timeout: 5 +python_path: /usr/local/bin/python3 +enable_network: True # please make sure there is no network risk in your environment +allowed_syscalls: # please leave it empty if you have no idea how seccomp works +proxy: + socks5: '' + http: '' + https: '' diff --git a/dify/code/volumes/sandbox/conf/config.yaml.example b/dify/code/volumes/sandbox/conf/config.yaml.example new file mode 100644 index 0000000..f92c19e --- /dev/null +++ b/dify/code/volumes/sandbox/conf/config.yaml.example @@ -0,0 +1,35 @@ +app: + port: 8194 + debug: True + key: dify-sandbox +max_workers: 4 +max_requests: 50 +worker_timeout: 5 +python_path: /usr/local/bin/python3 +python_lib_path: + - /usr/local/lib/python3.10 + - /usr/lib/python3.10 + - /usr/lib/python3 + - /usr/lib/x86_64-linux-gnu + - /etc/ssl/certs/ca-certificates.crt + - /etc/nsswitch.conf + - /etc/hosts + - /etc/resolv.conf + - /run/systemd/resolve/stub-resolv.conf + - /run/resolvconf/resolv.conf + - /etc/localtime + - /usr/share/zoneinfo + - /etc/timezone + # add more paths if needed +python_pip_mirror_url: https://pypi.tuna.tsinghua.edu.cn/simple +nodejs_path: /usr/local/bin/node +enable_network: True +allowed_syscalls: + - 1 + - 2 + - 3 + # add all the syscalls which you require +proxy: + socks5: '' + http: '' + https: '' diff --git a/plane/code/.env.example b/plane/code/.env.example index 41e78e3..b5221c7 100644 --- a/plane/code/.env.example +++ b/plane/code/.env.example @@ -1,5 +1,5 @@ APP_DOMAIN=localhost -APP_RELEASE=v0.23.1 +APP_RELEASE=stable WEB_REPLICAS=1 SPACE_REPLICAS=1 @@ -12,6 +12,7 @@ DEBUG=0 SENTRY_DSN= SENTRY_ENVIRONMENT=production CORS_ALLOWED_ORIGINS=http://${APP_DOMAIN} +API_BASE_URL=http://api:8000 #DB SETTINGS PGHOST=plane-db @@ -28,6 +29,14 @@ REDIS_HOST=plane-redis REDIS_PORT=6379 REDIS_URL= +# RabbitMQ Settings +RABBITMQ_HOST=plane-mq +RABBITMQ_PORT=5672 +RABBITMQ_USER=plane +RABBITMQ_PASSWORD=plane +RABBITMQ_VHOST=plane +AMQP_URL= + # Secret Key SECRET_KEY=60gp0byfz2dvffa45cxl20p1scy9xbpf6d8c5y0geejgkyp1b5 @@ -49,15 +58,3 @@ GUNICORN_WORKERS=1 # UNCOMMENT `DOCKER_PLATFORM` IF YOU ARE ON `ARM64` AND DOCKER IMAGE IS NOT AVAILABLE FOR RESPECTIVE `APP_RELEASE` # DOCKER_PLATFORM=linux/amd64 -LIVE_BASE_URL="/live" - -# RabbitMQ Settings -RABBITMQ_HOST="plane-mq" -RABBITMQ_PORT="5672" -RABBITMQ_USER="plane" -RABBITMQ_PASSWORD="plane" -RABBITMQ_VHOST="plane" - - - - diff --git a/plane/code/docker-compose.yml b/plane/code/docker-compose.yml index 388c539..7724c46 100644 --- a/plane/code/docker-compose.yml +++ b/plane/code/docker-compose.yml @@ -3,9 +3,9 @@ x-app-env: &app-env - NGINX_PORT=${NGINX_PORT:-80} - WEB_URL=${WEB_URL:-http://localhost} - DEBUG=${DEBUG:-0} - - SENTRY_DSN=${SENTRY_DSN} + - SENTRY_DSN=${SENTRY_DSN:-""} - SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT:-"production"} - - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS} + - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS:-} # Gunicorn Workers - GUNICORN_WORKERS=${GUNICORN_WORKERS:-1} #DB SETTINGS @@ -21,11 +21,20 @@ x-app-env: &app-env - REDIS_HOST=${REDIS_HOST:-plane-redis} - REDIS_PORT=${REDIS_PORT:-6379} - REDIS_URL=${REDIS_URL:-redis://plane-redis:6379/} + + # RabbitMQ Settings + - RABBITMQ_HOST=${RABBITMQ_HOST:-plane-mq} + - RABBITMQ_PORT=${RABBITMQ_PORT:-5672} + - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER:-plane} + - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD:-plane} + - RABBITMQ_DEFAULT_VHOST=${RABBITMQ_VHOST:-plane} + - RABBITMQ_VHOST=${RABBITMQ_VHOST:-plane} + - AMQP_URL=${AMQP_URL:-amqp://plane:plane@plane-mq:5672/plane} # Application secret - SECRET_KEY=${SECRET_KEY:-60gp0byfz2dvffa45cxl20p1scy9xbpf6d8c5y0geejgkyp1b5} # DATA STORE SETTINGS - USE_MINIO=${USE_MINIO:-1} - - AWS_REGION=${AWS_REGION:-""} + - AWS_REGION=${AWS_REGION:-} - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-"access-key"} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-"secret-key"} - AWS_S3_ENDPOINT_URL=${AWS_S3_ENDPOINT_URL:-http://plane-minio:9000} @@ -34,26 +43,13 @@ x-app-env: &app-env - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-"secret-key"} - BUCKET_NAME=${BUCKET_NAME:-uploads} - FILE_SIZE_LIMIT=${FILE_SIZE_LIMIT:-5242880} - # Admin, Space and Live URLs - - ADMIN_BASE_URL=${ADMIN_BASE_URL} - - SPACE_BASE_URL=${SPACE_BASE_URL} - - APP_BASE_URL=${APP_BASE_URL} - - LIVE_BASE_PATH=${LIVE_BASE_URL} - # RabbitMQ Settings for RabbitMQ Image - - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER:-plane} - - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD:-plane} - - RABBITMQ_DEFAULT_VHOST=${RABBITMQ_VHOST:-plane} - # RabbitMQ Settings for Services - - RABBITMQ_HOST=${RABBITMQ_HOST:-plane-mq} - - RABBITMQ_PORT=${RABBITMQ_PORT:-5672} - - RABBITMQ_USER=${RABBITMQ_USER:-plane} - - RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD} - - RABBITMQ_VHOST=${RABBITMQ_VHOST:-plane} + # Live server env + - API_BASE_URL=${API_BASE_URL:-http://api:8000} services: web: <<: *app-env - image: ${DOCKERHUB_USER:-makeplane}/plane-frontend:${APP_RELEASE:-v0.23.1} + image: ${DOCKERHUB_USER:-makeplane}/plane-frontend:${APP_RELEASE:-stable} platform: ${DOCKER_PLATFORM:-} pull_policy: if_not_present restart: unless-stopped @@ -66,7 +62,7 @@ services: space: <<: *app-env - image: ${DOCKERHUB_USER:-makeplane}/plane-space:${APP_RELEASE:-v0.23.1} + image: ${DOCKERHUB_USER:-makeplane}/plane-space:${APP_RELEASE:-stable} platform: ${DOCKER_PLATFORM:-} pull_policy: if_not_present restart: unless-stopped @@ -80,7 +76,7 @@ services: admin: <<: *app-env - image: ${DOCKERHUB_USER:-makeplane}/plane-admin:${APP_RELEASE:-v0.23.1} + image: ${DOCKERHUB_USER:-makeplane}/plane-admin:${APP_RELEASE:-stable} platform: ${DOCKER_PLATFORM:-} pull_policy: if_not_present restart: unless-stopped @@ -91,9 +87,22 @@ services: - api - web + live: + <<: *app-env + image: ${DOCKERHUB_USER:-makeplane}/plane-live:${APP_RELEASE:-stable} + platform: ${DOCKER_PLATFORM:-} + pull_policy: if_not_present + restart: unless-stopped + command: node live/dist/server.js live + deploy: + replicas: ${LIVE_REPLICAS:-1} + depends_on: + - api + - web + api: <<: *app-env - image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-v0.23.1} + image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-stable} platform: ${DOCKER_PLATFORM:-} pull_policy: if_not_present restart: unless-stopped @@ -105,10 +114,11 @@ services: depends_on: - plane-db - plane-redis + - plane-mq worker: <<: *app-env - image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-v0.23.1} + image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-stable} platform: ${DOCKER_PLATFORM:-} pull_policy: if_not_present restart: unless-stopped @@ -119,10 +129,11 @@ services: - api - plane-db - plane-redis + - plane-mq beat-worker: <<: *app-env - image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-v0.23.1} + image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-stable} platform: ${DOCKER_PLATFORM:-} pull_policy: if_not_present restart: unless-stopped @@ -133,10 +144,11 @@ services: - api - plane-db - plane-redis + - plane-mq migrator: <<: *app-env - image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-v0.23.1} + image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-stable} platform: ${DOCKER_PLATFORM:-} pull_policy: if_not_present restart: "no" @@ -171,12 +183,6 @@ services: volumes: - rabbitmq_data:/var/lib/rabbitmq - live: - <<: *app-env - image: ${DOCKERHUB_USER:-makeplane}/plane-live:${APP_RELEASE:-v0.23.1} - restart: always - command: node live/dist/server.js - plane-minio: <<: *app-env image: minio/minio:latest @@ -189,7 +195,7 @@ services: # Comment this if you already have a reverse proxy running proxy: <<: *app-env - image: ${DOCKERHUB_USER:-makeplane}/plane-proxy:${APP_RELEASE:-v0.23.1} + image: ${DOCKERHUB_USER:-makeplane}/plane-proxy:${APP_RELEASE:-stable} platform: ${DOCKER_PLATFORM:-} pull_policy: if_not_present restart: unless-stopped @@ -197,11 +203,11 @@ services: - web - api - space - - admin volumes: pgdata: redisdata: + uploads: logs_api: logs_worker: diff --git a/plane/code/install.sh b/plane/code/install.sh index 6ad3b07..08cd4d9 100755 --- a/plane/code/install.sh +++ b/plane/code/install.sh @@ -442,10 +442,10 @@ function backupData() { local BACKUP_FOLDER=$PLANE_INSTALL_DIR/backup/$datetime mkdir -p "$BACKUP_FOLDER" - volumes=$(docker volume ls -f "name=plane-app" --format "{{.Name}}" | grep -E "_pgdata|_redisdata|_uploads") + volumes=$(docker volume ls -f "name=$SERVICE_FOLDER" --format "{{.Name}}" | grep -E "_pgdata|_redisdata|_uploads") # Check if there are any matching volumes if [ -z "$volumes" ]; then - echo "No volumes found starting with 'plane-app'" + echo "No volumes found starting with '$SERVICE_FOLDER'" exit 1 fi diff --git a/supabase/code/.env.example b/supabase/code/.env.example index 132fe8f..1e77372 100644 --- a/supabase/code/.env.example +++ b/supabase/code/.env.example @@ -19,6 +19,15 @@ POSTGRES_DB=postgres POSTGRES_PORT=5432 # default user is postgres +############ +# Supavisor -- Database pooler +############ +POOLER_PROXY_PORT_TRANSACTION=6543 +POOLER_DEFAULT_POOL_SIZE=20 +POOLER_MAX_CLIENT_CONN=100 +POOLER_TENANT_ID=your-tenant-id + + ############ # API Proxy - Configuration for the Kong Reverse proxy. ############ @@ -81,6 +90,9 @@ SUPABASE_PUBLIC_URL=http://localhost:8000 # Enable webp support IMGPROXY_ENABLE_WEBP_DETECTION=true +# Add your OpenAI API key to enable SQL Editor Assistant +OPENAI_API_KEY= + ############ # Functions - Configuration for Functions ############ diff --git a/supabase/code/docker-compose.s3.yml b/supabase/code/docker-compose.s3.yml index bb8be62..043691a 100644 --- a/supabase/code/docker-compose.s3.yml +++ b/supabase/code/docker-compose.s3.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: minio: @@ -33,7 +31,7 @@ services: storage: container_name: supabase-storage - image: supabase/storage-api:v0.43.11 + image: supabase/storage-api:v1.11.13 depends_on: db: # Disable this if you are using an external Postgres database diff --git a/supabase/code/docker-compose.yml b/supabase/code/docker-compose.yml index 0815e9a..c90de69 100644 --- a/supabase/code/docker-compose.yml +++ b/supabase/code/docker-compose.yml @@ -5,12 +5,10 @@ # Destroy: docker compose -f docker-compose.yml -f ./dev/docker-compose.dev.yml down -v --remove-orphans name: supabase -version: "3.8" services: studio: - # container_name: supabase-studio - image: supabase/studio:20240923-2e3e90c + image: supabase/studio:20241202-71e5240 restart: unless-stopped healthcheck: test: @@ -18,9 +16,10 @@ services: "CMD", "node", "-e", - "require('http').get('http://localhost:3000/api/profile', (r) => {if (r.statusCode !== 200) throw new Error(r.statusCode)})" + "fetch('http://studio:3000/api/profile').then((r) => {if (r.status + !== 200) throw new Error(r.status)})" ] - timeout: 5s + timeout: 10s interval: 5s retries: 3 depends_on: @@ -32,6 +31,7 @@ services: DEFAULT_ORGANIZATION_NAME: ${STUDIO_DEFAULT_ORGANIZATION} DEFAULT_PROJECT_NAME: ${STUDIO_DEFAULT_PROJECT} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} SUPABASE_URL: http://kong:8000 SUPABASE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} @@ -48,14 +48,11 @@ services: # NEXT_ANALYTICS_BACKEND_PROVIDER: bigquery kong: - # container_name: supabase-kong image: kong:2.8.1 restart: unless-stopped # https://unix.stackexchange.com/a/294837 - entrypoint: bash -c 'eval "echo \"$$(cat ~/temp.yml)\"" > ~/kong.yml && /docker-entrypoint.sh kong docker-start' - # ports: - # - ${KONG_HTTP_PORT}:8000/tcp - # - ${KONG_HTTPS_PORT}:8443/tcp + entrypoint: bash -c 'eval "echo \"$$(cat ~/temp.yml)\"" > ~/kong.yml && + /docker-entrypoint.sh kong docker-start' depends_on: analytics: condition: service_healthy @@ -76,8 +73,7 @@ services: - ./volumes/api/kong.yml:/home/kong/temp.yml:ro auth: - # container_name: supabase-auth - image: supabase/gotrue:v2.158.1 + image: supabase/gotrue:v2.164.0 depends_on: db: # Disable this if you are using an external Postgres database @@ -119,6 +115,10 @@ services: GOTRUE_EXTERNAL_EMAIL_ENABLED: ${ENABLE_EMAIL_SIGNUP} GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED: ${ENABLE_ANONYMOUS_USERS} GOTRUE_MAILER_AUTOCONFIRM: ${ENABLE_EMAIL_AUTOCONFIRM} + + # Uncomment to bypass nonce check in ID Token flow. Commonly set to true when using Google Sign In on mobile. + # GOTRUE_EXTERNAL_SKIP_NONCE_CHECK: true + # GOTRUE_MAILER_SECURE_EMAIL_CHANGE_ENABLED: true # GOTRUE_SMTP_MAX_FREQUENCY: 1s GOTRUE_SMTP_ADMIN_EMAIL: ${SMTP_ADMIN_EMAIL} @@ -159,7 +159,6 @@ services: rest: - # container_name: supabase-rest image: postgrest/postgrest:v12.2.0 depends_on: db: @@ -180,8 +179,7 @@ services: realtime: # This container name looks inconsistent but is correct because realtime constructs tenant id by parsing the subdomain - # container_name: realtime-dev.supabase-realtime - image: supabase/realtime:v2.30.34 + image: supabase/realtime:v2.33.58 depends_on: db: # Disable this if you are using an external Postgres database @@ -224,8 +222,7 @@ services: # To use S3 backed storage: docker compose -f docker-compose.yml -f docker-compose.s3.yml up storage: - # container_name: supabase-storage - image: supabase/storage-api:v1.10.1 + image: supabase/storage-api:v1.11.13 depends_on: db: # Disable this if you are using an external Postgres database @@ -242,7 +239,7 @@ services: "--no-verbose", "--tries=1", "--spider", - "http://localhost:5000/status" + "http://storage:5000/status" ] timeout: 5s interval: 5s @@ -267,7 +264,6 @@ services: - ./volumes/storage:/var/lib/storage:z imgproxy: - # container_name: supabase-imgproxy image: darthsim/imgproxy:v3.8.0 healthcheck: test: [ "CMD", "imgproxy", "health" ] @@ -283,8 +279,7 @@ services: - ./volumes/storage:/var/lib/storage:z meta: - # container_name: supabase-meta - image: supabase/postgres-meta:v0.83.2 + image: supabase/postgres-meta:v0.84.2 depends_on: db: # Disable this if you are using an external Postgres database @@ -301,8 +296,7 @@ services: PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} functions: - # container_name: supabase-edge-functions - image: supabase/edge-runtime:v1.58.3 + image: supabase/edge-runtime:v1.65.3 restart: unless-stopped depends_on: analytics: @@ -323,7 +317,6 @@ services: - /home/deno/functions/main analytics: - # container_name: supabase-analytics image: supabase/logflare:1.4.0 healthcheck: test: [ "CMD", "curl", "http://localhost:4000/health" ] @@ -344,7 +337,7 @@ services: environment: LOGFLARE_NODE_HOST: 127.0.0.1 DB_USERNAME: supabase_admin - DB_DATABASE: ${POSTGRES_DB} + DB_DATABASE: _supabase DB_HOSTNAME: ${POSTGRES_HOST} DB_PORT: ${POSTGRES_PORT} DB_PASSWORD: ${POSTGRES_PASSWORD} @@ -355,19 +348,16 @@ services: LOGFLARE_MIN_CLUSTER_SIZE: 1 # Comment variables to use Big Query backend for analytics - POSTGRES_BACKEND_URL: postgresql://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + POSTGRES_BACKEND_URL: postgresql://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase POSTGRES_BACKEND_SCHEMA: _analytics LOGFLARE_FEATURE_FLAG_OVERRIDE: multibackend=true # Uncomment to use Big Query backend for analytics # GOOGLE_PROJECT_ID: ${GOOGLE_PROJECT_ID} # GOOGLE_PROJECT_NUMBER: ${GOOGLE_PROJECT_NUMBER} - # ports: - # - 4000:4000 # Comment out everything below this point if you are using an external Postgres database db: - # container_name: supabase-db - image: supabase/postgres:15.1.1.78 + image: supabase/postgres:15.6.1.139 healthcheck: test: pg_isready -U postgres -h localhost interval: 5s @@ -383,9 +373,6 @@ services: - -c - log_min_messages=fatal # prevents Realtime polling queries from appearing in logs restart: unless-stopped - # ports: - # # Pass down internal port because it's set dynamically by other services - # - ${POSTGRES_PORT}:${POSTGRES_PORT} environment: POSTGRES_HOST: /var/run/postgresql PGPORT: ${POSTGRES_PORT} @@ -406,13 +393,16 @@ services: - ./volumes/db/jwt.sql:/docker-entrypoint-initdb.d/init-scripts/99-jwt.sql:Z # PGDATA directory is persisted between restarts - ./volumes/db/data:/var/lib/postgresql/data:Z + # Changes required for internal supabase data such as _analytics + - ./volumes/db/_supabase.sql:/docker-entrypoint-initdb.d/migrations/97-_supabase.sql:Z # Changes required for Analytics support - ./volumes/db/logs.sql:/docker-entrypoint-initdb.d/migrations/99-logs.sql:Z + # Changes required for Pooler support + - ./volumes/db/pooler.sql:/docker-entrypoint-initdb.d/migrations/99-pooler.sql:Z # Use named volume to persist pgsodium decryption key between restarts - db-config:/etc/postgresql-custom vector: - # container_name: supabase-vector image: timberio/vector:0.28.1-alpine healthcheck: test: @@ -433,7 +423,46 @@ services: - ${DOCKER_SOCKET_LOCATION}:/var/run/docker.sock:ro environment: LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} - command: [ "--config", "etc/vector/vector.yml" ] + command: [ "--config", "/etc/vector/vector.yml" ] + + # Update the DATABASE_URL if you are using an external Postgres database + supavisor: + image: supabase/supavisor:1.1.56 + healthcheck: + test: curl -sSfL --head -o /dev/null "http://127.0.0.1:4000/api/health" + interval: 10s + timeout: 5s + retries: 5 + depends_on: + db: + condition: service_healthy + analytics: + condition: service_healthy + command: + - /bin/sh + - -c + - /app/bin/migrate && /app/bin/supavisor eval "$$(cat + /etc/pooler/pooler.exs)" && /app/bin/server + restart: unless-stopped + environment: + - PORT=4000 + - POSTGRES_PORT=${POSTGRES_PORT} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - DATABASE_URL=ecto://supabase_admin:${POSTGRES_PASSWORD}@db:${POSTGRES_PORT}/_supabase + - CLUSTER_POSTGRES=true + - SECRET_KEY_BASE=UpNVntn3cDxHJpq99YMc1T1AQgQpc8kfYTuRgBiYa15BLrx8etQoXz3gZv1/u2oq + - VAULT_ENC_KEY=your-encryption-key-32-chars-min + - API_JWT_SECRET=${JWT_SECRET} + - METRICS_JWT_SECRET=${JWT_SECRET} + - REGION=local + - ERL_AFLAGS=-proto_dist inet_tcp + - POOLER_TENANT_ID=${POOLER_TENANT_ID} + - POOLER_DEFAULT_POOL_SIZE=${POOLER_DEFAULT_POOL_SIZE} + - POOLER_MAX_CLIENT_CONN=${POOLER_MAX_CLIENT_CONN} + - POOLER_POOL_MODE=transaction + volumes: + - ./volumes/pooler/pooler.exs:/etc/pooler/pooler.exs:ro volumes: db-config: diff --git a/supabase/code/volumes/db/_supabase.sql b/supabase/code/volumes/db/_supabase.sql new file mode 100644 index 0000000..6236ae1 --- /dev/null +++ b/supabase/code/volumes/db/_supabase.sql @@ -0,0 +1,3 @@ +\set pguser `echo "$POSTGRES_USER"` + +CREATE DATABASE _supabase WITH OWNER :pguser; diff --git a/supabase/code/volumes/db/logs.sql b/supabase/code/volumes/db/logs.sql index 22fc247..255c0f4 100644 --- a/supabase/code/volumes/db/logs.sql +++ b/supabase/code/volumes/db/logs.sql @@ -1,4 +1,6 @@ \set pguser `echo "$POSTGRES_USER"` +\c _supabase create schema if not exists _analytics; alter schema _analytics owner to :pguser; +\c postgres diff --git a/supabase/code/volumes/db/pooler.sql b/supabase/code/volumes/db/pooler.sql new file mode 100644 index 0000000..162c5b9 --- /dev/null +++ b/supabase/code/volumes/db/pooler.sql @@ -0,0 +1,6 @@ +\set pguser `echo "$POSTGRES_USER"` + +\c _supabase +create schema if not exists _supavisor; +alter schema _supavisor owner to :pguser; +\c postgres diff --git a/supabase/code/volumes/pooler/pooler.exs b/supabase/code/volumes/pooler/pooler.exs new file mode 100644 index 0000000..791d61c --- /dev/null +++ b/supabase/code/volumes/pooler/pooler.exs @@ -0,0 +1,30 @@ +{:ok, _} = Application.ensure_all_started(:supavisor) + +{:ok, version} = + case Supavisor.Repo.query!("select version()") do + %{rows: [[ver]]} -> Supavisor.Helpers.parse_pg_version(ver) + _ -> nil + end + +params = %{ + "external_id" => System.get_env("POOLER_TENANT_ID"), + "db_host" => "db", + "db_port" => System.get_env("POSTGRES_PORT"), + "db_database" => System.get_env("POSTGRES_DB"), + "require_user" => false, + "auth_query" => "SELECT * FROM pgbouncer.get_auth($1)", + "default_max_clients" => System.get_env("POOLER_MAX_CLIENT_CONN"), + "default_pool_size" => System.get_env("POOLER_DEFAULT_POOL_SIZE"), + "default_parameter_status" => %{"server_version" => version}, + "users" => [%{ + "db_user" => "pgbouncer", + "db_password" => System.get_env("POSTGRES_PASSWORD"), + "mode_type" => System.get_env("POOLER_POOL_MODE"), + "pool_size" => System.get_env("POOLER_DEFAULT_POOL_SIZE"), + "is_manager" => true + }] +} + +if !Supavisor.Tenants.get_tenant_by_external_id(params["external_id"]) do + {:ok, _} = Supavisor.Tenants.create_tenant(params) +end diff --git a/twenty/code/.env.example b/twenty/code/.env.example index afc3806..5a3ab70 100644 --- a/twenty/code/.env.example +++ b/twenty/code/.env.example @@ -1,32 +1,20 @@ -TAG=v0.32.3 +TAG=latest -POSTGRES_ADMIN_PASSWORD=replace_me_with_a_strong_password +#PGUSER_SUPERUSER=postgres +#PGPASSWORD_SUPERUSER=replace_me_with_a_strong_password PG_DATABASE_HOST=db:5432 REDIS_URL=redis://redis:6379 -SERVER_URL=https://$(PRIMARY_DOMAIN) +SERVER_URL=http://localhost:3000 -# Use openssl rand -base64 32 for each secret - Please change these values -REFRESH_TOKEN_SECRET=IqPjqg9jCkrwp2KLS8k8mYRaOhvGtw3xmYF2dAkrkD0= -ACCESS_TOKEN_SECRET=IqPjqg9jCkrwp2KLS8k8mYRaOhvGtw3xmYF2dAkrkD0= -LOGIN_TOKEN_SECRET=IqPjqg9jCkrwp2KLS8k8mYRaOhvGtw3xmYF2dAkrkD0= -APP_SECRET=IqPjqg9jCkrwp2KLS8k8mYRaOhvGtw3xmYF2dAkrkD0= +# Use openssl rand -base64 32 for each secret +# APP_SECRET=replace_me_with_a_random_string SIGN_IN_PREFILLED=true STORAGE_TYPE=local -#S3 STORAGE # STORAGE_S3_REGION=eu-west3 # STORAGE_S3_NAME=my-bucket # STORAGE_S3_ENDPOINT= - - -#GOOGLE AUTH -# AUTH_GOOGLE_ENABLED=true -# AUTH_GOOGLE_CLIENT_ID= -# AUTH_GOOGLE_CLIENT_SECRET= -# AUTH_GOOGLE_CALLBACK_URL= -# AUTH_GOOGLE_APIS_CALLBACK_URL= - \ No newline at end of file diff --git a/twenty/code/Makefile b/twenty/code/Makefile index deee953..2d89c4c 100644 --- a/twenty/code/Makefile +++ b/twenty/code/Makefile @@ -1,37 +1,9 @@ -dev-build: - @docker compose -f dev/docker-compose.yml down -v - @docker compose -f dev/docker-compose.yml build - -dev-up: - @docker compose -f dev/docker-compose.yml up -d - -dev-start: - @docker compose -f dev/docker-compose.yml start - -dev-stop: - @docker compose -f dev/docker-compose.yml stop - -dev-down: - @docker compose -f dev/docker-compose.yml down -v - -dev-sh: - @docker compose -f dev/docker-compose.yml exec twenty-dev sh - -dev-postgres-build: - @docker stop twenty_postgres || true - @docker rm twenty_postgres || true - @docker volume rm twenty_db_data || true - @docker compose -f dev/docker-compose.yml up --build postgres -d - prod-build: @cd ../.. && docker build -f ./packages/twenty-docker/twenty/Dockerfile --tag twenty . && cd - prod-run: @docker run -d -p 3000:3000 --name twenty twenty -prod-postgres-build: - @cd ../.. && docker build -f ./packages/twenty-docker/twenty-postgres/Dockerfile --tag twenty-postgres . && cd - - prod-postgres-run: @docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres --name twenty-postgres twenty-postgres @@ -40,11 +12,3 @@ prod-website-build: prod-website-run: @docker run -d -p 3000:3000 --name twenty-website twenty-website - -release-postgres: - @cd ../.. && docker buildx build \ - --push \ - --no-cache \ - --platform linux/amd64,linux/arm64 \ - -f ./packages/twenty-docker/twenty-postgres/Dockerfile -t twentycrm/twenty-postgres:$(version) -t twentycrm/twenty-postgres:latest . \ - && cd - diff --git a/twenty/code/docker-compose.yml b/twenty/code/docker-compose.yml index 8b2c806..f4164fd 100644 --- a/twenty/code/docker-compose.yml +++ b/twenty/code/docker-compose.yml @@ -9,24 +9,21 @@ services: - server-local-data:/tmp/server-local-data - docker-data:/tmp/docker-data command: > - bash -c " - chown -R 1000:1000 /tmp/server-local-data - && chown -R 1000:1000 /tmp/docker-data" + bash -c " chown -R 1000:1000 /tmp/server-local-data && chown -R 1000:1000 + /tmp/docker-data" server: image: twentycrm/twenty:${TAG} volumes: - server-local-data:/app/packages/twenty-server/${STORAGE_LOCAL_PATH:-.local-storage} - docker-data:/app/docker-data - ports: - - "3000" environment: PORT: 3000 - PG_DATABASE_URL: postgres://twenty:twenty@${PG_DATABASE_HOST}/default + PG_DATABASE_URL: postgres://${PGUSER_SUPERUSER:-postgres}:${PGPASSWORD_SUPERUSER:-postgres}@${PG_DATABASE_HOST:-db:5432}/default SERVER_URL: ${SERVER_URL} FRONT_BASE_URL: ${FRONT_BASE_URL:-$SERVER_URL} - REDIS_URL: ${REDIS_URL:-redis://localhost:6379} - REDIS_HOST: ${REDIS_HOST} + REDIS_URL: ${REDIS_URL:-redis://redis:6379} + ENABLE_DB_MIGRATIONS: "true" SIGN_IN_PREFILLED: ${SIGN_IN_PREFILLED} @@ -34,12 +31,8 @@ services: STORAGE_S3_REGION: ${STORAGE_S3_REGION} STORAGE_S3_NAME: ${STORAGE_S3_NAME} STORAGE_S3_ENDPOINT: ${STORAGE_S3_ENDPOINT} - - ACCESS_TOKEN_SECRET: ${ACCESS_TOKEN_SECRET} - LOGIN_TOKEN_SECRET: ${LOGIN_TOKEN_SECRET} - REFRESH_TOKEN_SECRET: ${REFRESH_TOKEN_SECRET} + APP_SECRET: ${APP_SECRET} - depends_on: change-vol-ownership: condition: service_completed_successfully @@ -54,13 +47,13 @@ services: worker: image: twentycrm/twenty:${TAG} - command: ["yarn", "worker:prod"] + command: [ "yarn", "worker:prod" ] environment: - PG_DATABASE_URL: postgres://twenty:twenty@${PG_DATABASE_HOST}/default + PG_DATABASE_URL: postgres://${PGUSER_SUPERUSER:-postgres}:${PGPASSWORD_SUPERUSER:-postgres}@${PG_DATABASE_HOST:-db:5432}/default SERVER_URL: ${SERVER_URL} FRONT_BASE_URL: ${FRONT_BASE_URL:-$SERVER_URL} - REDIS_URL: ${REDIS_URL:-redis://localhost:6379} - REDIS_HOST: redis + REDIS_URL: ${REDIS_URL:-redis://redis:6379} + ENABLE_DB_MIGRATIONS: "false" # it already runs on the server STORAGE_TYPE: ${STORAGE_TYPE} @@ -68,11 +61,7 @@ services: STORAGE_S3_NAME: ${STORAGE_S3_NAME} STORAGE_S3_ENDPOINT: ${STORAGE_S3_ENDPOINT} - ACCESS_TOKEN_SECRET: ${ACCESS_TOKEN_SECRET} - LOGIN_TOKEN_SECRET: ${LOGIN_TOKEN_SECRET} - REFRESH_TOKEN_SECRET: ${REFRESH_TOKEN_SECRET} APP_SECRET: ${APP_SECRET} - depends_on: db: condition: service_healthy @@ -81,13 +70,16 @@ services: restart: always db: - image: twentycrm/twenty-postgres:${TAG} + image: twentycrm/twenty-postgres-spilo:${TAG} volumes: - - db-data:/bitnami/postgresql + - db-data:/home/postgres/pgdata environment: - POSTGRES_PASSWORD: ${POSTGRES_ADMIN_PASSWORD} + PGUSER_SUPERUSER: ${PGUSER_SUPERUSER:-postgres} + PGPASSWORD_SUPERUSER: ${PGPASSWORD_SUPERUSER:-postgres} + ALLOW_NOSSL: "true" + SPILO_PROVIDER: "local" healthcheck: - test: pg_isready -U twenty -d default + test: pg_isready -U ${PGUSER_SUPERUSER:-postgres} -h localhost -d postgres interval: 5s timeout: 5s retries: 10 @@ -95,11 +87,9 @@ services: redis: image: redis - ports: - - "6379:6379" restart: always volumes: docker-data: db-data: - server-local-data: \ No newline at end of file + server-local-data: diff --git a/twenty/code/k8s/manifests/deployment-db.yaml b/twenty/code/k8s/manifests/deployment-db.yaml index 2e31737..02a97c3 100644 --- a/twenty/code/k8s/manifests/deployment-db.yaml +++ b/twenty/code/k8s/manifests/deployment-db.yaml @@ -22,33 +22,33 @@ spec: app: twentycrm-db spec: volumes: - - name: twentycrm-db-data - persistentVolumeClaim: - claimName: twentycrm-db-pvc + - name: twentycrm-db-data + persistentVolumeClaim: + claimName: twentycrm-db-pvc containers: - - env: - - name: POSTGRES_PASSWORD - value: "twenty" - - name: BITNAMI_DEBUG - value: "true" - - image: twentycrm/twenty-postgres:latest - imagePullPolicy: Always - name: twentycrm - ports: - - containerPort: 5432 - name: tcp - protocol: TCP - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "1024Mi" - cpu: "1000m" - stdin: true - tty: true - volumeMounts: - - mountPath: /bitnami/postgresql - name: twentycrm-db-data + - name: twentycrm + image: twentycrm/twenty-postgres:latest + imagePullPolicy: Always + env: + - name: PGUSER_SUPERUSER + value: "postgres" + - name: PGPASSWORD_SUPERUSER + value: "postgres" + ports: + - containerPort: 5432 + name: tcp + protocol: TCP + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "1024Mi" + cpu: "1000m" + stdin: true + tty: true + volumeMounts: + - mountPath: /home/postgres/pgdata + name: twentycrm-db-data dnsPolicy: ClusterFirst restartPolicy: Always diff --git a/twenty/code/k8s/manifests/deployment-redis.yaml b/twenty/code/k8s/manifests/deployment-redis.yaml new file mode 100644 index 0000000..e09874a --- /dev/null +++ b/twenty/code/k8s/manifests/deployment-redis.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: twentycrm-redis + name: twentycrm-redis + namespace: twentycrm +spec: + progressDeadlineSeconds: 600 + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: twentycrm-redis + template: + metadata: + labels: + app: twentycrm-redis + spec: + containers: + - name: redis + image: redis/redis-stack-server:latest + imagePullPolicy: Always + env: + - name: PORT + value: 6379 + ports: + - containerPort: 6379 + name: redis + protocol: TCP + resources: + requests: + memory: "1024Mi" + cpu: "250m" + limits: + memory: "2048Mi" + cpu: "500m" + + dnsPolicy: ClusterFirst + restartPolicy: Always diff --git a/twenty/code/k8s/manifests/deployment-server.yaml b/twenty/code/k8s/manifests/deployment-server.yaml index b4596e9..5c79044 100644 --- a/twenty/code/k8s/manifests/deployment-server.yaml +++ b/twenty/code/k8s/manifests/deployment-server.yaml @@ -22,67 +22,61 @@ spec: app: twentycrm-server spec: volumes: - - name: twentycrm-server-data - persistentVolumeClaim: - claimName: twentycrm-server-pvc + - name: twentycrm-server-data + persistentVolumeClaim: + claimName: twentycrm-server-pvc + - name: twentycrm-docker-data + persistentVolumeClaim: + claimName: twentycrm-docker-data-pvc containers: - - env: - - name: PORT - value: 3000 - - name: SERVER_URL - value: "https://crm.example.com:443" - - name: FRONT_BASE_URL - value: "https://crm.example.com:443" - - name: PG_DATABASE_URL - value: "postgres://twenty:twenty@twenty-db.twentycrm.svc.cluster.local/default" - - name: ENABLE_DB_MIGRATIONS - value: "true" - - name: SIGN_IN_PREFILLED - value: "true" - - name: STORAGE_TYPE - value: "local" - - name: "MESSAGE_QUEUE_TYPE" - value: "pg-boss" - - name: ACCESS_TOKEN_SECRET - valueFrom: - secretKeyRef: - name: tokens - key: accessToken - - name: LOGIN_TOKEN_SECRET - valueFrom: - secretKeyRef: - name: tokens - key: loginToken - - name: REFRESH_TOKEN_SECRET - valueFrom: - secretKeyRef: - name: tokens - key: refreshToken - - name: FILE_TOKEN_SECRET - valueFrom: - secretKeyRef: - name: tokens - key: fileToken - - image: twentycrm/twenty:latest - imagePullPolicy: Always - name: twentycrm - ports: - - containerPort: 3000 - name: http-tcp - protocol: TCP - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "1024Mi" - cpu: "1000m" - stdin: true - tty: true - volumeMounts: - - mountPath: /app/docker-data - name: twentycrm-server-data - - mountPath: /app/.local-storage - name: twentycrm-server-data + - name: twentycrm + image: twentycrm/twenty:latest + imagePullPolicy: Always + env: + - name: PORT + value: 3000 + - name: SERVER_URL + value: "https://crm.example.com:443" + - name: FRONT_BASE_URL + value: "https://crm.example.com:443" + - name: "PG_DATABASE_URL" + value: "postgres://postgres:postgres@twenty-db.twentycrm.svc.cluster.local/default" + - name: "REDIS_URL" + value: "redis://twentycrm-redis.twentycrm.svc.cluster.local:6379" + - name: ENABLE_DB_MIGRATIONS + value: "true" + - name: SIGN_IN_PREFILLED + value: "true" + - name: STORAGE_TYPE + value: "local" + - name: "MESSAGE_QUEUE_TYPE" + value: "bull-mq" + - name: "ACCESS_TOKEN_EXPIRES_IN" + value: "7d" + - name: "LOGIN_TOKEN_EXPIRES_IN" + value: "1h" + - name: APP_SECRET + valueFrom: + secretKeyRef: + name: tokens + key: accessToken + ports: + - containerPort: 3000 + name: http-tcp + protocol: TCP + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "1024Mi" + cpu: "1000m" + stdin: true + tty: true + volumeMounts: + - mountPath: /app/docker-data + name: twentycrm-docker-data + - mountPath: /app/packages/twenty-server/.local-storage + name: twentycrm-server-data dnsPolicy: ClusterFirst restartPolicy: Always diff --git a/twenty/code/k8s/manifests/deployment-worker.yaml b/twenty/code/k8s/manifests/deployment-worker.yaml index b3834c4..a92ef25 100644 --- a/twenty/code/k8s/manifests/deployment-worker.yaml +++ b/twenty/code/k8s/manifests/deployment-worker.yaml @@ -21,58 +21,43 @@ spec: labels: app: twentycrm-worker spec: - volumes: - - name: twentycrm-worker-data - persistentVolumeClaim: - claimName: twentycrm-worker-pvc containers: - - env: - - name: SERVER_URL - value: "https://crm.example.com:443" - - name: FRONT_BASE_URL - value: "https://crm.example.com:443" - - name: PG_DATABASE_URL - value: "postgres://twenty:twenty@twenty-db.twentycrm.svc.cluster.local/default" - - name: ENABLE_DB_MIGRATIONS - value: "false" # it already runs on the server - - name: STORAGE_TYPE - value: "local" - - name: "MESSAGE_QUEUE_TYPE" - value: "pg-boss" - - name: ACCESS_TOKEN_SECRET - valueFrom: - secretKeyRef: - name: tokens - key: accessToken - - name: LOGIN_TOKEN_SECRET - valueFrom: - secretKeyRef: - name: tokens - key: loginToken - - name: REFRESH_TOKEN_SECRET - valueFrom: - secretKeyRef: - name: tokens - key: refreshToken - - name: FILE_TOKEN_SECRET - valueFrom: - secretKeyRef: - name: tokens - key: fileToken - - image: twentycrm/twenty:latest - imagePullPolicy: Always - name: twentycrm - command: - - yarn - - worker:prod - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "1024Mi" - cpu: "1000m" - stdin: true - tty: true + - name: twentycrm + image: twentycrm/twenty:latest + imagePullPolicy: Always + env: + - name: SERVER_URL + value: "https://crm.example.com:443" + - name: FRONT_BASE_URL + value: "https://crm.example.com:443" + - name: PG_DATABASE_URL + value: "postgres://postgres:postgres@twenty-db.twentycrm.svc.cluster.local/default" + - name: ENABLE_DB_MIGRATIONS + value: "false" # it already runs on the server + - name: STORAGE_TYPE + value: "local" + - name: "MESSAGE_QUEUE_TYPE" + value: "bull-mq" + - name: "CACHE_STORAGE_TYPE" + value: "redis" + - name: "REDIS_URL" + value: "redis://twentycrm-redis.twentycrm.svc.cluster.local:6379" + - name: APP_SECRET + valueFrom: + secretKeyRef: + name: tokens + key: accessToken + command: + - yarn + - worker:prod + resources: + requests: + memory: "1024Mi" + cpu: "250m" + limits: + memory: "2048Mi" + cpu: "1000m" + stdin: true + tty: true dnsPolicy: ClusterFirst restartPolicy: Always diff --git a/twenty/code/k8s/manifests/ingress.yaml b/twenty/code/k8s/manifests/ingress.yaml index b334aac..0bbae11 100644 --- a/twenty/code/k8s/manifests/ingress.yaml +++ b/twenty/code/k8s/manifests/ingress.yaml @@ -4,21 +4,21 @@ metadata: name: twentycrm namespace: twentycrm annotations: - nginx.ingress.kubernetes.io/configuration-snippet: | + nginx.ingress.kubernetes.io/configuration-snippet: | more_set_headers "X-Forwarded-For $http_x_forwarded_for"; - nginx.ingress.kubernetes.io/force-ssl-redirect: "false" - kubernetes.io/ingress.class: "nginx" - nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" spec: ingressClassName: nginx rules: - - host: crm.example.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: twentycrm-server - port: - name: http-tcp + - host: crm.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: twentycrm-server + port: + name: http-tcp diff --git a/twenty/code/k8s/manifests/pv-docker-data.yaml b/twenty/code/k8s/manifests/pv-docker-data.yaml new file mode 100644 index 0000000..95fc52a --- /dev/null +++ b/twenty/code/k8s/manifests/pv-docker-data.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: twentycrm-docker-data-pv +spec: + storageClassName: default + capacity: + storage: 100Mi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain diff --git a/twenty/code/k8s/manifests/pvc-docker-data.yaml b/twenty/code/k8s/manifests/pvc-docker-data.yaml new file mode 100644 index 0000000..12dd071 --- /dev/null +++ b/twenty/code/k8s/manifests/pvc-docker-data.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: twentycrm-docker-data-pvc + namespace: twentycrm +spec: + storageClassName: default + volumeName: twentycrm-docker-data-pv + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi diff --git a/twenty/code/k8s/manifests/service-db.yaml b/twenty/code/k8s/manifests/service-db.yaml index bb0e38d..89dbd14 100644 --- a/twenty/code/k8s/manifests/service-db.yaml +++ b/twenty/code/k8s/manifests/service-db.yaml @@ -6,9 +6,9 @@ metadata: spec: internalTrafficPolicy: Cluster ports: - - port: 5432 - protocol: TCP - targetPort: 5432 + - port: 5432 + protocol: TCP + targetPort: 5432 selector: app: twentycrm-db sessionAffinity: ClientIP diff --git a/twenty/code/k8s/manifests/service-redis.yaml b/twenty/code/k8s/manifests/service-redis.yaml new file mode 100644 index 0000000..49f5088 --- /dev/null +++ b/twenty/code/k8s/manifests/service-redis.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: twentycrm-redis + namespace: twentycrm +spec: + internalTrafficPolicy: Cluster + ports: + - port: 6379 + protocol: TCP + targetPort: 6379 + selector: + app: twentycrm-redis + sessionAffinity: ClientIP + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + type: ClusterIP diff --git a/twenty/code/k8s/manifests/service-server.yaml b/twenty/code/k8s/manifests/service-server.yaml index 7fcc869..b45b28f 100644 --- a/twenty/code/k8s/manifests/service-server.yaml +++ b/twenty/code/k8s/manifests/service-server.yaml @@ -6,10 +6,10 @@ metadata: spec: internalTrafficPolicy: Cluster ports: - - name: http-tcp - port: 3000 - protocol: TCP - targetPort: 3000 + - name: http-tcp + port: 3000 + protocol: TCP + targetPort: 3000 selector: app: twentycrm-server sessionAffinity: ClientIP diff --git a/twenty/code/k8s/terraform/.terraform-docs.yml b/twenty/code/k8s/terraform/.terraform-docs.yml index 0077816..792c543 100644 --- a/twenty/code/k8s/terraform/.terraform-docs.yml +++ b/twenty/code/k8s/terraform/.terraform-docs.yml @@ -15,12 +15,12 @@ output: # TwentyCRM Terraform Docs - This file was generated by [terraform-docs](https://terraform-docs.io/), for more information on how to install, configure and use visit their website. + This file was generated by [terraform-docs](https://terraform-docs.io/), for more information on how to install, configure, and use visit their website. - To update this `README.md` after changes to the Terraform code in this folder, run: `terraform-docs .` + To update this `README.md` after changes to the Terraform code in this folder, run: `terraform-docs -c `./.terraform-docs.yml .` To make configuration changes to how this doc is generated, see `./.terraform-docs.yml` - + {{ .Content }} @@ -45,4 +45,4 @@ settings: read-comments: true required: true sensitive: true - type: true \ No newline at end of file + type: true diff --git a/twenty/code/k8s/terraform/README.md b/twenty/code/k8s/terraform/README.md index 10a7ab5..32facfd 100644 --- a/twenty/code/k8s/terraform/README.md +++ b/twenty/code/k8s/terraform/README.md @@ -1,9 +1,9 @@ # TwentyCRM Terraform Docs -This file was generated by [terraform-docs](https://terraform-docs.io/), for more information on how to install, configure and use visit their website. +This file was generated by [terraform-docs](https://terraform-docs.io/), for more information on how to install, configure, and use visit their website. -To update this `README.md` after changes to the Terraform code in this folder, run: `terraform-docs .` +To update this `README.md` after changes to the Terraform code in this folder, run: `terraform-docs -c `./.terraform-docs.yml .` To make configuration changes to how this doc is generated, see `./.terraform-docs.yml` @@ -12,30 +12,37 @@ To make configuration changes to how this doc is generated, see `./.terraform-do | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.9.2 | -| [kubernetes](#requirement\_kubernetes) | >= 2.31.0 | +| [kubernetes](#requirement\_kubernetes) | >= 2.32.0 | +| [random](#requirement\_random) | >= 3.6.3 | ## Providers | Name | Version | |------|---------| -| [kubernetes](#provider\_kubernetes) | >= 2.31.0 | +| [kubernetes](#provider\_kubernetes) | >= 2.32.0 | +| [random](#provider\_random) | >= 3.6.3 | ## Resources | Name | Type | |------|------| | [kubernetes_deployment.twentycrm_db](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/deployment) | resource | +| [kubernetes_deployment.twentycrm_redis](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/deployment) | resource | | [kubernetes_deployment.twentycrm_server](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/deployment) | resource | | [kubernetes_deployment.twentycrm_worker](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/deployment) | resource | | [kubernetes_ingress.twentycrm](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/ingress) | resource | | [kubernetes_namespace.twentycrm](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | | [kubernetes_persistent_volume.db](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume) | resource | +| [kubernetes_persistent_volume.docker_data](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume) | resource | | [kubernetes_persistent_volume.server](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume) | resource | | [kubernetes_persistent_volume_claim.db](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume_claim) | resource | +| [kubernetes_persistent_volume_claim.docker_data](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume_claim) | resource | | [kubernetes_persistent_volume_claim.server](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume_claim) | resource | | [kubernetes_secret.twentycrm_tokens](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource | | [kubernetes_service.twentycrm_db](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service) | resource | +| [kubernetes_service.twentycrm_redis](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service) | resource | | [kubernetes_service.twentycrm_server](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service) | resource | +| [random_bytes.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/bytes) | resource | ## Inputs @@ -43,22 +50,24 @@ To make configuration changes to how this doc is generated, see `./.terraform-do |------|-------------|------|---------|:--------:| | [twentycrm\_app\_hostname](#input\_twentycrm\_app\_hostname) | The protocol, DNS fully qualified hostname, and port used to access TwentyCRM in your environment. Ex: https://crm.example.com:443 | `string` | n/a | yes | | [twentycrm\_pgdb\_admin\_password](#input\_twentycrm\_pgdb\_admin\_password) | TwentyCRM password for postgres database. | `string` | n/a | yes | -| [twentycrm\_token\_accessToken](#input\_twentycrm\_token\_accessToken) | TwentyCRM access Token | `string` | n/a | yes | -| [twentycrm\_token\_fileToken](#input\_twentycrm\_token\_fileToken) | TwentyCRM file Token | `string` | n/a | yes | -| [twentycrm\_token\_loginToken](#input\_twentycrm\_token\_loginToken) | TwentyCRM login Token | `string` | n/a | yes | -| [twentycrm\_token\_refreshToken](#input\_twentycrm\_token\_refreshToken) | TwentyCRM refresh Token | `string` | n/a | yes | | [twentycrm\_app\_name](#input\_twentycrm\_app\_name) | A friendly name prefix to use for every component deployed. | `string` | `"twentycrm"` | no | -| [twentycrm\_db\_image](#input\_twentycrm\_db\_image) | TwentyCRM image for database deployment. This defaults to latest. | `string` | `"twentycrm/twenty-postgres:latest"` | no | +| [twentycrm\_db\_image](#input\_twentycrm\_db\_image) | TwentyCRM image for database deployment. This defaults to latest. | `string` | `"twentycrm/twenty-postgres-spilo:latest"` | no | | [twentycrm\_db\_pv\_capacity](#input\_twentycrm\_db\_pv\_capacity) | Storage capacity provisioned for database persistent volume. | `string` | `"10Gi"` | no | | [twentycrm\_db\_pv\_path](#input\_twentycrm\_db\_pv\_path) | Local path to use to store the physical volume if using local storage on nodes. | `string` | `""` | no | | [twentycrm\_db\_pvc\_requests](#input\_twentycrm\_db\_pvc\_requests) | Storage capacity reservation for database persistent volume claim. | `string` | `"10Gi"` | no | | [twentycrm\_db\_replicas](#input\_twentycrm\_db\_replicas) | Number of replicas for the TwentyCRM database deployment. This defaults to 1. | `number` | `1` | no | +| [twentycrm\_docker\_data\_mount\_path](#input\_twentycrm\_docker\_data\_mount\_path) | TwentyCRM mount path for servers application data. Defaults to '/app/docker-data'. | `string` | `"/app/docker-data"` | no | +| [twentycrm\_docker\_data\_pv\_capacity](#input\_twentycrm\_docker\_data\_pv\_capacity) | Storage capacity provisioned for server persistent volume. | `string` | `"10Gi"` | no | +| [twentycrm\_docker\_data\_pv\_path](#input\_twentycrm\_docker\_data\_pv\_path) | Local path to use to store the physical volume if using local storage on nodes. | `string` | `""` | no | +| [twentycrm\_docker\_data\_pvc\_requests](#input\_twentycrm\_docker\_data\_pvc\_requests) | Storage capacity reservation for server persistent volume claim. | `string` | `"10Gi"` | no | | [twentycrm\_namespace](#input\_twentycrm\_namespace) | Namespace for all TwentyCRM resources | `string` | `"twentycrm"` | no | -| [twentycrm\_server\_data\_mount\_path](#input\_twentycrm\_server\_data\_mount\_path) | TwentyCRM mount path for servers application data. Defaults to '/app/docker-data'. | `string` | `"/app/docker-data"` | no | +| [twentycrm\_redis\_image](#input\_twentycrm\_redis\_image) | TwentyCRM image for Redis deployment. This defaults to latest. | `string` | `"redis/redis-stack-server:latest"` | no | +| [twentycrm\_redis\_replicas](#input\_twentycrm\_redis\_replicas) | Number of replicas for the TwentyCRM Redis deployment. This defaults to 1. | `number` | `1` | no | +| [twentycrm\_server\_data\_mount\_path](#input\_twentycrm\_server\_data\_mount\_path) | TwentyCRM mount path for servers application data. Defaults to '/app/packages/twenty-server/.local-storage'. | `string` | `"/app/packages/twenty-server/.local-storage"` | no | | [twentycrm\_server\_image](#input\_twentycrm\_server\_image) | TwentyCRM server image for the server deployment. This defaults to latest. This value is also used for the workers image. | `string` | `"twentycrm/twenty:latest"` | no | | [twentycrm\_server\_pv\_capacity](#input\_twentycrm\_server\_pv\_capacity) | Storage capacity provisioned for server persistent volume. | `string` | `"10Gi"` | no | | [twentycrm\_server\_pv\_path](#input\_twentycrm\_server\_pv\_path) | Local path to use to store the physical volume if using local storage on nodes. | `string` | `""` | no | | [twentycrm\_server\_pvc\_requests](#input\_twentycrm\_server\_pvc\_requests) | Storage capacity reservation for server persistent volume claim. | `string` | `"10Gi"` | no | | [twentycrm\_server\_replicas](#input\_twentycrm\_server\_replicas) | Number of replicas for the TwentyCRM server deployment. This defaults to 1. | `number` | `1` | no | | [twentycrm\_worker\_replicas](#input\_twentycrm\_worker\_replicas) | Number of replicas for the TwentyCRM worker deployment. This defaults to 1. | `number` | `1` | no | - \ No newline at end of file + diff --git a/twenty/code/k8s/terraform/deployment-redis.tf b/twenty/code/k8s/terraform/deployment-redis.tf new file mode 100644 index 0000000..d867dac --- /dev/null +++ b/twenty/code/k8s/terraform/deployment-redis.tf @@ -0,0 +1,60 @@ +resource "kubernetes_deployment" "twentycrm_redis" { + metadata { + name = "${var.twentycrm_app_name}-redis" + namespace = kubernetes_namespace.twentycrm.metadata.0.name + + labels = { + app = "${var.twentycrm_app_name}-redis" + } + } + + spec { + replicas = var.twentycrm_redis_replicas + selector { + match_labels = { + app = "${var.twentycrm_app_name}-redis" + } + } + + strategy { + type = "RollingUpdate" + rolling_update { + max_surge = "1" + max_unavailable = "1" + } + } + + template { + metadata { + labels = { + app = "${var.twentycrm_app_name}-redis" + } + } + + spec { + container { + image = var.twentycrm_redis_image + name = "redis" + + port { + container_port = 6379 + protocol = "TCP" + } + + resources { + requests = { + cpu = "250m" + memory = "1024Mi" + } + limits = { + cpu = "500m" + memory = "2048Mi" + } + } + } + dns_policy = "ClusterFirst" + restart_policy = "Always" + } + } + } +} diff --git a/twenty/code/k8s/terraform/deployment-server.tf b/twenty/code/k8s/terraform/deployment-server.tf index a3c1f9a..5276d57 100644 --- a/twenty/code/k8s/terraform/deployment-server.tf +++ b/twenty/code/k8s/terraform/deployment-server.tf @@ -37,20 +37,14 @@ resource "kubernetes_deployment" "twentycrm_server" { stdin = true tty = true - security_context { - allow_privilege_escalation = true - privileged = true - run_as_user = 1000 - } - env { name = "PORT" value = "3000" } - env { - name = "DEBUG_MODE" - value = false - } + # env { + # name = "DEBUG_MODE" + # value = false + # } env { name = "SERVER_URL" @@ -64,9 +58,12 @@ resource "kubernetes_deployment" "twentycrm_server" { env { name = "PG_DATABASE_URL" - value = "postgres://twenty:${var.twentycrm_pgdb_admin_password}@${var.twentycrm_app_name}-db.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local/default" + value = "postgres://twenty:${var.twentycrm_pgdb_admin_password}@${kubernetes_service.twentycrm_db.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local/default" + } + env { + name = "REDIS_URL" + value = "redis://${kubernetes_service.twentycrm_redis.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local:6379" } - env { name = "ENABLE_DB_MIGRATIONS" value = "true" @@ -83,10 +80,18 @@ resource "kubernetes_deployment" "twentycrm_server" { } env { name = "MESSAGE_QUEUE_TYPE" - value = "pg-boss" + value = "bull-mq" } env { - name = "ACCESS_TOKEN_SECRET" + name = "ACCESS_TOKEN_EXPIRES_IN" + value = "7d" + } + env { + name = "LOGIN_TOKEN_EXPIRES_IN" + value = "1h" + } + env { + name = "APP_SECRET" value_from { secret_key_ref { name = "tokens" @@ -95,36 +100,6 @@ resource "kubernetes_deployment" "twentycrm_server" { } } - env { - name = "LOGIN_TOKEN_SECRET" - value_from { - secret_key_ref { - name = "tokens" - key = "loginToken" - } - } - } - - env { - name = "REFRESH_TOKEN_SECRET" - value_from { - secret_key_ref { - name = "tokens" - key = "refreshToken" - } - } - } - - env { - name = "FILE_TOKEN_SECRET" - value_from { - secret_key_ref { - name = "tokens" - key = "fileToken" - } - } - } - port { container_port = 3000 protocol = "TCP" @@ -145,6 +120,11 @@ resource "kubernetes_deployment" "twentycrm_server" { name = "server-data" mount_path = var.twentycrm_server_data_mount_path } + + volume_mount { + name = "docker-data" + mount_path = var.twentycrm_docker_data_mount_path + } } volume { @@ -155,6 +135,14 @@ resource "kubernetes_deployment" "twentycrm_server" { } } + volume { + name = "docker-data" + + persistent_volume_claim { + claim_name = kubernetes_persistent_volume_claim.docker_data.metadata.0.name + } + } + dns_policy = "ClusterFirst" restart_policy = "Always" } @@ -162,6 +150,7 @@ resource "kubernetes_deployment" "twentycrm_server" { } depends_on = [ kubernetes_deployment.twentycrm_db, + kubernetes_deployment.twentycrm_redis, kubernetes_secret.twentycrm_tokens ] } diff --git a/twenty/code/k8s/terraform/deployment-worker.tf b/twenty/code/k8s/terraform/deployment-worker.tf index 9a00583..aa68fd3 100644 --- a/twenty/code/k8s/terraform/deployment-worker.tf +++ b/twenty/code/k8s/terraform/deployment-worker.tf @@ -50,7 +50,17 @@ resource "kubernetes_deployment" "twentycrm_worker" { env { name = "PG_DATABASE_URL" - value = "postgres://twenty:${var.twentycrm_pgdb_admin_password}@${var.twentycrm_app_name}-db.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local/default" + value = "postgres://twenty:${var.twentycrm_pgdb_admin_password}@${kubernetes_service.twentycrm_db.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local/default" + } + + env { + name = "CACHE_STORAGE_TYPE" + value = "redis" + } + + env { + name = "REDIS_URL" + value = "redis://${kubernetes_service.twentycrm_redis.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local:6379" } env { @@ -64,11 +74,11 @@ resource "kubernetes_deployment" "twentycrm_worker" { } env { name = "MESSAGE_QUEUE_TYPE" - value = "pg-boss" + value = "bull-mq" } env { - name = "ACCESS_TOKEN_SECRET" + name = "APP_SECRET" value_from { secret_key_ref { name = "tokens" @@ -77,44 +87,14 @@ resource "kubernetes_deployment" "twentycrm_worker" { } } - env { - name = "LOGIN_TOKEN_SECRET" - value_from { - secret_key_ref { - name = "tokens" - key = "loginToken" - } - } - } - - env { - name = "REFRESH_TOKEN_SECRET" - value_from { - secret_key_ref { - name = "tokens" - key = "refreshToken" - } - } - } - - env { - name = "FILE_TOKEN_SECRET" - value_from { - secret_key_ref { - name = "tokens" - key = "fileToken" - } - } - } - resources { requests = { cpu = "250m" - memory = "256Mi" + memory = "1024Mi" } limits = { cpu = "1000m" - memory = "1024Mi" + memory = "2048Mi" } } } @@ -126,6 +106,8 @@ resource "kubernetes_deployment" "twentycrm_worker" { } depends_on = [ kubernetes_deployment.twentycrm_db, - kubernetes_secret.twentycrm_tokens + kubernetes_deployment.twentycrm_redis, + kubernetes_deployment.twentycrm_server, + kubernetes_secret.twentycrm_tokens, ] } diff --git a/twenty/code/k8s/terraform/main.tf b/twenty/code/k8s/terraform/main.tf index 66ae6e1..a0e208d 100644 --- a/twenty/code/k8s/terraform/main.tf +++ b/twenty/code/k8s/terraform/main.tf @@ -13,7 +13,11 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" - version = ">= 2.31.0" + version = ">= 2.32.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.6.3" } } } diff --git a/twenty/code/k8s/terraform/pv-docker-data.tf b/twenty/code/k8s/terraform/pv-docker-data.tf new file mode 100644 index 0000000..9195fff --- /dev/null +++ b/twenty/code/k8s/terraform/pv-docker-data.tf @@ -0,0 +1,19 @@ +resource "kubernetes_persistent_volume" "docker_data" { + metadata { + name = "${var.twentycrm_app_name}-docker-data-pv" + } + spec { + storage_class_name = "default" + capacity = { + storage = var.twentycrm_docker_data_pv_capacity + } + access_modes = ["ReadWriteOnce"] + # refer to Terraform Docs for your specific implementation requirements + # https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume + persistent_volume_source { + local { + path = var.twentycrm_docker_data_pv_path + } + } + } +} diff --git a/twenty/code/k8s/terraform/pvc-docker-data.tf b/twenty/code/k8s/terraform/pvc-docker-data.tf new file mode 100644 index 0000000..daac13d --- /dev/null +++ b/twenty/code/k8s/terraform/pvc-docker-data.tf @@ -0,0 +1,15 @@ +resource "kubernetes_persistent_volume_claim" "docker_data" { + metadata { + name = "${var.twentycrm_app_name}-docker-data-pvc" + namespace = kubernetes_namespace.twentycrm.metadata.0.name + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = var.twentycrm_docker_data_pvc_requests + } + } + volume_name = kubernetes_persistent_volume.docker_data.metadata.0.name + } +} diff --git a/twenty/code/k8s/terraform/secret.tf b/twenty/code/k8s/terraform/secret.tf index 664d078..2aa7ccf 100644 --- a/twenty/code/k8s/terraform/secret.tf +++ b/twenty/code/k8s/terraform/secret.tf @@ -1,3 +1,18 @@ +locals { + tokens = [ + "accessToken", + "loginToken", + "refreshToken", + "fileToken" + ] +} + +resource "random_bytes" "this" { + for_each = toset(local.tokens) + + length = 32 +} + resource "kubernetes_secret" "twentycrm_tokens" { metadata { name = "tokens" @@ -5,11 +20,9 @@ resource "kubernetes_secret" "twentycrm_tokens" { } data = { - accessToken = var.twentycrm_token_accessToken - loginToken = var.twentycrm_token_loginToken - refreshToken = var.twentycrm_token_refreshToken - fileToken = var.twentycrm_token_fileToken + accessToken = random_bytes.this["accessToken"].base64 + loginToken = random_bytes.this["loginToken"].base64 + refreshToken = random_bytes.this["refreshToken"].base64 + fileToken = random_bytes.this["fileToken"].base64 } - - # type = "kubernetes.io/basic-auth" } diff --git a/twenty/code/k8s/terraform/service-redis.tf b/twenty/code/k8s/terraform/service-redis.tf new file mode 100644 index 0000000..fab1c00 --- /dev/null +++ b/twenty/code/k8s/terraform/service-redis.tf @@ -0,0 +1,18 @@ +resource "kubernetes_service" "twentycrm_redis" { + metadata { + name = "${var.twentycrm_app_name}-redis" + namespace = kubernetes_namespace.twentycrm.metadata.0.name + } + spec { + selector = { + app = "${var.twentycrm_app_name}-redis" + } + session_affinity = "ClientIP" + port { + port = 6379 + target_port = 6379 + } + + type = "ClusterIP" + } +} diff --git a/twenty/code/k8s/terraform/variables.tf b/twenty/code/k8s/terraform/variables.tf index 53255aa..3ce7394 100644 --- a/twenty/code/k8s/terraform/variables.tf +++ b/twenty/code/k8s/terraform/variables.tf @@ -1,30 +1,6 @@ ###################### # Required Variables # ###################### -variable "twentycrm_token_accessToken" { - type = string - description = "TwentyCRM access Token" - sensitive = true -} - -variable "twentycrm_token_loginToken" { - type = string - description = "TwentyCRM login Token" - sensitive = true -} - -variable "twentycrm_token_refreshToken" { - type = string - description = "TwentyCRM refresh Token" - sensitive = true -} - -variable "twentycrm_token_fileToken" { - type = string - description = "TwentyCRM file Token" - sensitive = true -} - variable "twentycrm_pgdb_admin_password" { type = string description = "TwentyCRM password for postgres database." @@ -53,7 +29,7 @@ variable "twentycrm_server_image" { variable "twentycrm_db_image" { type = string - default = "twentycrm/twenty-postgres:latest" + default = "twentycrm/twenty-postgres-spilo:latest" description = "TwentyCRM image for database deployment. This defaults to latest." } @@ -77,8 +53,8 @@ variable "twentycrm_db_replicas" { variable "twentycrm_server_data_mount_path" { type = string - default = "/app/docker-data" - description = "TwentyCRM mount path for servers application data. Defaults to '/app/docker-data'." + default = "/app/packages/twenty-server/.local-storage" + description = "TwentyCRM mount path for servers application data. Defaults to '/app/packages/twenty-server/.local-storage'." } variable "twentycrm_db_pv_path" { @@ -122,3 +98,39 @@ variable "twentycrm_namespace" { default = "twentycrm" description = "Namespace for all TwentyCRM resources" } + +variable "twentycrm_redis_replicas" { + type = number + default = 1 + description = "Number of replicas for the TwentyCRM Redis deployment. This defaults to 1." +} + +variable "twentycrm_redis_image" { + type = string + default = "redis/redis-stack-server:latest" + description = "TwentyCRM image for Redis deployment. This defaults to latest." +} + +variable "twentycrm_docker_data_mount_path" { + type = string + default = "/app/docker-data" + description = "TwentyCRM mount path for servers application data. Defaults to '/app/docker-data'." +} + +variable "twentycrm_docker_data_pv_path" { + type = string + default = "" + description = "Local path to use to store the physical volume if using local storage on nodes." +} + +variable "twentycrm_docker_data_pv_capacity" { + type = string + default = "100Mi" + description = "Storage capacity provisioned for server persistent volume." +} + +variable "twentycrm_docker_data_pvc_requests" { + type = string + default = "100Mi" + description = "Storage capacity reservation for server persistent volume claim." +} diff --git a/twenty/code/scripts/1-click.sh b/twenty/code/scripts/1-click.sh new file mode 100644 index 0000000..7e9c535 --- /dev/null +++ b/twenty/code/scripts/1-click.sh @@ -0,0 +1,22 @@ +pull_version=${VERSION:-$(curl -s https://api.github.com/repos/twentyhq/twenty/tags | grep '"name":' | head -n 1 | cut -d '"' -f 4)} + +if [[ -z "$pull_version" ]]; then + echo "Error: Unable to fetch the latest version tag. Please check your network connection or the GitHub API response." + exit 1 +fi +pull_branch=${BRANCH:-$pull_version} + +version_num=${pull_version#v} +target_version="0.32.4" + +# We moved the install script to a different location in v0.32.4 +if [[ -n "$BRANCH" ]] || [[ "$(printf '%s\n' "$target_version" "$version_num" | sort -V | head -n1)" != "$version_num" ]]; then + curl -sL "https://raw.githubusercontent.com/twentyhq/twenty/$pull_branch/packages/twenty-docker/scripts/install.sh" -o twenty_install.sh +else + curl -sL "https://raw.githubusercontent.com/twentyhq/twenty/$pull_branch/install.sh" -o twenty_install.sh +fi + +chmod +x twenty_install.sh +VERSION="$VERSION" BRANCH="$BRANCH" ./twenty_install.sh + +rm twenty_install.sh diff --git a/twenty/code/scripts/install.sh b/twenty/code/scripts/install.sh new file mode 100755 index 0000000..0eec572 --- /dev/null +++ b/twenty/code/scripts/install.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +echo "🔧 Checking dependencies..." +if ! command -v docker &>/dev/null; then + echo -e "\t❌ Docker is not installed or not in PATH. Please install Docker first.\n\t\tSee https://docs.docker.com/get-docker/" + exit 1 +fi +# Check if docker compose plugin is installed +if ! docker compose version &>/dev/null; then + echo -e "\t❌ Docker Compose is not installed or not in PATH (n.b. docker-compose is deprecated)\n\t\tUpdate docker or install docker-compose-plugin\n\t\tOn Linux: sudo apt-get install docker-compose-plugin\n\t\tSee https://docs.docker.com/compose/install/" + exit 1 +fi +# Check if docker is started +if ! docker info &>/dev/null; then + echo -e "\t❌ Docker is not running.\n\t\tPlease start Docker Desktop, Docker or check documentation at https://docs.docker.com/config/daemon/start/" + exit 1 +fi +if ! command -v curl &>/dev/null; then + echo -e "\t❌ Curl is not installed or not in PATH.\n\t\tOn macOS: brew install curl\n\t\tOn Linux: sudo apt install curl" + exit 1 +fi + +# Check if docker compose version is >= 2 +if [ "$(docker compose version --short | cut -d' ' -f3 | cut -d'.' -f1)" -lt 2 ]; then + echo -e "\t❌ Docker Compose is outdated. Please update Docker Compose to version 2 or higher.\n\t\tSee https://docs.docker.com/compose/install/linux/" + exit 1 +fi +# Check if docker-compose is installed, if so issue a warning if version is < 2 +if command -v docker-compose &>/dev/null; then + if [ "$(docker-compose version --short | cut -d' ' -f3 | cut -d'.' -f1)" -lt 2 ]; then + echo -e "\n\t⚠️ 'docker-compose' is installed but outdated. Make sure to use 'docker compose' or to upgrade 'docker-compose' to version 2.\n\t\tSee https://docs.docker.com/compose/install/standalone/\n" + fi +fi + +# Catch errors +set -e +function on_exit { + # $? is the exit status of the last command executed + local exit_status=$? + if [ $exit_status -ne 0 ]; then + echo "❌ Something went wrong, exiting: $exit_status" + fi +} +trap on_exit EXIT + +# Use environment variables VERSION and BRANCH, with defaults if not set +version=${VERSION:-$(curl -s https://api.github.com/repos/twentyhq/twenty/tags | grep '"name":' | head -n 1 | cut -d '"' -f 4)} +branch=${BRANCH:-$version} + +echo "🚀 Using version $version and branch $branch" + +dir_name="twenty" +function ask_directory { + read -p "📁 Enter the directory name to setup the project (default: $dir_name): " answer + if [ -n "$answer" ]; then + dir_name=$answer + fi +} + +ask_directory + +while [ -d "$dir_name" ]; do + read -p "🚫 Directory '$dir_name' already exists. Do you want to overwrite it? (y/N) " answer + if [ "$answer" = "y" ]; then + break + else + ask_directory + fi +done + +# Create a directory named twenty +echo "📁 Creating directory '$dir_name'" +mkdir -p "$dir_name" && cd "$dir_name" || { echo "❌ Failed to create/access directory '$dir_name'"; exit 1; } + +# Copy twenty/packages/twenty-docker/docker-compose.yml in it +echo -e "\t• Copying docker-compose.yml" +curl -sLo docker-compose.yml https://raw.githubusercontent.com/twentyhq/twenty/$branch/packages/twenty-docker/docker-compose.yml + +# Copy twenty/packages/twenty-docker/.env.example to .env +echo -e "\t• Setting up .env file" +curl -sLo .env https://raw.githubusercontent.com/twentyhq/twenty/$branch/packages/twenty-docker/.env.example + +# Replace TAG=latest by TAG= +if [[ $(uname) == "Darwin" ]]; then + # Running on macOS + sed -i '' "s/TAG=latest/TAG=$version/g" .env +else + # Assuming Linux + sed -i'' "s/TAG=latest/TAG=$version/g" .env +fi + +# Generate random strings for secrets +echo "# === Randomly generated secrets ===" >>.env +echo "APP_SECRET=$(openssl rand -base64 32)" >>.env +echo "" >>.env +echo "PGPASSWORD_SUPERUSER=$(openssl rand -hex 16)" >>.env + +echo -e "\t• .env configuration completed" + +port=3000 +# Check if command nc is available +if command -v nc &> /dev/null; then + # Check if port 3000 is already in use, propose to change it + while nc -zv localhost $port &>/dev/null; do + read -p "🚫 Port $port is already in use. Do you want to use another port? (Y/n) " answer + if [ "$answer" = "n" ]; then + continue + fi + read -p "Enter a new port number: " new_port + if [[ $(uname) == "Darwin" ]]; then + sed -i '' "s/$port:$port/$new_port:$port/g" docker-compose.yml + sed -E -i '' "s|^SERVER_URL=http://localhost:[0-9]+|SERVER_URL=http://localhost:$new_port|g" .env + else + sed -i'' "s/$port:$port/$new_port:$port/g" docker-compose.yml + sed -E -i'' "s|^SERVER_URL=http://localhost:[0-9]+|SERVER_URL=http://localhost:$new_port|g" .env + fi + port=$new_port + done +fi + +# Ask user if they want to start the project +read -p "🚀 Do you want to start the project now? (Y/n) " answer +if [ "$answer" = "n" ]; then + echo "✅ Project setup completed. Run 'docker compose up -d' to start." + exit 0 +else + echo "🐳 Starting Docker containers..." + docker compose up -d + # Check if port is listening + echo "Waiting for server to be healthy, it might take a few minutes while we initialize the database..." + # Tail logs of the server until it's ready + docker compose logs -f server & + pid=$! + while [ ! $(docker inspect --format='{{.State.Health.Status}}' twenty-server-1) = "healthy" ]; do + sleep 1 + done + kill $pid + echo "" + echo "✅ Server is up and running" +fi + +function ask_open_browser { + read -p "🌐 Do you want to open the project in your browser? (Y/n) " answer + if [ "$answer" = "n" ]; then + echo "✅ Setup completed. Access your project at http://localhost:$port" + exit 0 + fi +} + +# Ask user if they want to open the project +# Running on macOS +if [[ $(uname) == "Darwin" ]]; then + ask_open_browser + + open "http://localhost:$port" +# Assuming Linux +else + # xdg-open is not installed, we could be running in a non gui environment + if command -v xdg-open >/dev/null 2>&1; then + ask_open_browser + + xdg-open "http://localhost:$port" + else + echo "✅ Setup completed. Your project is available at http://localhost:$port" + fi +fi diff --git a/twenty/code/twenty-postgres-spilo/Dockerfile b/twenty/code/twenty-postgres-spilo/Dockerfile index 21e107c..9a84c12 100644 --- a/twenty/code/twenty-postgres-spilo/Dockerfile +++ b/twenty/code/twenty-postgres-spilo/Dockerfile @@ -1,13 +1,12 @@ ARG POSTGRES_VERSION=15 ARG SPILO_VERSION=3.2-p1 -ARG PG_GRAPHQL_VERSION=1.5.6 ARG WRAPPERS_VERSION=0.2.0 # Build the mysql_fdw extension -FROM debian:bookworm as build-mysql_fdw +FROM debian:bookworm AS build-mysql_fdw ARG POSTGRES_VERSION -ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_FRONTEND=noninteractive RUN apt update && \ apt install -y \ build-essential \ @@ -18,14 +17,14 @@ RUN apt update && \ # Install mysql_fdw RUN git clone https://github.com/EnterpriseDB/mysql_fdw.git -WORKDIR mysql_fdw +WORKDIR /mysql_fdw RUN make USE_PGXS=1 # Build libssl for wrappers -FROM ubuntu:22.04 as build-libssl +FROM ubuntu:22.04 AS build-libssl -ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_FRONTEND=noninteractive RUN apt update && \ apt install -y \ build-essential \ @@ -38,10 +37,9 @@ WORKDIR /build/openssl RUN ./config && make && make install -# Extend the Spilo image with the pg_graphql and mysql_fdw extensions +# Extend the Spilo image with the mysql_fdw extensions FROM ghcr.io/zalando/spilo-${POSTGRES_VERSION}:${SPILO_VERSION} ARG POSTGRES_VERSION -ARG PG_GRAPHQL_VERSION ARG WRAPPERS_VERSION ARG TARGETARCH @@ -63,14 +61,6 @@ RUN curl -L "https://github.com/supabase/wrappers/releases/download/v${WRAPPERS_ COPY --from=build-libssl /usr/local/lib/libssl* /usr/local/lib/libcrypto* /usr/lib/ COPY --from=build-libssl /usr/local/lib/engines-1.1 /usr/lib/engines-1.1 -# Copy pg_graphql -COPY ./packages/twenty-postgres/linux/${TARGETARCH}/${POSTGRES_VERSION}/pg_graphql/${PG_GRAPHQL_VERSION}/pg_graphql--${PG_GRAPHQL_VERSION}.sql \ - /usr/share/postgresql/${POSTGRES_VERSION}/extension -COPY ./packages/twenty-postgres/linux/${TARGETARCH}/${POSTGRES_VERSION}/pg_graphql/${PG_GRAPHQL_VERSION}/pg_graphql.control \ - /usr/share/postgresql/${POSTGRES_VERSION}/extension -COPY ./packages/twenty-postgres/linux/${TARGETARCH}/${POSTGRES_VERSION}/pg_graphql/${PG_GRAPHQL_VERSION}/pg_graphql.so \ - /usr/lib/postgresql/${POSTGRES_VERSION}/lib/pg_graphql.so - # Copy mysql_fdw COPY --from=build-mysql_fdw /mysql_fdw/mysql_fdw.so \ /usr/lib/postgresql/${POSTGRES_VERSION}/lib/mysql_fdw.so diff --git a/twenty/code/twenty-postgres/Dockerfile b/twenty/code/twenty-postgres/Dockerfile deleted file mode 100644 index 5647a6c..0000000 --- a/twenty/code/twenty-postgres/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -ARG IMAGE_TAG='15.5.0-debian-11-r15' - -FROM bitnami/postgresql:${IMAGE_TAG} - -ARG PG_MAIN_VERSION=15 -ARG PG_GRAPHQL_VERSION=1.5.6 -ARG WRAPPERS_VERSION=0.2.0 -ARG TARGETARCH - -USER root - -RUN set -eux; \ - ARCH="$(dpkg --print-architecture)"; \ - case "${ARCH}" in \ - aarch64|arm64) \ - TARGETARCH='arm64'; \ - ;; \ - amd64|x86_64) \ - TARGETARCH='amd64'; \ - ;; \ - *) \ - echo "Unsupported arch: ${ARCH}"; \ - exit 1; \ - ;; \ - esac; - -RUN apt update && apt install build-essential git curl default-libmysqlclient-dev -y - -# Install precompiled pg_graphql extensions -COPY ./packages/twenty-postgres/linux/${TARGETARCH}/${PG_MAIN_VERSION}/pg_graphql/${PG_GRAPHQL_VERSION}/pg_graphql--${PG_GRAPHQL_VERSION}.sql \ - /opt/bitnami/postgresql/share/extension/ -COPY ./packages/twenty-postgres/linux/${TARGETARCH}/${PG_MAIN_VERSION}/pg_graphql/${PG_GRAPHQL_VERSION}/pg_graphql.control \ - /opt/bitnami/postgresql/share/extension/ -COPY ./packages/twenty-postgres/linux/${TARGETARCH}/${PG_MAIN_VERSION}/pg_graphql/${PG_GRAPHQL_VERSION}/pg_graphql.so \ - /opt/bitnami/postgresql/lib/ - -# Install precompiled supabase wrappers extensions -RUN curl -L "https://github.com/supabase/wrappers/releases/download/v${WRAPPERS_VERSION}/wrappers-v${WRAPPERS_VERSION}-pg${PG_MAIN_VERSION}-${TARGETARCH}-linux-gnu.deb" -o wrappers.deb -RUN dpkg --install wrappers.deb -RUN cp /usr/share/postgresql/${PG_MAIN_VERSION}/extension/wrappers* /opt/bitnami/postgresql/share/extension/ -RUN cp /usr/lib/postgresql/${PG_MAIN_VERSION}/lib/wrappers* /opt/bitnami/postgresql/lib/ - -RUN export PATH=/usr/local/pgsql/bin/:$PATH -RUN export PATH=/usr/local/mysql/bin/:$PATH -RUN git clone https://github.com/EnterpriseDB/mysql_fdw.git -WORKDIR mysql_fdw -RUN make USE_PGXS=1 -RUN make USE_PGXS=1 install - -COPY ./packages/twenty-docker/twenty-postgres/init.sql /docker-entrypoint-initdb.d/ - -USER 1001 -ENTRYPOINT ["/opt/bitnami/scripts/postgresql/entrypoint.sh"] -CMD ["/opt/bitnami/scripts/postgresql/run.sh"] diff --git a/twenty/code/twenty-postgres/init.sql b/twenty/code/twenty-postgres/init.sql deleted file mode 100644 index 16a4788..0000000 --- a/twenty/code/twenty-postgres/init.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE DATABASE "default"; -CREATE DATABASE "test"; -CREATE USER twenty PASSWORD 'twenty'; -ALTER ROLE twenty superuser; diff --git a/twenty/code/twenty/Dockerfile b/twenty/code/twenty/Dockerfile index e57cf30..b1d5d4d 100644 --- a/twenty/code/twenty/Dockerfile +++ b/twenty/code/twenty/Dockerfile @@ -52,9 +52,10 @@ RUN apk add --no-cache curl jq RUN npm install -g tsx +RUN apk add --no-cache postgresql-client + COPY ./packages/twenty-docker/twenty/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh - WORKDIR /app/packages/twenty-server ARG REACT_APP_SERVER_BASE_URL diff --git a/twenty/code/twenty/entrypoint.sh b/twenty/code/twenty/entrypoint.sh index 9733781..331a463 100755 --- a/twenty/code/twenty/entrypoint.sh +++ b/twenty/code/twenty/entrypoint.sh @@ -1,9 +1,18 @@ #!/bin/sh +set -e # Check if the initialization has already been done and that we enabled automatic migration if [ "${ENABLE_DB_MIGRATIONS}" = "true" ] && [ ! -f /app/docker-data/db_status ]; then echo "Running database setup and migrations..." + # Creating the database if it doesn't exist + PGUSER=$(echo $PG_DATABASE_URL | awk -F '//' '{print $2}' | awk -F ':' '{print $1}') + PGPASS=$(echo $PG_DATABASE_URL | awk -F ':' '{print $3}' | awk -F '@' '{print $1}') + PGHOST=$(echo $PG_DATABASE_URL | awk -F '@' '{print $2}' | awk -F ':' '{print $1}') + PGPORT=$(echo $PG_DATABASE_URL | awk -F ':' '{print $4}' | awk -F '/' '{print $1}') + PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'default'" | grep -q 1 || \ + PGPASSWORD=${PGPASS} psql -h ${PGHOST} -p ${PGPORT} -U ${PGUSER} -d postgres -c "CREATE DATABASE \"default\"" + # Run setup and migration scripts NODE_OPTIONS="--max-old-space-size=1500" tsx ./scripts/setup-db.ts yarn database:migrate:prod