diff --git a/dify/code/.env.example b/dify/code/.env.example index b21bdc7..afd2caf 100644 --- a/dify/code/.env.example +++ b/dify/code/.env.example @@ -887,6 +887,10 @@ SSRF_HTTP_PORT=3128 SSRF_COREDUMP_DIR=/var/spool/squid SSRF_REVERSE_PROXY_PORT=8194 SSRF_SANDBOX_HOST=sandbox +SSRF_DEFAULT_TIME_OUT=5 +SSRF_DEFAULT_CONNECT_TIME_OUT=5 +SSRF_DEFAULT_READ_TIME_OUT=5 +SSRF_DEFAULT_WRITE_TIME_OUT=5 # ------------------------------ # docker env var for specifying vector db type at startup @@ -932,3 +936,30 @@ MAX_SUBMIT_COUNT=100 # The maximum number of top-k value for RAG. TOP_K_MAX_VALUE=10 + +# ------------------------------ +# Plugin Daemon Configuration +# ------------------------------ + +DB_PLUGIN_DATABASE=dify_plugin +EXPOSE_PLUGIN_DAEMON_PORT=5002 +PLUGIN_DAEMON_PORT=5002 +PLUGIN_DAEMON_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi +PLUGIN_DAEMON_URL=http://plugin_daemon:5002 +PLUGIN_MAX_PACKAGE_SIZE=52428800 +PLUGIN_PPROF_ENABLED=false + +PLUGIN_DEBUGGING_HOST=0.0.0.0 +PLUGIN_DEBUGGING_PORT=5003 +EXPOSE_PLUGIN_DEBUGGING_HOST=localhost +EXPOSE_PLUGIN_DEBUGGING_PORT=5003 + +PLUGIN_DIFY_INNER_API_KEY=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1 +PLUGIN_DIFY_INNER_API_URL=http://api:5001 + +ENDPOINT_URL_TEMPLATE=http://localhost/e/{hook_id} + +MARKETPLACE_ENABLED=true +MARKETPLACE_API_URL=https://marketplace.dify.ai + +FORCE_VERIFYING_SIGNATURE=true diff --git a/dify/code/docker-compose-template.yaml b/dify/code/docker-compose-template.yaml index 8aafc61..f284449 100644 --- a/dify/code/docker-compose-template.yaml +++ b/dify/code/docker-compose-template.yaml @@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:0.15.2 + image: langgenius/dify-api:0.15.3 restart: always environment: # Use the shared environment variables. @@ -12,6 +12,8 @@ services: SENTRY_DSN: ${API_SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} + PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} depends_on: - db - redis @@ -25,7 +27,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.15.2 + image: langgenius/dify-api:0.15.3 restart: always environment: # Use the shared environment variables. @@ -35,6 +37,8 @@ services: SENTRY_DSN: ${API_SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} + PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} depends_on: - db - redis @@ -47,7 +51,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.15.2 + image: langgenius/dify-web:0.15.3 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -56,8 +60,11 @@ services: NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-0} TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} CSP_WHITELIST: ${CSP_WHITELIST:-} + MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} + MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai} TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-} INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-} + PM2_INSTANCES: ${PM2_INSTANCES:-2} # The postgres database. db: @@ -81,6 +88,8 @@ services: interval: 1s timeout: 3s retries: 30 + ports: + - '${EXPOSE_DB_PORT:-5432}:5432' # The redis cache. redis: @@ -118,6 +127,30 @@ services: networks: - ssrf_proxy_network + # plugin daemon + plugin_daemon: + image: langgenius/dify-plugin-daemon:0.0.2-local + restart: always + environment: + # Use the shared environment variables. + <<: *shared-api-worker-env + DB_DATABASE: ${DB_PLUGIN_DATABASE:-dify_plugin} + SERVER_PORT: ${PLUGIN_DAEMON_PORT:-5002} + SERVER_KEY: ${PLUGIN_DAEMON_KEY:-lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi} + MAX_PLUGIN_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false} + DIFY_INNER_API_URL: ${PLUGIN_DIFY_INNER_API_URL:-http://api:5001} + DIFY_INNER_API_KEY: ${INNER_API_KEY_FOR_PLUGIN:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_REMOTE_INSTALL_HOST:-0.0.0.0} + PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_REMOTE_INSTALL_PORT:-5003} + PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} + FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + ports: + - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}" + volumes: + - ./volumes/plugin_daemon:/app/storage + + # ssrf_proxy server # for more information, please refer to # https://docs.dify.ai/learn-more/faq/install-faq#id-18.-why-is-ssrf_proxy-needed @@ -199,16 +232,6 @@ services: - '${EXPOSE_NGINX_PORT:-80}:${NGINX_PORT:-80}' - '${EXPOSE_NGINX_SSL_PORT:-443}:${NGINX_SSL_PORT:-443}' - # The TiDB vector store. - # For production use, please refer to https://github.com/pingcap/tidb-docker-compose - tidb: - image: pingcap/tidb:v8.4.0 - profiles: - - tidb - command: - - --store=unistore - restart: always - # The Weaviate vector store. weaviate: image: semitechnologies/weaviate:1.19.0 diff --git a/dify/code/docker-compose.middleware.yaml b/dify/code/docker-compose.middleware.yaml index 11f5302..961f953 100644 --- a/dify/code/docker-compose.middleware.yaml +++ b/dify/code/docker-compose.middleware.yaml @@ -64,6 +64,36 @@ services: networks: - ssrf_proxy_network + # plugin daemon + plugin_daemon: + image: langgenius/dify-plugin-daemon:0.0.2-local + restart: always + environment: + # Use the shared environment variables. + DB_HOST: ${DB_HOST:-db} + DB_PORT: ${DB_PORT:-5432} + DB_USERNAME: ${DB_USER:-postgres} + DB_PASSWORD: ${DB_PASSWORD:-difyai123456} + DB_DATABASE: ${DB_PLUGIN_DATABASE:-dify_plugin} + REDIS_HOST: ${REDIS_HOST:-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + REDIS_PASSWORD: ${REDIS_PASSWORD:-difyai123456} + SERVER_PORT: ${PLUGIN_DAEMON_PORT:-5002} + SERVER_KEY: ${PLUGIN_DAEMON_KEY:-lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi} + MAX_PLUGIN_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false} + DIFY_INNER_API_URL: ${PLUGIN_DIFY_INNER_API_URL:-http://host.docker.internal:5001} + DIFY_INNER_API_KEY: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_DEBUGGING_HOST:-0.0.0.0} + PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003} + PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} + FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + ports: + - "${EXPOSE_PLUGIN_DAEMON_PORT:-5002}:${PLUGIN_DAEMON_PORT:-5002}" + - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}" + volumes: + - ./volumes/plugin_daemon:/app/storage + # ssrf_proxy server # for more information, please refer to # https://docs.dify.ai/learn-more/faq/install-faq#id-18.-why-is-ssrf_proxy-needed diff --git a/dify/code/docker-compose.yaml b/dify/code/docker-compose.yaml index 3f7d374..5ee9cb3 100644 --- a/dify/code/docker-compose.yaml +++ b/dify/code/docker-compose.yaml @@ -377,6 +377,10 @@ x-shared-env: &shared-api-worker-env SSRF_COREDUMP_DIR: ${SSRF_COREDUMP_DIR:-/var/spool/squid} SSRF_REVERSE_PROXY_PORT: ${SSRF_REVERSE_PROXY_PORT:-8194} SSRF_SANDBOX_HOST: ${SSRF_SANDBOX_HOST:-sandbox} + SSRF_DEFAULT_TIME_OUT: ${SSRF_DEFAULT_TIME_OUT:-5} + SSRF_DEFAULT_CONNECT_TIME_OUT: ${SSRF_DEFAULT_CONNECT_TIME_OUT:-5} + SSRF_DEFAULT_READ_TIME_OUT: ${SSRF_DEFAULT_READ_TIME_OUT:-5} + SSRF_DEFAULT_WRITE_TIME_OUT: ${SSRF_DEFAULT_WRITE_TIME_OUT:-5} EXPOSE_NGINX_PORT: ${EXPOSE_NGINX_PORT:-80} EXPOSE_NGINX_SSL_PORT: ${EXPOSE_NGINX_SSL_PORT:-443} POSITION_TOOL_PINS: ${POSITION_TOOL_PINS:-} @@ -389,11 +393,28 @@ x-shared-env: &shared-api-worker-env CREATE_TIDB_SERVICE_JOB_ENABLED: ${CREATE_TIDB_SERVICE_JOB_ENABLED:-false} MAX_SUBMIT_COUNT: ${MAX_SUBMIT_COUNT:-100} TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-10} + DB_PLUGIN_DATABASE: ${DB_PLUGIN_DATABASE:-dify_plugin} + EXPOSE_PLUGIN_DAEMON_PORT: ${EXPOSE_PLUGIN_DAEMON_PORT:-5002} + PLUGIN_DAEMON_PORT: ${PLUGIN_DAEMON_PORT:-5002} + PLUGIN_DAEMON_KEY: ${PLUGIN_DAEMON_KEY:-lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi} + PLUGIN_DAEMON_URL: ${PLUGIN_DAEMON_URL:-http://plugin_daemon:5002} + PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + PLUGIN_PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false} + PLUGIN_DEBUGGING_HOST: ${PLUGIN_DEBUGGING_HOST:-0.0.0.0} + PLUGIN_DEBUGGING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003} + EXPOSE_PLUGIN_DEBUGGING_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost} + EXPOSE_PLUGIN_DEBUGGING_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003} + PLUGIN_DIFY_INNER_API_KEY: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + PLUGIN_DIFY_INNER_API_URL: ${PLUGIN_DIFY_INNER_API_URL:-http://api:5001} + ENDPOINT_URL_TEMPLATE: ${ENDPOINT_URL_TEMPLATE:-http://localhost/e/{hook_id}} + MARKETPLACE_ENABLED: ${MARKETPLACE_ENABLED:-true} + MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} + FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} services: # API service api: - image: langgenius/dify-api:0.15.2 + image: langgenius/dify-api:0.15.3 restart: always environment: # Use the shared environment variables. @@ -403,6 +424,8 @@ services: SENTRY_DSN: ${API_SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} + PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} depends_on: - db - redis @@ -416,7 +439,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.15.2 + image: langgenius/dify-api:0.15.3 restart: always environment: # Use the shared environment variables. @@ -426,6 +449,8 @@ services: SENTRY_DSN: ${API_SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} + PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} depends_on: - db - redis @@ -438,7 +463,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.15.2 + image: langgenius/dify-web:0.15.3 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} @@ -447,8 +472,11 @@ services: NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-0} TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} CSP_WHITELIST: ${CSP_WHITELIST:-} + MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} + MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai} TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-} INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-} + PM2_INSTANCES: ${PM2_INSTANCES:-2} # The postgres database. db: @@ -509,6 +537,27 @@ services: networks: - ssrf_proxy_network + # plugin daemon + plugin_daemon: + image: langgenius/dify-plugin-daemon:0.0.2-local + restart: always + environment: + # Use the shared environment variables. + <<: *shared-api-worker-env + DB_DATABASE: ${DB_PLUGIN_DATABASE:-dify_plugin} + SERVER_PORT: ${PLUGIN_DAEMON_PORT:-5002} + SERVER_KEY: ${PLUGIN_DAEMON_KEY:-lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi} + MAX_PLUGIN_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false} + DIFY_INNER_API_URL: ${PLUGIN_DIFY_INNER_API_URL:-http://api:5001} + DIFY_INNER_API_KEY: ${INNER_API_KEY_FOR_PLUGIN:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} + PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_REMOTE_INSTALL_HOST:-0.0.0.0} + PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_REMOTE_INSTALL_PORT:-5003} + PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} + FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + volumes: + - ./volumes/plugin_daemon:/app/storage + # ssrf_proxy server # for more information, please refer to # https://docs.dify.ai/learn-more/faq/install-faq#id-18.-why-is-ssrf_proxy-needed @@ -601,16 +650,6 @@ services: - api - web - # The TiDB vector store. - # For production use, please refer to https://github.com/pingcap/tidb-docker-compose - tidb: - image: pingcap/tidb:v8.4.0 - profiles: - - tidb - command: - - --store=unistore - restart: always - # The Weaviate vector store. weaviate: image: semitechnologies/weaviate:1.19.0 diff --git a/dify/code/middleware.env.example b/dify/code/middleware.env.example index c4ce9f0..aaecad7 100644 --- a/dify/code/middleware.env.example +++ b/dify/code/middleware.env.example @@ -87,3 +87,31 @@ EXPOSE_REDIS_PORT=6379 EXPOSE_SANDBOX_PORT=8194 EXPOSE_SSRF_PROXY_PORT=3128 EXPOSE_WEAVIATE_PORT=8080 + +# ------------------------------ +# Plugin Daemon Configuration +# ------------------------------ + +DB_PLUGIN_DATABASE=dify_plugin +EXPOSE_PLUGIN_DAEMON_PORT=5002 +PLUGIN_DAEMON_PORT=5002 +PLUGIN_DAEMON_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi +PLUGIN_DAEMON_URL=http://host.docker.internal:5002 +PLUGIN_MAX_PACKAGE_SIZE=52428800 +PLUGIN_PPROF_ENABLED=false +PLUGIN_WORKING_PATH=/app/storage/cwd + +ENDPOINT_URL_TEMPLATE=http://localhost:5002/e/{hook_id} + +PLUGIN_DEBUGGING_PORT=5003 +PLUGIN_DEBUGGING_HOST=0.0.0.0 +EXPOSE_PLUGIN_DEBUGGING_HOST=localhost +EXPOSE_PLUGIN_DEBUGGING_PORT=5003 + +PLUGIN_DIFY_INNER_API_KEY=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1 +PLUGIN_DIFY_INNER_API_URL=http://api:5001 + +MARKETPLACE_ENABLED=true +MARKETPLACE_API_URL=https://marketplace.dify.ai + +FORCE_VERIFYING_SIGNATURE=true \ No newline at end of file diff --git a/dify/code/nginx/conf.d/default.conf.template b/dify/code/nginx/conf.d/default.conf.template index 9691122..a24219e 100644 --- a/dify/code/nginx/conf.d/default.conf.template +++ b/dify/code/nginx/conf.d/default.conf.template @@ -24,6 +24,17 @@ server { include proxy.conf; } + location /explore { + proxy_pass http://web:3000; + include proxy.conf; + } + + location /e { + proxy_pass http://plugin_daemon:5002; + proxy_set_header Dify-Hook-Url $scheme://$host$request_uri; + include proxy.conf; + } + location / { proxy_pass http://web:3000; include proxy.conf; diff --git a/dify/code/nginx/proxy.conf.template b/dify/code/nginx/proxy.conf.template index 6b52d23..117f806 100644 --- a/dify/code/nginx/proxy.conf.template +++ b/dify/code/nginx/proxy.conf.template @@ -3,6 +3,7 @@ proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; +proxy_set_header X-Forwarded-Port $server_port; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_buffering off; diff --git a/dify/code/ssrf_proxy/squid.conf.template b/dify/code/ssrf_proxy/squid.conf.template index a0875a8..676fe73 100644 --- a/dify/code/ssrf_proxy/squid.conf.template +++ b/dify/code/ssrf_proxy/squid.conf.template @@ -7,6 +7,7 @@ acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) acl localnet src fc00::/7 # RFC 4193 local private network range acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines acl SSL_ports port 443 +# acl SSL_ports port 1025-65535 # Enable the configuration to resolve this issue: https://github.com/langgenius/dify/issues/12792 acl Safe_ports port 80 # http acl Safe_ports port 21 # ftp acl Safe_ports port 443 # https diff --git a/dify/code/tidb/config/pd.toml b/dify/code/tidb/config/pd.toml new file mode 100644 index 0000000..042b251 --- /dev/null +++ b/dify/code/tidb/config/pd.toml @@ -0,0 +1,4 @@ +# PD Configuration File reference: +# https://docs.pingcap.com/tidb/stable/pd-configuration-file#pd-configuration-file +[replication] +max-replicas = 1 \ No newline at end of file diff --git a/dify/code/tidb/config/tiflash-learner.toml b/dify/code/tidb/config/tiflash-learner.toml new file mode 100644 index 0000000..5098829 --- /dev/null +++ b/dify/code/tidb/config/tiflash-learner.toml @@ -0,0 +1,13 @@ +# TiFlash tiflash-learner.toml Configuration File reference: +# https://docs.pingcap.com/tidb/stable/tiflash-configuration#configure-the-tiflash-learnertoml-file + +log-file = "/logs/tiflash_tikv.log" + +[server] +engine-addr = "tiflash:4030" +addr = "0.0.0.0:20280" +advertise-addr = "tiflash:20280" +status-addr = "tiflash:20292" + +[storage] +data-dir = "/data/flash" diff --git a/dify/code/tidb/config/tiflash.toml b/dify/code/tidb/config/tiflash.toml new file mode 100644 index 0000000..30ac13e --- /dev/null +++ b/dify/code/tidb/config/tiflash.toml @@ -0,0 +1,19 @@ +# TiFlash tiflash.toml Configuration File reference: +# https://docs.pingcap.com/tidb/stable/tiflash-configuration#configure-the-tiflashtoml-file + +listen_host = "0.0.0.0" +path = "/data" + +[flash] +tidb_status_addr = "tidb:10080" +service_addr = "tiflash:4030" + +[flash.proxy] +config = "/tiflash-learner.toml" + +[logger] +errorlog = "/logs/tiflash_error.log" +log = "/logs/tiflash.log" + +[raft] +pd_addr = "pd0:2379" diff --git a/dify/code/tidb/docker-compose.yaml b/dify/code/tidb/docker-compose.yaml new file mode 100644 index 0000000..fa15770 --- /dev/null +++ b/dify/code/tidb/docker-compose.yaml @@ -0,0 +1,62 @@ +services: + pd0: + image: pingcap/pd:v8.5.1 + # ports: + # - "2379" + volumes: + - ./config/pd.toml:/pd.toml:ro + - ./volumes/data:/data + - ./volumes/logs:/logs + command: + - --name=pd0 + - --client-urls=http://0.0.0.0:2379 + - --peer-urls=http://0.0.0.0:2380 + - --advertise-client-urls=http://pd0:2379 + - --advertise-peer-urls=http://pd0:2380 + - --initial-cluster=pd0=http://pd0:2380 + - --data-dir=/data/pd + - --config=/pd.toml + - --log-file=/logs/pd.log + restart: on-failure + tikv: + image: pingcap/tikv:v8.5.1 + volumes: + - ./volumes/data:/data + - ./volumes/logs:/logs + command: + - --addr=0.0.0.0:20160 + - --advertise-addr=tikv:20160 + - --status-addr=tikv:20180 + - --data-dir=/data/tikv + - --pd=pd0:2379 + - --log-file=/logs/tikv.log + depends_on: + - "pd0" + restart: on-failure + tidb: + image: pingcap/tidb:v8.5.1 + # ports: + # - "4000:4000" + volumes: + - ./volumes/logs:/logs + command: + - --advertise-address=tidb + - --store=tikv + - --path=pd0:2379 + - --log-file=/logs/tidb.log + depends_on: + - "tikv" + restart: on-failure + tiflash: + image: pingcap/tiflash:v8.5.1 + volumes: + - ./config/tiflash.toml:/tiflash.toml:ro + - ./config/tiflash-learner.toml:/tiflash-learner.toml:ro + - ./volumes/data:/data + - ./volumes/logs:/logs + command: + - --config=/tiflash.toml + depends_on: + - "tikv" + - "tidb" + restart: on-failure diff --git a/supabase/code/docker-compose.yml b/supabase/code/docker-compose.yml index 6963681..9f98f0a 100644 --- a/supabase/code/docker-compose.yml +++ b/supabase/code/docker-compose.yml @@ -296,7 +296,7 @@ services: PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} functions: - image: supabase/edge-runtime:v1.66.5 + image: supabase/edge-runtime:v1.67.0 restart: unless-stopped volumes: - ./volumes/functions:/home/deno/functions:Z diff --git a/twenty/code/.env.example b/twenty/code/.env.example index 344c20c..941e7df 100644 --- a/twenty/code/.env.example +++ b/twenty/code/.env.example @@ -4,13 +4,13 @@ TAG=latest #PG_DATABASE_PASSWORD=replace_me_with_a_strong_password_without_special_characters #PG_DATABASE_HOST=db #PG_DATABASE_PORT=5432 -#REDIS_URL=redis://redis:6379 +REDIS_URL=redis://redis:6379 -SERVER_URL=http://localhost:3000 +SERVER_URL=https://$(PRIMARY_DOMAIN) SIGN_IN_PREFILLED=false # Use openssl rand -base64 32 for each secret -# APP_SECRET=replace_me_with_a_random_string +APP_SECRET=replace_me_with_a_random_string STORAGE_TYPE=local diff --git a/twenty/code/docker-compose.yml b/twenty/code/docker-compose.yml index b8665b5..7ea2bd6 100644 --- a/twenty/code/docker-compose.yml +++ b/twenty/code/docker-compose.yml @@ -17,7 +17,7 @@ services: - server-local-data:/app/packages/twenty-server/${STORAGE_LOCAL_PATH:-.local-storage} - docker-data:/app/docker-data environment: - PORT: 3000 + NODE_PORT: 3000 PG_DATABASE_URL: postgres://${PG_DATABASE_USER:-postgres}:${PG_DATABASE_PASSWORD:-postgres}@${PG_DATABASE_HOST:-db}:${PG_DATABASE_PORT:-5432}/default SERVER_URL: ${SERVER_URL} REDIS_URL: ${REDIS_URL:-redis://redis:6379} diff --git a/twenty/code/k8s/manifests/deployment-server.yaml b/twenty/code/k8s/manifests/deployment-server.yaml index 13a1492..9d21993 100644 --- a/twenty/code/k8s/manifests/deployment-server.yaml +++ b/twenty/code/k8s/manifests/deployment-server.yaml @@ -33,7 +33,7 @@ spec: image: twentycrm/twenty:latest imagePullPolicy: Always env: - - name: PORT + - name: NODE_PORT value: 3000 - name: SERVER_URL value: "https://crm.example.com:443" @@ -45,8 +45,6 @@ spec: value: "false" - name: STORAGE_TYPE value: "local" - - name: "MESSAGE_QUEUE_TYPE" - value: "bull-mq" - name: "ACCESS_TOKEN_EXPIRES_IN" value: "7d" - name: "LOGIN_TOKEN_EXPIRES_IN" diff --git a/twenty/code/k8s/manifests/deployment-worker.yaml b/twenty/code/k8s/manifests/deployment-worker.yaml index dc6a15e..fbee277 100644 --- a/twenty/code/k8s/manifests/deployment-worker.yaml +++ b/twenty/code/k8s/manifests/deployment-worker.yaml @@ -34,10 +34,6 @@ spec: value: "false" # it already runs on the server - name: STORAGE_TYPE value: "local" - - name: "MESSAGE_QUEUE_TYPE" - value: "bull-mq" - - name: "CACHE_STORAGE_TYPE" - value: "redis" - name: "REDIS_URL" value: "redis://twentycrm-redis.twentycrm.svc.cluster.local:6379" - name: APP_SECRET diff --git a/twenty/code/k8s/terraform/deployment-server.tf b/twenty/code/k8s/terraform/deployment-server.tf index 3add086..a8bb43b 100644 --- a/twenty/code/k8s/terraform/deployment-server.tf +++ b/twenty/code/k8s/terraform/deployment-server.tf @@ -38,13 +38,9 @@ resource "kubernetes_deployment" "twentycrm_server" { tty = true env { - name = "PORT" + name = "NODE_PORT" value = "3000" } - # env { - # name = "DEBUG_MODE" - # value = false - # } env { name = "SERVER_URL" @@ -68,10 +64,6 @@ resource "kubernetes_deployment" "twentycrm_server" { name = "STORAGE_TYPE" value = "local" } - env { - name = "MESSAGE_QUEUE_TYPE" - value = "bull-mq" - } env { name = "ACCESS_TOKEN_EXPIRES_IN" value = "7d" diff --git a/twenty/code/k8s/terraform/deployment-worker.tf b/twenty/code/k8s/terraform/deployment-worker.tf index f9f83e7..9e4da3e 100644 --- a/twenty/code/k8s/terraform/deployment-worker.tf +++ b/twenty/code/k8s/terraform/deployment-worker.tf @@ -48,11 +48,6 @@ resource "kubernetes_deployment" "twentycrm_worker" { value = "postgres://twenty:${var.twentycrm_pgdb_admin_password}@${kubernetes_service.twentycrm_db.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local/default" } - env { - name = "CACHE_STORAGE_TYPE" - value = "redis" - } - env { name = "REDIS_URL" value = "redis://${kubernetes_service.twentycrm_redis.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local:6379" @@ -67,10 +62,6 @@ resource "kubernetes_deployment" "twentycrm_worker" { name = "STORAGE_TYPE" value = "local" } - env { - name = "MESSAGE_QUEUE_TYPE" - value = "bull-mq" - } env { name = "APP_SECRET" diff --git a/twenty/update.js b/twenty/update.js index e9f9d3c..b6131b7 100644 --- a/twenty/update.js +++ b/twenty/update.js @@ -5,3 +5,21 @@ await utils.copyDir("./repo/packages/twenty-docker", "./code"); await utils.removeContainerNames("./code/docker-compose.yml"); await utils.removePorts("./code/docker-compose.yml"); + +await utils.searchReplace( + "./code/.env.example", + "#REDIS_URL=redis://redis:6379", + "REDIS_URL=redis://redis:6379" +); + +await utils.searchReplace( + "./code/.env.example", + "SERVER_URL=http://localhost:3000", + "SERVER_URL=https://$(PRIMARY_DOMAIN)" +); + +await utils.searchReplace( + "./code/.env.example", + "# APP_SECRET=replace_me_with_a_random_string", + "APP_SECRET=replace_me_with_a_random_string" +); diff --git a/utils.js b/utils.js index 4c7d769..d22ef20 100644 --- a/utils.js +++ b/utils.js @@ -72,6 +72,15 @@ async function renameFile(src, dest) { await execa("mv", [src, dest]); } +async function searchReplace(path, search, replace) { + console.log(`Searching and replacing ${search} with ${replace} in ${path}`); + + const file = await fs.promises.readFile(path, "utf8"); + const newFile = file.replaceAll(search, replace); + + await fs.promises.writeFile(path, newFile); +} + export default { cloneOrPullRepo, removeContainerNames, @@ -79,4 +88,5 @@ export default { copyDir, downloadFile, renameFile, + searchReplace, };