From eab0ac3a13bea3a2f952fa42eccf2a12cd3e2508 Mon Sep 17 00:00:00 2001 From: takatost Date: Sun, 30 Jun 2024 10:31:31 +0800 Subject: [PATCH] chore: remove port expose in docker compose (#5754) Co-authored-by: Chenhe Gu --- .github/workflows/api-tests.yml | 3 ++ .github/workflows/expose_service_ports.sh | 10 ++++ docker/.env.example | 10 +++- docker/docker-compose.middleware.yaml | 64 ++++++++++++----------- docker/docker-compose.yaml | 49 ++--------------- docker/middleware.env.example | 9 ++++ 6 files changed, 66 insertions(+), 79 deletions(-) create mode 100755 .github/workflows/expose_service_ports.sh diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index e67b33bed5..e424171019 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -60,6 +60,9 @@ jobs: cp docker/.env.example docker/.env cp docker/middleware.env.example docker/middleware.env + - name: Expose Service Ports + run: sh .github/workflows/expose_service_ports.sh + - name: Set up Sandbox uses: hoverkraft-tech/compose-action@v2.0.0 with: diff --git a/.github/workflows/expose_service_ports.sh b/.github/workflows/expose_service_ports.sh new file mode 100755 index 0000000000..3418bf0c6f --- /dev/null +++ b/.github/workflows/expose_service_ports.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +yq eval '.services.weaviate.ports += ["8080:8080"]' -i docker/docker-compose.yaml +yq eval '.services.qdrant.ports += ["6333:6333"]' -i docker/docker-compose.yaml +yq eval '.services.chroma.ports += ["8000:8000"]' -i docker/docker-compose.yaml +yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/docker-compose.yaml +yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml +yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml + +echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs." \ No newline at end of file diff --git a/docker/.env.example b/docker/.env.example index 7ff82530fa..eb4a04351f 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -585,7 +585,7 @@ NGINX_PROXY_SEND_TIMEOUT=3600s # ------------------------------ # Environment Variables for SSRF Proxy # ------------------------------ -HTTP_PORT=3128 +SSRF_HTTP_PORT=3128 COREDUMP_DIR=/var/spool/squid REVERSE_PROXY_PORT=8194 SANDBOX_HOST=sandbox @@ -595,4 +595,10 @@ SANDBOX_HOST=sandbox # (based on the vector db type, the corresponding docker # compose profile will be used) # ------------------------------ -COMPOSE_PROFILES=${VECTOR_STORE:-weaviate} \ No newline at end of file +COMPOSE_PROFILES=${VECTOR_STORE:-weaviate} + +# ------------------------------ +# Docker Compose Service Expose Host Port Configurations +# ------------------------------ +EXPOSE_NGINX_PORT=80 +EXPOSE_NGINX_SSL_PORT=443 diff --git a/docker/docker-compose.middleware.yaml b/docker/docker-compose.middleware.yaml index d031e01bf7..ec0f5ba5e7 100644 --- a/docker/docker-compose.middleware.yaml +++ b/docker/docker-compose.middleware.yaml @@ -12,7 +12,7 @@ services: volumes: - ./volumes/db/data:/var/lib/postgresql/data ports: - - "5432:5432" + - "${EXPOSE_POSTGRES_PORT:-5432}:5432" # The redis cache. redis: @@ -24,32 +24,7 @@ services: # Set the redis password when startup redis server. command: redis-server --requirepass difyai123456 ports: - - "6379:6379" - - # The Weaviate vector store. - weaviate: - image: semitechnologies/weaviate:1.19.0 - restart: always - volumes: - # Mount the Weaviate data directory to the container. - - ./volumes/weaviate:/var/lib/weaviate - env_file: - - ./middleware.env - environment: - # The Weaviate configurations - # You can refer to the [Weaviate](https://weaviate.io/developers/weaviate/config-refs/env-vars) documentation for more information. - PERSISTENCE_DATA_PATH: ${PERSISTENCE_DATA_PATH:-'/var/lib/weaviate'} - QUERY_DEFAULTS_LIMIT: ${QUERY_DEFAULTS_LIMIT:-25} - AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: ${AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED:-false} - DEFAULT_VECTORIZER_MODULE: ${DEFAULT_VECTORIZER_MODULE:-none} - CLUSTER_HOSTNAME: ${CLUSTER_HOSTNAME:-node1} - AUTHENTICATION_APIKEY_ENABLED: ${AUTHENTICATION_APIKEY_ENABLED:-true} - AUTHENTICATION_APIKEY_ALLOWED_KEYS: ${AUTHENTICATION_APIKEY_ALLOWED_KEYS:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} - AUTHENTICATION_APIKEY_USERS: ${AUTHENTICATION_APIKEY_USERS:-hello@dify.ai} - AUTHORIZATION_ADMINLIST_ENABLED: ${AUTHORIZATION_ADMINLIST_ENABLED:-true} - AUTHORIZATION_ADMINLIST_USERS: ${AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai} - ports: - - "8080:8080" + - "${EXPOSE_REDIS_PORT:-6379}:6379" # The DifySandbox sandbox: @@ -81,20 +56,47 @@ services: - ./ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template - ./ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint.sh entrypoint: /docker-entrypoint.sh - ports: - - "3128:3128" - - "8194:8194" environment: # pls clearly modify the squid env vars to fit your network environment. - HTTP_PORT: ${HTTP_PORT:-3128} + HTTP_PORT: ${SSRF_HTTP_PORT:-3128} COREDUMP_DIR: ${COREDUMP_DIR:-/var/spool/squid} REVERSE_PROXY_PORT: ${REVERSE_PROXY_PORT:-8194} SANDBOX_HOST: ${SANDBOX_HOST:-sandbox} SANDBOX_PORT: ${SANDBOX_PORT:-8194} + ports: + - "${EXPOSE_SSRF_PROXY_PORT:-3128}:${SSRF_HTTP_PORT:-3128}" + - "${EXPOSE_SANDBOX_PORT:-8194}:${SANDBOX_PORT:-8194}" networks: - ssrf_proxy_network - default + # The Weaviate vector store. + weaviate: + image: semitechnologies/weaviate:1.19.0 + profiles: + - weaviate + restart: always + volumes: + # Mount the Weaviate data directory to the container. + - ./volumes/weaviate:/var/lib/weaviate + env_file: + - ./middleware.env + environment: + # The Weaviate configurations + # You can refer to the [Weaviate](https://weaviate.io/developers/weaviate/config-refs/env-vars) documentation for more information. + PERSISTENCE_DATA_PATH: ${PERSISTENCE_DATA_PATH:-'/var/lib/weaviate'} + QUERY_DEFAULTS_LIMIT: ${QUERY_DEFAULTS_LIMIT:-25} + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: ${AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED:-false} + DEFAULT_VECTORIZER_MODULE: ${DEFAULT_VECTORIZER_MODULE:-none} + CLUSTER_HOSTNAME: ${CLUSTER_HOSTNAME:-node1} + AUTHENTICATION_APIKEY_ENABLED: ${AUTHENTICATION_APIKEY_ENABLED:-true} + AUTHENTICATION_APIKEY_ALLOWED_KEYS: ${AUTHENTICATION_APIKEY_ALLOWED_KEYS:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} + AUTHENTICATION_APIKEY_USERS: ${AUTHENTICATION_APIKEY_USERS:-hello@dify.ai} + AUTHORIZATION_ADMINLIST_ENABLED: ${AUTHORIZATION_ADMINLIST_ENABLED:-true} + AUTHORIZATION_ADMINLIST_USERS: ${AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai} + ports: + - "${EXPOSE_WEAVIATE_PORT:-8080}:8080" + networks: # create a network between sandbox, api and ssrf_proxy, and can not access outside. ssrf_proxy_network: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 2d25f93392..0fbd8e24df 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -272,9 +272,6 @@ services: volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage - # uncomment to expose dify-api port to host - # ports: - # - "5001:5001" networks: - ssrf_proxy_network - default @@ -307,9 +304,6 @@ services: CONSOLE_API_URL: ${CONSOLE_API_URL:-} APP_API_URL: ${APP_API_URL:-} SENTRY_DSN: ${SENTRY_DSN:-} - # uncomment to expose dify-web port to host - # ports: - # - "3000:3000" # The postgres database. db: @@ -322,9 +316,6 @@ services: PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata} volumes: - ./volumes/db/data:/var/lib/postgresql/data - # uncomment to expose db(postgresql) port to host - # ports: - # - "5432:5432" healthcheck: test: [ "CMD", "pg_isready" ] interval: 1s @@ -342,9 +333,6 @@ services: command: redis-server --requirepass ${REDIS_PASSWORD:-difyai123456} healthcheck: test: [ "CMD", "redis-cli", "ping" ] - # uncomment to expose redis port to host - # ports: - # - "6379:6379" # The DifySandbox sandbox: @@ -378,7 +366,7 @@ services: entrypoint: ["sh", "-c", "chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh"] environment: # pls clearly modify the squid env vars to fit your network environment. - HTTP_PORT: ${HTTP_PORT:-3128} + HTTP_PORT: ${SSRF_HTTP_PORT:-3128} COREDUMP_DIR: ${COREDUMP_DIR:-/var/spool/squid} REVERSE_PROXY_PORT: ${REVERSE_PROXY_PORT:-8194} SANDBOX_HOST: ${SANDBOX_HOST:-sandbox} @@ -417,8 +405,8 @@ services: - api - web ports: - - "${NGINX_PORT:-80}:80" - - "${NGINX_SSL_PORT:-443}:443" + - "${EXPOSE_NGINX_PORT:-80}:80" + - "${EXPOSE_NGINX_SSL_PORT:-443}:443" # The Weaviate vector store. weaviate: @@ -442,10 +430,6 @@ services: AUTHENTICATION_APIKEY_USERS: ${AUTHENTICATION_APIKEY_USERS:-hello@dify.ai} AUTHORIZATION_ADMINLIST_ENABLED: ${AUTHORIZATION_ADMINLIST_ENABLED:-true} AUTHORIZATION_ADMINLIST_USERS: ${AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai} - # uncomment to expose weaviate port to host - ports: - - "8080:8080" - # Qdrant vector store. # (if used, you need to set VECTOR_STORE to qdrant in the api & worker service.) @@ -458,10 +442,6 @@ services: - ./volumes/qdrant:/qdrant/storage environment: QDRANT_API_KEY: ${QDRANT_API_KEY:-difyai123456} - # uncomment to expose qdrant port to host - ports: - - "6333:6333" - - "6334:6334" # The pgvector vector database. pgvector: @@ -479,9 +459,6 @@ services: PGDATA: ${PGVECTOR_PGDATA:-/var/lib/postgresql/data/pgdata} volumes: - ./volumes/pgvector/data:/var/lib/postgresql/data - # uncomment to expose db(postgresql) port to host - ports: - - "5433:5432" healthcheck: test: [ "CMD", "pg_isready" ] interval: 1s @@ -504,9 +481,6 @@ services: PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata} volumes: - ./volumes/pgvecto_rs/data:/var/lib/postgresql/data - # uncomment to expose db(postgresql) port to host - ports: - - "5431:5432" healthcheck: test: [ "CMD", "pg_isready" ] interval: 1s @@ -525,16 +499,12 @@ services: CHROMA_SERVER_AUTHN_CREDENTIALS: ${CHROMA_SERVER_AUTHN_CREDENTIALS:-difyai123456} CHROMA_SERVER_AUTHN_PROVIDER: ${CHROMA_SERVER_AUTHN_PROVIDER:-chromadb.auth.token_authn.TokenAuthenticationServerProvider} IS_PERSISTENT: ${IS_PERSISTENT:-TRUE} - ports: - - "8000:8000" oracle: image: container-registry.oracle.com/database/free:latest profiles: - oracle restart: always - ports: - - 1521:1521 volumes: - type: volume source: oradata @@ -574,9 +544,6 @@ services: environment: MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin} - ports: - - "9001:9001" - - "9000:9000" volumes: - ./volumes/milvus/minio:/minio_data command: minio server /minio_data --console-address ":9001" @@ -606,9 +573,6 @@ services: start_period: 90s timeout: 20s retries: 3 - ports: - - "19530:19530" - - "9091:9091" depends_on: - "etcd" - "minio" @@ -634,9 +598,6 @@ services: hard: ${OPENSEARCH_NOFILE_HARD:-65536} volumes: - ./volumes/opensearch/data:/usr/share/opensearch/data - ports: - - "9200:9200" - - "9600:9600" networks: - opensearch-net @@ -645,10 +606,6 @@ services: image: opensearchproject/opensearch-dashboards:latest profiles: - opensearch - ports: - - "5601:5601" - expose: - - "5601" environment: OPENSEARCH_HOSTS: '["https://opensearch:9200"]' volumes: diff --git a/docker/middleware.env.example b/docker/middleware.env.example index e17d67f612..051a79d54e 100644 --- a/docker/middleware.env.example +++ b/docker/middleware.env.example @@ -40,3 +40,12 @@ AUTHENTICATION_APIKEY_ALLOWED_KEYS=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih AUTHENTICATION_APIKEY_USERS=hello@dify.ai AUTHORIZATION_ADMINLIST_ENABLED=true AUTHORIZATION_ADMINLIST_USERS=hello@dify.ai + +# ------------------------------ +# Docker Compose Service Expose Host Port Configurations +# ------------------------------ +EXPOSE_POSTGRES_PORT=5432 +EXPOSE_REDIS_PORT=6379 +EXPOSE_SANDBOX_PORT=8194 +EXPOSE_SSRF_PROXY_PORT=3128 +EXPOSE_WEAVIATE_PORT=8080