From 27f0ae84167b91b7ea818fe6dc518b997b203626 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Sat, 22 Jun 2024 01:34:08 +0800 Subject: [PATCH] build: support Poetry for depencencies tool in api's Dockerfile (#5105) Co-authored-by: takatost --- .github/workflows/api-tests.yml | 2 +- .github/workflows/build-push.yml | 113 ++++++++++++++++++++++++++----- .github/workflows/style.yml | 5 +- api/.dockerignore | 5 +- api/Dockerfile | 33 ++++++--- 5 files changed, 128 insertions(+), 30 deletions(-) diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 4af4daadeb..612d698e3b 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -108,7 +108,7 @@ jobs: - name: Poetry check run: | - poetry check -C api + poetry check -C api --lock poetry show -C api - name: Install dependencies diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 89d301c4fd..2678f23a77 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -8,6 +8,10 @@ on: release: types: [published] +concurrency: + group: build-push-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + env: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} @@ -15,19 +19,35 @@ env: DIFY_API_IMAGE_NAME: ${{ vars.DIFY_API_IMAGE_NAME || 'langgenius/dify-api' }} jobs: - build-and-push: - runs-on: ubuntu-latest + build: + runs-on: ${{ matrix.platform == 'linux/arm64' && 'arm64_runner' || 'ubuntu-latest' }} if: github.repository == 'langgenius/dify' strategy: matrix: include: - - service_name: "web" - image_name_env: "DIFY_WEB_IMAGE_NAME" - context: "web" - - service_name: "api" + - service_name: "build-api-amd64" image_name_env: "DIFY_API_IMAGE_NAME" context: "api" + platform: linux/amd64 + - service_name: "build-api-arm64" + image_name_env: "DIFY_API_IMAGE_NAME" + context: "api" + platform: linux/arm64 + - service_name: "build-web-amd64" + image_name_env: "DIFY_WEB_IMAGE_NAME" + context: "web" + platform: linux/amd64 + - service_name: "build-web-arm64" + image_name_env: "DIFY_WEB_IMAGE_NAME" + context: "web" + platform: linux/arm64 + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -40,7 +60,66 @@ jobs: username: ${{ env.DOCKERHUB_USER }} password: ${{ env.DOCKERHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env[matrix.image_name_env] }} + + - name: Build Docker image + id: build + uses: docker/build-push-action@v6 + with: + context: "{{defaultContext}}:${{ matrix.context }}" + platforms: ${{ matrix.platform }} + build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env[matrix.image_name_env] }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha,scope=${{ matrix.service_name }} + cache-to: type=gha,mode=max,scope=${{ matrix.service_name }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + create-manifest: + needs: build + runs-on: ubuntu-latest + if: github.repository == 'langgenius/dify' + strategy: + matrix: + include: + - service_name: "merge-api-images" + image_name_env: "DIFY_API_IMAGE_NAME" + context: "api" + - service_name: "merge-web-images" + image_name_env: "DIFY_WEB_IMAGE_NAME" + context: "web" + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-${{ matrix.context }}-* + merge-multiple: true + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ env.DOCKERHUB_USER }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Extract metadata for Docker id: meta uses: docker/metadata-action@v5 with: @@ -51,14 +130,12 @@ jobs: type=sha,enable=true,priority=100,prefix=,suffix=,format=long type=raw,value=${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }} - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: "{{defaultContext}}:${{ matrix.context }}" - platforms: ${{ startsWith(github.ref, 'refs/tags/') && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env[matrix.image_name_env] }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env[matrix.image_name_env] }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index cc4302117f..f6092c8633 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -99,7 +99,7 @@ jobs: **.sh **.yaml **.yml - Dockerfile + **Dockerfile dev/** - name: Super-linter @@ -113,7 +113,8 @@ jobs: IGNORE_GITIGNORED_FILES: true VALIDATE_BASH: true VALIDATE_BASH_EXEC: true - VALIDATE_GITHUB_ACTIONS: true + # FIXME: temporarily disabled until api-docker.yaml's run script is fixed for shellcheck + # VALIDATE_GITHUB_ACTIONS: true VALIDATE_DOCKERFILE_HADOLINT: true VALIDATE_XML: true VALIDATE_YAML: true diff --git a/api/.dockerignore b/api/.dockerignore index 0ee003b912..91a5254ea7 100644 --- a/api/.dockerignore +++ b/api/.dockerignore @@ -8,4 +8,7 @@ logs *.log* # jetbrains -.idea \ No newline at end of file +.idea + +# venv +.venv \ No newline at end of file diff --git a/api/Dockerfile b/api/Dockerfile index 96b230e173..15fd9d88e0 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,18 +1,28 @@ # base image -FROM python:3.10-slim-bookworm AS base +FROM python:3.10-slim-bookworm as base -LABEL maintainer="takatost@gmail.com" +WORKDIR /app/api + +# Install Poetry +ENV POETRY_VERSION=1.8.3 +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir --upgrade poetry==${POETRY_VERSION} + +# Configure Poetry +ENV POETRY_CACHE_DIR=/tmp/poetry_cache +ENV POETRY_NO_INTERACTION=1 +ENV POETRY_VIRTUALENVS_IN_PROJECT=true +ENV POETRY_VIRTUALENVS_CREATE=true -# install packages FROM base as packages RUN apt-get update \ && apt-get install -y --no-install-recommends gcc g++ libc-dev libffi-dev libgmp-dev libmpfr-dev libmpc-dev -COPY requirements.txt /requirements.txt +# Install Python dependencies +COPY pyproject.toml poetry.lock ./ +RUN poetry install --sync --no-cache --no-root -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --prefix=/pkg -r requirements.txt # production stage FROM base AS production @@ -37,13 +47,20 @@ RUN apt-get update \ && apt-get autoremove \ && rm -rf /var/lib/apt/lists/* -COPY --from=packages /pkg /usr/local +# Copy Python environment and packages +ENV VIRTUAL_ENV=/app/api/.venv +COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +# Copy source code COPY . /app/api/ +# Copy entrypoint COPY docker/entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh + ARG COMMIT_SHA ENV COMMIT_SHA ${COMMIT_SHA} -ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]