From 86739bea8b293719e17eca6737e28dd447beb4c4 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Fri, 1 Nov 2024 20:59:40 +0800 Subject: [PATCH 1/3] fix(tools): suppress RuntimeWarnings in podcast audio generator (#10182) --- .../podcast_generator/tools/podcast_audio_generator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/core/tools/provider/builtin/podcast_generator/tools/podcast_audio_generator.py b/api/core/tools/provider/builtin/podcast_generator/tools/podcast_audio_generator.py index 2300b69e49..476e2d01e1 100644 --- a/api/core/tools/provider/builtin/podcast_generator/tools/podcast_audio_generator.py +++ b/api/core/tools/provider/builtin/podcast_generator/tools/podcast_audio_generator.py @@ -1,8 +1,8 @@ import concurrent.futures import io import random +import warnings from typing import Any, Literal, Optional, Union -from warnings import catch_warnings import openai @@ -10,7 +10,8 @@ from core.tools.entities.tool_entities import ToolInvokeMessage from core.tools.errors import ToolParameterValidationError, ToolProviderCredentialValidationError from core.tools.tool.builtin_tool import BuiltinTool -with catch_warnings(action="ignore", category=RuntimeWarning): +with warnings.catch_warnings(): + warnings.simplefilter("ignore") from pydub import AudioSegment From 53a7cb0e9ddb5a5b04d8c4f48033c67edae32be8 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Fri, 1 Nov 2024 23:19:11 +0800 Subject: [PATCH 2/3] feat(document_extractor): integrate unstructured API for PPTX extraction (#10180) --- api/core/workflow/nodes/document_extractor/node.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/api/core/workflow/nodes/document_extractor/node.py b/api/core/workflow/nodes/document_extractor/node.py index c2f51ad1e5..aacee94095 100644 --- a/api/core/workflow/nodes/document_extractor/node.py +++ b/api/core/workflow/nodes/document_extractor/node.py @@ -6,12 +6,14 @@ import docx import pandas as pd import pypdfium2 import yaml +from unstructured.partition.api import partition_via_api from unstructured.partition.email import partition_email from unstructured.partition.epub import partition_epub from unstructured.partition.msg import partition_msg from unstructured.partition.ppt import partition_ppt from unstructured.partition.pptx import partition_pptx +from configs import dify_config from core.file import File, FileTransferMethod, file_manager from core.helper import ssrf_proxy from core.variables import ArrayFileSegment @@ -263,7 +265,14 @@ def _extract_text_from_ppt(file_content: bytes) -> str: def _extract_text_from_pptx(file_content: bytes) -> str: try: with io.BytesIO(file_content) as file: - elements = partition_pptx(file=file) + if dify_config.UNSTRUCTURED_API_URL and dify_config.UNSTRUCTURED_API_KEY: + elements = partition_via_api( + file=file, + api_url=dify_config.UNSTRUCTURED_API_URL, + api_key=dify_config.UNSTRUCTURED_API_KEY, + ) + else: + elements = partition_pptx(file=file) return "\n".join([getattr(element, "text", "") for element in elements]) except Exception as e: raise TextExtractionError(f"Failed to extract text from PPTX: {str(e)}") from e From 091d609b58cd91913effa308eb27e4a31f50a30f Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Sat, 2 Nov 2024 09:37:28 +0800 Subject: [PATCH 3/3] fix: webapp upload file --- web/app/components/base/file-uploader/hooks.ts | 2 +- web/service/common.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/app/components/base/file-uploader/hooks.ts b/web/app/components/base/file-uploader/hooks.ts index a78c414913..088160691b 100644 --- a/web/app/components/base/file-uploader/hooks.ts +++ b/web/app/components/base/file-uploader/hooks.ts @@ -216,7 +216,7 @@ export const useFile = (fileConfig: FileUpload) => { handleAddFile(uploadingFile) startProgressTimer(uploadingFile.id) - uploadRemoteFileInfo(url).then((res) => { + uploadRemoteFileInfo(url, !!params.token).then((res) => { const newFile = { ...uploadingFile, type: res.mime_type, diff --git a/web/service/common.ts b/web/service/common.ts index 4ea2d9fd27..81b96aa97c 100644 --- a/web/service/common.ts +++ b/web/service/common.ts @@ -324,8 +324,8 @@ export const verifyForgotPasswordToken: Fetcher = ({ url, body }) => post(url, { body }) -export const uploadRemoteFileInfo = (url: string) => { - return post<{ id: string; name: string; size: number; mime_type: string; url: string }>('/remote-files/upload', { body: { url } }) +export const uploadRemoteFileInfo = (url: string, isPublic?: boolean) => { + return post<{ id: string; name: string; size: number; mime_type: string; url: string }>('/remote-files/upload', { body: { url } }, { isPublicAPI: isPublic }) } export const sendEMailLoginCode = (email: string, language = 'en-US') =>