Austin/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2024-11-16 11:42:29 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

fix: non utf8 code decode close #10691 (#10698)

Browse Source 
Signed-off-by: yihong0618 <zouzou0208@gmail.com>
This commit is contained in:
yihong 2024-11-14 17:29:49 +08:00 committed by GitHub
parent fbb9c1c249
commit 722964667f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 15 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
api/core/workflow/nodes/document_extractor/node.py
View File

@ -143,14 +143,14 @@ def _extract_text_by_file_extension(*, file_content: bytes, file_extension: str)
def _extract_text_from_plain_text(file_content: bytes) -> str: def _extract_text_from_plain_text(file_content: bytes) -> str:
try: try:
return file_content.decode("utf-8") return file_content.decode("utf-8", "ignore")
except UnicodeDecodeError as e: except UnicodeDecodeError as e:
raise TextExtractionError("Failed to decode plain text file") from e raise TextExtractionError("Failed to decode plain text file") from e
def _extract_text_from_json(file_content: bytes) -> str: def _extract_text_from_json(file_content: bytes) -> str:
try: try:
json_data = json.loads(file_content.decode("utf-8")) json_data = json.loads(file_content.decode("utf-8", "ignore"))
return json.dumps(json_data, indent=2, ensure_ascii=False) return json.dumps(json_data, indent=2, ensure_ascii=False)
except (UnicodeDecodeError, json.JSONDecodeError) as e: except (UnicodeDecodeError, json.JSONDecodeError) as e:
raise TextExtractionError(f"Failed to decode or parse JSON file: {e}") from e raise TextExtractionError(f"Failed to decode or parse JSON file: {e}") from e
@ -159,7 +159,7 @@ def _extract_text_from_json(file_content: bytes) -> str:
def _extract_text_from_yaml(file_content: bytes) -> str: def _extract_text_from_yaml(file_content: bytes) -> str:
"""Extract the content from yaml file""" """Extract the content from yaml file"""
try: try:
yaml_data = yaml.safe_load_all(file_content.decode("utf-8")) yaml_data = yaml.safe_load_all(file_content.decode("utf-8", "ignore"))
return yaml.dump_all(yaml_data, allow_unicode=True, sort_keys=False) return yaml.dump_all(yaml_data, allow_unicode=True, sort_keys=False)
except (UnicodeDecodeError, yaml.YAMLError) as e: except (UnicodeDecodeError, yaml.YAMLError) as e:
raise TextExtractionError(f"Failed to decode or parse YAML file: {e}") from e raise TextExtractionError(f"Failed to decode or parse YAML file: {e}") from e
@ -217,7 +217,7 @@ def _extract_text_from_file(file: File):
def _extract_text_from_csv(file_content: bytes) -> str: def _extract_text_from_csv(file_content: bytes) -> str:
try: try:
csv_file = io.StringIO(file_content.decode("utf-8")) csv_file = io.StringIO(file_content.decode("utf-8", "ignore"))
csv_reader = csv.reader(csv_file) csv_reader = csv.reader(csv_file)
rows = list(csv_reader) rows = list(csv_reader)

11
api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py
View File

@ -140,6 +140,17 @@ def test_extract_text_from_plain_text():
assert text == "Hello, world!" assert text == "Hello, world!"
def tet_extract_text_from_plain_text_non_utf8():
import tempfile
non_utf8_content = b"Hello world\xa9." # \xA9 represents © in Latin-1
with tempfile.NamedTemporaryFile(delete=True) as temp_file:
temp_file.write(non_utf8_content)
temp_file.seek(0)
text = _extract_text_from_plain_text(temp_file.read())
assert text == "Hello, world."
@patch("pypdfium2.PdfDocument") @patch("pypdfium2.PdfDocument")
def test_extract_text_from_pdf(mock_pdf_document): def test_extract_text_from_pdf(mock_pdf_document):
mock_page = Mock() mock_page = Mock()