From 9f637ead3890ad07e6dba64d45424b5820817ae7 Mon Sep 17 00:00:00 2001 From: takatost Date: Thu, 1 Feb 2024 18:11:57 +0800 Subject: [PATCH] bump version to 0.5.3 (#2306) --- api/app.py | 2 +- api/config.py | 2 +- api/constants/languages.py | 1 + api/controllers/console/__init__.py | 4 +- api/controllers/console/admin.py | 2 +- api/controllers/console/app/app.py | 3 +- api/controllers/console/app/site.py | 2 +- api/controllers/console/auth/activate.py | 2 +- api/controllers/console/datasets/file.py | 2 +- api/controllers/console/explore/message.py | 4 +- api/controllers/console/explore/parameter.py | 4 +- .../console/explore/recommended_app.py | 2 +- api/controllers/console/init_validate.py | 1 + api/controllers/console/setup.py | 2 +- api/controllers/console/workspace/account.py | 2 +- api/controllers/console/workspace/members.py | 7 ++- .../console/workspace/tool_providers.py | 14 +++--- api/controllers/console/wraps.py | 3 +- api/controllers/files/__init__.py | 3 +- api/controllers/files/tool_files.py | 4 +- api/controllers/service_api/app/app.py | 7 ++- .../service_api/dataset/dataset.py | 2 +- api/controllers/service_api/index.py | 3 +- api/controllers/web/app.py | 7 ++- api/core/app_runner/app_runner.py | 16 ++++--- api/core/app_runner/assistant_app_runner.py | 7 ++- api/core/app_runner/basic_app_runner.py | 3 +- api/core/app_runner/generate_task_pipeline.py | 14 +++--- api/core/application_manager.py | 7 +-- api/core/application_queue_manager.py | 8 ++-- .../agent_tool_callback_handler.py | 3 +- .../loader/unstructured/unstructured_ppt.py | 1 + api/core/embedding/cached_embedding.py | 3 +- api/core/entities/application_entities.py | 5 +- api/core/features/agent_runner.py | 17 ++++--- api/core/features/assistant_base_runner.py | 47 +++++++++---------- api/core/features/assistant_cot_runner.py | 23 ++++----- api/core/features/assistant_fc_runner.py | 19 +++----- api/core/file/message_file_parser.py | 4 +- api/core/hosting_configuration.py | 2 +- api/core/indexing_runner.py | 2 +- api/core/model_manager.py | 2 +- .../model_providers/__base/tts_model.py | 5 +- .../model_providers/bedrock/llm/llm.py | 11 ++--- .../model_providers/chatglm/llm/llm.py | 3 +- .../model_providers/cohere/llm/llm.py | 17 ++++--- .../cohere/text_embedding/text_embedding.py | 5 +- .../model_providers/minimax/llm/llm.py | 2 +- .../model_providers/ollama/llm/llm.py | 23 +++++---- .../ollama/text_embedding/text_embedding.py | 15 +++--- .../model_providers/openai/tts/tts.py | 9 ++-- .../model_providers/tongyi/llm/llm.py | 5 +- .../model_providers/tongyi/tts/tts.py | 15 +++--- .../model_providers/xinference/llm/llm.py | 8 ++-- .../text_embedding/text_embedding.py | 4 +- .../xinference/xinference_helper.py | 2 +- .../model_providers/zhipuai/llm/llm.py | 13 +++-- .../zhipuai/text_embedding/text_embedding.py | 2 +- .../zhipuai/zhipuai_sdk/__init__.py | 19 ++------ .../zhipuai/zhipuai_sdk/_client.py | 18 +++---- .../zhipuai_sdk/api_resource/__init__.py | 2 +- .../api_resource/chat/async_completions.py | 6 +-- .../zhipuai_sdk/api_resource/chat/chat.py | 5 +- .../api_resource/chat/completions.py | 4 +- .../zhipuai_sdk/api_resource/embeddings.py | 4 +- .../zhipuai/zhipuai_sdk/api_resource/files.py | 6 +-- .../api_resource/fine_tuning/fine_tuning.py | 3 +- .../api_resource/fine_tuning/jobs.py | 13 ++--- .../zhipuai_sdk/api_resource/images.py | 4 +- .../zhipuai/zhipuai_sdk/core/_base_api.py | 1 + .../zhipuai/zhipuai_sdk/core/_base_type.py | 14 +----- .../zhipuai/zhipuai_sdk/core/_files.py | 7 +-- .../zhipuai/zhipuai_sdk/core/_http_client.py | 10 +--- .../zhipuai/zhipuai_sdk/core/_request_opt.py | 8 ++-- .../zhipuai/zhipuai_sdk/core/_response.py | 4 +- .../zhipuai/zhipuai_sdk/core/_sse_client.py | 2 +- .../zhipuai/zhipuai_sdk/core/_utils.py | 2 +- .../zhipuai/zhipuai_sdk/types/embeddings.py | 4 +- .../zhipuai/zhipuai_sdk/types/file_object.py | 2 +- .../types/fine_tuning/fine_tuning_job.py | 4 +- .../fine_tuning/fine_tuning_job_event.py | 4 +- .../zhipuai/zhipuai_sdk/types/image.py | 2 +- api/core/tools/entities/tool_bundle.py | 7 +-- api/core/tools/entities/tool_entities.py | 5 +- api/core/tools/entities/user_entities.py | 5 +- api/core/tools/model/tool_model_manager.py | 22 ++++----- api/core/tools/provider/api_tool_provider.py | 11 +++-- api/core/tools/provider/app_tool_provider.py | 19 ++++---- api/core/tools/provider/builtin/_positions.py | 3 +- .../provider/builtin/azuredalle/azuredalle.py | 9 ++-- .../builtin/azuredalle/tools/dalle3.py | 8 ++-- .../tools/provider/builtin/chart/chart.py | 9 ++-- .../tools/provider/builtin/chart/tools/bar.py | 9 ++-- .../provider/builtin/chart/tools/line.py | 9 ++-- .../tools/provider/builtin/chart/tools/pie.py | 9 ++-- .../tools/provider/builtin/dalle/dalle.py | 9 ++-- .../provider/builtin/dalle/tools/dalle2.py | 8 ++-- .../provider/builtin/dalle/tools/dalle3.py | 8 ++-- .../tools/provider/builtin/gaode/gaode.py | 5 +- .../builtin/gaode/tools/gaode_weather.py | 7 +-- .../tools/provider/builtin/github/github.py | 2 +- .../github/tools/github_repositories.py | 10 ++-- .../tools/provider/builtin/google/google.py | 10 ++-- .../builtin/google/tools/google_search.py | 9 ++-- .../stablediffusion/stablediffusion.py | 9 ++-- .../stablediffusion/tools/stable_diffusion.py | 28 +++++------ api/core/tools/provider/builtin/time/time.py | 10 ++-- .../builtin/time/tools/current_time.py | 6 +-- .../builtin/vectorizer/tools/vectorizer.py | 15 +++--- .../provider/builtin/vectorizer/vectorizer.py | 10 ++-- .../builtin/webscraper/tools/webscraper.py | 5 +- .../provider/builtin/webscraper/webscraper.py | 10 ++-- .../wikipedia/tools/wikipedia_search.py | 9 ++-- .../provider/builtin/wikipedia/wikipedia.py | 4 +- .../wolframalpha/tools/wolframalpha.py | 8 ++-- .../builtin/wolframalpha/wolframalpha.py | 14 +++--- .../provider/builtin/yahoo/tools/analytics.py | 12 ++--- .../provider/builtin/yahoo/tools/news.py | 8 ++-- .../provider/builtin/yahoo/tools/ticker.py | 8 ++-- .../tools/provider/builtin/yahoo/yahoo.py | 4 +- .../provider/builtin/youtube/tools/videos.py | 8 ++-- .../tools/provider/builtin/youtube/youtube.py | 4 +- .../tools/provider/builtin_tool_provider.py | 28 +++++------ api/core/tools/provider/tool_provider.py | 13 +++-- api/core/tools/tool/api_tool.py | 14 +++--- api/core/tools/tool/builtin_tool.py | 15 +++--- api/core/tools/tool/dataset_retriever_tool.py | 11 +++-- api/core/tools/tool/tool.py | 16 +++---- api/core/tools/tool_file_manager.py | 24 +++++----- api/core/tools/tool_manager.py | 34 ++++++-------- api/core/tools/utils/configuration.py | 7 +-- api/core/tools/utils/encoder.py | 4 +- api/core/tools/utils/parser.py | 16 +++---- api/core/tools/utils/web_reader_tool.py | 13 +++-- api/libs/gmpy2_pkcs10aep_cipher.py | 17 +++---- api/libs/rsa.py | 2 +- ...ef91f18_rename_api_provider_description.py | 3 +- .../053da0c1d756_add_api_tool_privacy.py | 2 +- ...84c228_remove_tool_id_from_model_invoke.py | 2 +- ...9d_add_message_files_into_agent_thought.py | 3 +- ...a5a70d_add_tool_labels_to_agent_thought.py | 3 +- .../3ef9b2b6bee6_add_assistant_app.py | 2 +- .../versions/4823da1d26cf_add_tool_file.py | 2 +- ...fee_change_message_chain_id_to_nullable.py | 2 +- api/migrations/versions/89c7899ca936_.py | 3 +- ...daa_add_tool_conversation_variables_idx.py | 3 +- ...6f3c800_rename_api_provider_credentails.py | 3 +- ...fafbd60eca1_add_message_file_belongs_to.py | 3 +- .../ad472b61a054_add_api_provider_icon.py | 3 +- api/migrations/versions/b24be59fbb04_.py | 3 +- .../c71211c8f604_add_tool_invoke_model_log.py | 2 +- .../de95f5c77138_migration_serpapi_api_key.py | 4 +- .../versions/dfb3b7f477da_add_tool_index.py | 3 +- .../f25003750af4_add_created_updated_at.py | 3 +- api/models/model.py | 2 +- api/models/tools.py | 12 ++--- api/services/account_service.py | 2 +- api/services/audio_service.py | 6 +-- api/services/tools_manage_service.py | 25 +++++----- api/tasks/mail_invite_member_task.py | 3 +- .../model_runtime/bedrock/test_llm.py | 1 + .../model_runtime/bedrock/test_provider.py | 1 + .../model_runtime/cohere/test_llm.py | 3 +- .../model_runtime/ollama/test_llm.py | 9 ++-- .../ollama/test_text_embedding.py | 1 - .../model_runtime/zhipuai/test_llm.py | 4 +- .../tools/test_all_provider.py | 1 + docker/docker-compose.yaml | 6 +-- web/package.json | 2 +- 169 files changed, 591 insertions(+), 670 deletions(-) diff --git a/api/app.py b/api/app.py index caf4e8f459..cb3b226e4c 100644 --- a/api/app.py +++ b/api/app.py @@ -30,7 +30,7 @@ from flask import Flask, Response, request from flask_cors import CORS from libs.passport import PassportService # DO NOT REMOVE BELOW -from models import account, dataset, model, source, task, tool, web, tools +from models import account, dataset, model, source, task, tool, tools, web from services.account_service import AccountService # DO NOT REMOVE ABOVE diff --git a/api/config.py b/api/config.py index 385ea32494..66e4c90a6f 100644 --- a/api/config.py +++ b/api/config.py @@ -87,7 +87,7 @@ class Config: # ------------------------ # General Configurations. # ------------------------ - self.CURRENT_VERSION = "0.5.2" + self.CURRENT_VERSION = "0.5.3" self.COMMIT_SHA = get_env('COMMIT_SHA') self.EDITION = "SELF_HOSTED" self.DEPLOY_ENV = get_env('DEPLOY_ENV') diff --git a/api/constants/languages.py b/api/constants/languages.py index a2c5a8b683..6a6a3cf311 100644 --- a/api/constants/languages.py +++ b/api/constants/languages.py @@ -1,5 +1,6 @@ import json + from models.model import AppModelConfig languages = ['en-US', 'zh-Hans', 'pt-BR', 'es-ES', 'fr-FR', 'de-DE', 'ja-JP', 'ko-KR', 'ru-RU', 'it-IT'] diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py index 9642849732..ecfdc38612 100644 --- a/api/controllers/console/__init__.py +++ b/api/controllers/console/__init__.py @@ -11,11 +11,11 @@ from .app import (advanced_prompt_template, annotation, app, audio, completion, model_config, site, statistic) # Import auth controllers from .auth import activate, data_source_oauth, login, oauth +# Import billing controllers +from .billing import billing # Import datasets controllers from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing # Import explore controllers from .explore import audio, completion, conversation, installed_app, message, parameter, recommended_app, saved_message # Import workspace controllers from .workspace import account, members, model_providers, models, tool_providers, workspace -# Import billing controllers -from .billing import billing diff --git a/api/controllers/console/admin.py b/api/controllers/console/admin.py index 699605eb13..4b8d27434d 100644 --- a/api/controllers/console/admin.py +++ b/api/controllers/console/admin.py @@ -1,12 +1,12 @@ import os from functools import wraps +from constants.languages import supported_language from controllers.console import api from controllers.console.wraps import only_edition_cloud from extensions.ext_database import db from flask import request from flask_restful import Resource, reqparse -from constants.languages import supported_language from models.model import App, InstalledApp, RecommendedApp from werkzeug.exceptions import NotFound, Unauthorized diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index e81c0dccbc..2aac27af3e 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -3,8 +3,8 @@ import json import logging from datetime import datetime -from constants.model_template import model_templates from constants.languages import demo_model_templates, languages +from constants.model_template import model_templates from controllers.console import api from controllers.console.app.error import AppNotFoundError, ProviderNotInitializeError from controllers.console.setup import setup_required @@ -26,6 +26,7 @@ from models.tools import ApiToolProvider from services.app_model_config_service import AppModelConfigService from werkzeug.exceptions import Forbidden + def _get_app(app_id, tenant_id): app = db.session.query(App).filter(App.id == app_id, App.tenant_id == tenant_id).first() if not app: diff --git a/api/controllers/console/app/site.py b/api/controllers/console/app/site.py index 0dc3ecc1e6..daba012bd9 100644 --- a/api/controllers/console/app/site.py +++ b/api/controllers/console/app/site.py @@ -1,4 +1,5 @@ # -*- coding:utf-8 -*- +from constants.languages import supported_language from controllers.console import api from controllers.console.app import _get_app from controllers.console.setup import setup_required @@ -7,7 +8,6 @@ from extensions.ext_database import db from fields.app_fields import app_site_fields from flask_login import current_user from flask_restful import Resource, marshal_with, reqparse -from constants.languages import supported_language from libs.login import login_required from models.model import Site from werkzeug.exceptions import Forbidden, NotFound diff --git a/api/controllers/console/auth/activate.py b/api/controllers/console/auth/activate.py index 41832189a8..0b3672efc9 100644 --- a/api/controllers/console/auth/activate.py +++ b/api/controllers/console/auth/activate.py @@ -2,12 +2,12 @@ import base64 import secrets from datetime import datetime +from constants.languages import supported_language from controllers.console import api from controllers.console.error import AlreadyActivateError from extensions.ext_database import db from flask_restful import Resource, reqparse from libs.helper import email, str_len, timezone -from constants.languages import supported_language from libs.password import hash_password, valid_password from models.account import AccountStatus, Tenant from services.account_service import RegisterService diff --git a/api/controllers/console/datasets/file.py b/api/controllers/console/datasets/file.py index 476249d0f3..c15b3c0cd4 100644 --- a/api/controllers/console/datasets/file.py +++ b/api/controllers/console/datasets/file.py @@ -9,7 +9,7 @@ from flask import current_app, request from flask_login import current_user from flask_restful import Resource, marshal_with from libs.login import login_required -from services.file_service import FileService, ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS +from services.file_service import ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS, FileService PREVIEW_WORDS_LIMIT = 3000 diff --git a/api/controllers/console/explore/message.py b/api/controllers/console/explore/message.py index 404b855bb2..684ecd8b28 100644 --- a/api/controllers/console/explore/message.py +++ b/api/controllers/console/explore/message.py @@ -17,9 +17,9 @@ from core.model_runtime.errors.invoke import InvokeError from fields.message_fields import message_infinite_scroll_pagination_fields from flask import Response, stream_with_context from flask_login import current_user -from flask_restful import marshal_with, reqparse, fields +from flask_restful import fields, marshal_with, reqparse from flask_restful.inputs import int_range -from libs.helper import uuid_value, TimestampField +from libs.helper import TimestampField, uuid_value from services.completion_service import CompletionService from services.errors.app import MoreLikeThisDisabledError from services.errors.conversation import ConversationNotExistsError diff --git a/api/controllers/console/explore/parameter.py b/api/controllers/console/explore/parameter.py index 6bdfa34095..c073ebad01 100644 --- a/api/controllers/console/explore/parameter.py +++ b/api/controllers/console/explore/parameter.py @@ -3,12 +3,12 @@ import json from controllers.console import api from controllers.console.explore.wraps import InstalledAppResource +from extensions.ext_database import db from flask import current_app from flask_restful import fields, marshal_with -from models.model import InstalledApp, AppModelConfig +from models.model import AppModelConfig, InstalledApp from models.tools import ApiToolProvider -from extensions.ext_database import db class AppParameterApi(InstalledAppResource): """Resource for app variables.""" diff --git a/api/controllers/console/explore/recommended_app.py b/api/controllers/console/explore/recommended_app.py index 2f0fa51d77..4ce8fbfbe9 100644 --- a/api/controllers/console/explore/recommended_app.py +++ b/api/controllers/console/explore/recommended_app.py @@ -1,4 +1,5 @@ # -*- coding:utf-8 -*- +from constants.languages import languages from controllers.console import api from controllers.console.app.error import AppNotFoundError from controllers.console.wraps import account_initialization_required @@ -9,7 +10,6 @@ from libs.login import login_required from models.model import App, InstalledApp, RecommendedApp from services.account_service import TenantService from sqlalchemy import and_ -from constants.languages import languages app_fields = { 'id': fields.String, diff --git a/api/controllers/console/init_validate.py b/api/controllers/console/init_validate.py index ebe8b96d61..d1994a84c9 100644 --- a/api/controllers/console/init_validate.py +++ b/api/controllers/console/init_validate.py @@ -1,4 +1,5 @@ import os + from flask import current_app, session from flask_restful import Resource, reqparse from libs.helper import str_len diff --git a/api/controllers/console/setup.py b/api/controllers/console/setup.py index 040f9f23b7..765161ff9d 100644 --- a/api/controllers/console/setup.py +++ b/api/controllers/console/setup.py @@ -10,7 +10,7 @@ from models.model import DifySetup from services.account_service import AccountService, RegisterService, TenantService from . import api -from .error import AlreadySetupError, NotSetupError, NotInitValidateError +from .error import AlreadySetupError, NotInitValidateError, NotSetupError from .init_validate import get_init_validate_status from .wraps import only_edition_self_hosted diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index ae08c3a982..1f856394e2 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -2,6 +2,7 @@ from datetime import datetime import pytz +from constants.languages import supported_language from controllers.console import api from controllers.console.setup import setup_required from controllers.console.workspace.error import (AccountAlreadyInitedError, CurrentPasswordIncorrectError, @@ -12,7 +13,6 @@ from flask import current_app, request from flask_login import current_user from flask_restful import Resource, fields, marshal_with, reqparse from libs.helper import TimestampField, timezone -from constants.languages import supported_language from libs.login import login_required from models.account import AccountIntegrate, InvitationCode from services.account_service import AccountService diff --git a/api/controllers/console/workspace/members.py b/api/controllers/console/workspace/members.py index 1b7d08a879..84be878545 100644 --- a/api/controllers/console/workspace/members.py +++ b/api/controllers/console/workspace/members.py @@ -1,13 +1,12 @@ # -*- coding:utf-8 -*- -from flask import current_app -from flask_login import current_user -from flask_restful import Resource, abort, fields, marshal_with, reqparse - import services from controllers.console import api from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check from extensions.ext_database import db +from flask import current_app +from flask_login import current_user +from flask_restful import Resource, abort, fields, marshal_with, reqparse from libs.helper import TimestampField from libs.login import login_required from models.account import Account diff --git a/api/controllers/console/workspace/tool_providers.py b/api/controllers/console/workspace/tool_providers.py index 1d5fd74314..fb42146eee 100644 --- a/api/controllers/console/workspace/tool_providers.py +++ b/api/controllers/console/workspace/tool_providers.py @@ -1,18 +1,16 @@ +import io import json -from libs.login import login_required -from flask_login import current_user -from flask_restful import Resource, reqparse -from flask import send_file -from werkzeug.exceptions import Forbidden - from controllers.console import api from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required - +from flask import send_file +from flask_login import current_user +from flask_restful import Resource, reqparse +from libs.login import login_required from services.tools_manage_service import ToolManageService +from werkzeug.exceptions import Forbidden -import io class ToolProviderListApi(Resource): @setup_required diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index 7e01b26846..7bfb064a23 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -1,10 +1,9 @@ # -*- coding:utf-8 -*- import json from functools import wraps -from flask import request from controllers.console.workspace.error import AccountNotInitializedError -from flask import abort, current_app +from flask import abort, current_app, request from flask_login import current_user from services.feature_service import FeatureService from services.operation_service import OperationService diff --git a/api/controllers/files/__init__.py b/api/controllers/files/__init__.py index 7c3e848b53..c7bc7d26d2 100644 --- a/api/controllers/files/__init__.py +++ b/api/controllers/files/__init__.py @@ -6,5 +6,4 @@ bp = Blueprint('files', __name__) api = ExternalApi(bp) -from . import image_preview -from . import tool_files \ No newline at end of file +from . import image_preview, tool_files diff --git a/api/controllers/files/tool_files.py b/api/controllers/files/tool_files.py index f8b1936fc9..b4a290ec87 100644 --- a/api/controllers/files/tool_files.py +++ b/api/controllers/files/tool_files.py @@ -1,10 +1,10 @@ from controllers.files import api +from core.tools.tool_file_manager import ToolFileManager from flask import Response from flask_restful import Resource, reqparse from libs.exception import BaseHTTPException -from werkzeug.exceptions import NotFound, Forbidden +from werkzeug.exceptions import Forbidden, NotFound -from core.tools.tool_file_manager import ToolFileManager class ToolFilePreviewApi(Resource): def get(self, file_id, extension): diff --git a/api/controllers/service_api/app/app.py b/api/controllers/service_api/app/app.py index aa4323a146..63591f8f49 100644 --- a/api/controllers/service_api/app/app.py +++ b/api/controllers/service_api/app/app.py @@ -1,15 +1,14 @@ # -*- coding:utf-8 -*- +import json + from controllers.service_api import api from controllers.service_api.wraps import AppApiResource +from extensions.ext_database import db from flask import current_app from flask_restful import fields, marshal_with from models.model import App, AppModelConfig from models.tools import ApiToolProvider -import json - -from extensions.ext_database import db - class AppParameterApi(AppApiResource): """Resource for app variables.""" diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index 6827d47dfc..900a796674 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -1,4 +1,3 @@ -from models.dataset import Dataset import services.dataset_service from controllers.service_api import api from controllers.service_api.dataset.error import DatasetNameDuplicateError @@ -9,6 +8,7 @@ from fields.dataset_fields import dataset_detail_fields from flask import request from flask_restful import marshal, reqparse from libs.login import current_user +from models.dataset import Dataset from services.dataset_service import DatasetService diff --git a/api/controllers/service_api/index.py b/api/controllers/service_api/index.py index 932388b562..489018cf9b 100644 --- a/api/controllers/service_api/index.py +++ b/api/controllers/service_api/index.py @@ -1,8 +1,7 @@ +from controllers.service_api import api from flask import current_app from flask_restful import Resource -from controllers.service_api import api - class IndexApi(Resource): def get(self): diff --git a/api/controllers/web/app.py b/api/controllers/web/app.py index 913f7fbee1..82a9ad8683 100644 --- a/api/controllers/web/app.py +++ b/api/controllers/web/app.py @@ -1,15 +1,14 @@ # -*- coding:utf-8 -*- +import json + from controllers.web import api from controllers.web.wraps import WebApiResource +from extensions.ext_database import db from flask import current_app from flask_restful import fields, marshal_with from models.model import App, AppModelConfig from models.tools import ApiToolProvider -from extensions.ext_database import db - -import json - class AppParameterApi(WebApiResource): """Resource for app variables.""" diff --git a/api/core/app_runner/app_runner.py b/api/core/app_runner/app_runner.py index c7c5474b2a..d751e301cc 100644 --- a/api/core/app_runner/app_runner.py +++ b/api/core/app_runner/app_runner.py @@ -2,8 +2,13 @@ import time from typing import Generator, List, Optional, Tuple, Union, cast from core.application_queue_manager import ApplicationQueueManager, PublishFrom -from core.entities.application_entities import AppOrchestrationConfigEntity, ModelConfigEntity, \ - PromptTemplateEntity, ExternalDataVariableEntity, ApplicationGenerateEntity, InvokeFrom +from core.entities.application_entities import (ApplicationGenerateEntity, AppOrchestrationConfigEntity, + ExternalDataVariableEntity, InvokeFrom, ModelConfigEntity, + PromptTemplateEntity) +from core.features.annotation_reply import AnnotationReplyFeature +from core.features.external_data_fetch import ExternalDataFetchFeature +from core.features.hosting_moderation import HostingModerationFeature +from core.features.moderation import ModerationFeature from core.file.file_obj import FileObj from core.memory.token_buffer_memory import TokenBufferMemory from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage @@ -11,12 +16,9 @@ from core.model_runtime.entities.message_entities import AssistantPromptMessage, from core.model_runtime.entities.model_entities import ModelPropertyKey from core.model_runtime.errors.invoke import InvokeBadRequestError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel -from core.features.hosting_moderation import HostingModerationFeature -from core.features.moderation import ModerationFeature -from core.features.external_data_fetch import ExternalDataFetchFeature -from core.features.annotation_reply import AnnotationReplyFeature from core.prompt.prompt_transform import PromptTransform -from models.model import App, MessageAnnotation, Message +from models.model import App, Message, MessageAnnotation + class AppRunner: def get_pre_calculate_rest_tokens(self, app_record: App, diff --git a/api/core/app_runner/assistant_app_runner.py b/api/core/app_runner/assistant_app_runner.py index b7af18abf0..d0b9bb872c 100644 --- a/api/core/app_runner/assistant_app_runner.py +++ b/api/core/app_runner/assistant_app_runner.py @@ -3,11 +3,10 @@ import logging from typing import cast from core.app_runner.app_runner import AppRunner +from core.application_queue_manager import ApplicationQueueManager, PublishFrom +from core.entities.application_entities import AgentEntity, ApplicationGenerateEntity, ModelConfigEntity from core.features.assistant_cot_runner import AssistantCotApplicationRunner from core.features.assistant_fc_runner import AssistantFunctionCallApplicationRunner -from core.entities.application_entities import ApplicationGenerateEntity, ModelConfigEntity, \ - AgentEntity -from core.application_queue_manager import ApplicationQueueManager, PublishFrom from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance from core.model_runtime.entities.llm_entities import LLMUsage @@ -16,7 +15,7 @@ from core.model_runtime.model_providers.__base.large_language_model import Large from core.moderation.base import ModerationException from core.tools.entities.tool_entities import ToolRuntimeVariablePool from extensions.ext_database import db -from models.model import Conversation, Message, App, MessageChain, MessageAgentThought +from models.model import App, Conversation, Message, MessageAgentThought, MessageChain from models.tools import ToolConversationVariables logger = logging.getLogger(__name__) diff --git a/api/core/app_runner/basic_app_runner.py b/api/core/app_runner/basic_app_runner.py index 0864786530..ae2b712187 100644 --- a/api/core/app_runner/basic_app_runner.py +++ b/api/core/app_runner/basic_app_runner.py @@ -4,8 +4,7 @@ from typing import Optional from core.app_runner.app_runner import AppRunner from core.application_queue_manager import ApplicationQueueManager, PublishFrom from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler -from core.entities.application_entities import (ApplicationGenerateEntity, DatasetEntity, - InvokeFrom, ModelConfigEntity) +from core.entities.application_entities import ApplicationGenerateEntity, DatasetEntity, InvokeFrom, ModelConfigEntity from core.features.dataset_retrieval import DatasetRetrievalFeature from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance diff --git a/api/core/app_runner/generate_task_pipeline.py b/api/core/app_runner/generate_task_pipeline.py index 4bb8b1dd70..964228bf9b 100644 --- a/api/core/app_runner/generate_task_pipeline.py +++ b/api/core/app_runner/generate_task_pipeline.py @@ -6,21 +6,21 @@ from typing import Generator, Optional, Union, cast from core.app_runner.moderation_handler import ModerationRule, OutputModerationHandler from core.application_queue_manager import ApplicationQueueManager, PublishFrom from core.entities.application_entities import ApplicationGenerateEntity, InvokeFrom -from core.entities.queue_entities import (AnnotationReplyEvent, QueueAgentThoughtEvent, QueueErrorEvent, - QueueMessageEndEvent, QueueMessageEvent, QueueMessageReplaceEvent, - QueuePingEvent, QueueRetrieverResourcesEvent, QueueStopEvent, - QueueMessageFileEvent, QueueAgentMessageEvent) -from core.errors.error import ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError +from core.entities.queue_entities import (AnnotationReplyEvent, QueueAgentMessageEvent, QueueAgentThoughtEvent, + QueueErrorEvent, QueueMessageEndEvent, QueueMessageEvent, + QueueMessageFileEvent, QueueMessageReplaceEvent, QueuePingEvent, + QueueRetrieverResourcesEvent, QueueStopEvent) +from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent, PromptMessage, PromptMessageContentType, PromptMessageRole, TextPromptMessageContent) from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel -from core.tools.tool_file_manager import ToolFileManager -from core.tools.tool_manager import ToolManager from core.model_runtime.utils.encoders import jsonable_encoder from core.prompt.prompt_template import PromptTemplateParser +from core.tools.tool_file_manager import ToolFileManager +from core.tools.tool_manager import ToolManager from events.message_event import message_was_created from extensions.ext_database import db from models.model import Conversation, Message, MessageAgentThought, MessageFile diff --git a/api/core/application_manager.py b/api/core/application_manager.py index 86c0bc0d0d..7f07bed3a5 100644 --- a/api/core/application_manager.py +++ b/api/core/application_manager.py @@ -9,11 +9,12 @@ from core.app_runner.basic_app_runner import BasicApplicationRunner from core.app_runner.generate_task_pipeline import GenerateTaskPipeline from core.application_queue_manager import ApplicationQueueManager, ConversationTaskStoppedException, PublishFrom from core.entities.application_entities import (AdvancedChatPromptTemplateEntity, - AdvancedCompletionPromptTemplateEntity, AgentEntity, AgentToolEntity, - ApplicationGenerateEntity, AppOrchestrationConfigEntity, DatasetEntity, + AdvancedCompletionPromptTemplateEntity, AgentEntity, AgentPromptEntity, + AgentToolEntity, ApplicationGenerateEntity, + AppOrchestrationConfigEntity, DatasetEntity, DatasetRetrieveConfigEntity, ExternalDataVariableEntity, FileUploadEntity, InvokeFrom, ModelConfigEntity, PromptTemplateEntity, - SensitiveWordAvoidanceEntity, AgentPromptEntity) + SensitiveWordAvoidanceEntity) from core.entities.model_entities import ModelStatus from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from core.file.file_obj import FileObj diff --git a/api/core/application_queue_manager.py b/api/core/application_queue_manager.py index 605255f3bf..65b52fd1f3 100644 --- a/api/core/application_queue_manager.py +++ b/api/core/application_queue_manager.py @@ -4,10 +4,10 @@ from enum import Enum from typing import Any, Generator from core.entities.application_entities import InvokeFrom -from core.entities.queue_entities import (AnnotationReplyEvent, AppQueueEvent, QueueAgentThoughtEvent, QueueErrorEvent, - QueueMessage, QueueMessageEndEvent, QueueMessageEvent, - QueueMessageReplaceEvent, QueuePingEvent, QueueRetrieverResourcesEvent, - QueueStopEvent, QueueMessageFileEvent, QueueAgentMessageEvent) +from core.entities.queue_entities import (AnnotationReplyEvent, AppQueueEvent, QueueAgentMessageEvent, + QueueAgentThoughtEvent, QueueErrorEvent, QueueMessage, QueueMessageEndEvent, + QueueMessageEvent, QueueMessageFileEvent, QueueMessageReplaceEvent, + QueuePingEvent, QueueRetrieverResourcesEvent, QueueStopEvent) from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk from extensions.ext_redis import redis_client from models.model import MessageAgentThought, MessageFile diff --git a/api/core/callback_handler/agent_tool_callback_handler.py b/api/core/callback_handler/agent_tool_callback_handler.py index c1037035ee..ae77bf6cd1 100644 --- a/api/core/callback_handler/agent_tool_callback_handler.py +++ b/api/core/callback_handler/agent_tool_callback_handler.py @@ -1,9 +1,10 @@ import os from typing import Any, Dict, Optional, Union -from pydantic import BaseModel from langchain.callbacks.base import BaseCallbackHandler from langchain.input import print_text +from pydantic import BaseModel + class DifyAgentCallbackHandler(BaseCallbackHandler, BaseModel): """Callback Handler that prints to std out.""" diff --git a/api/core/data_loader/loader/unstructured/unstructured_ppt.py b/api/core/data_loader/loader/unstructured/unstructured_ppt.py index a0d918cce7..9b1e6b5abf 100644 --- a/api/core/data_loader/loader/unstructured/unstructured_ppt.py +++ b/api/core/data_loader/loader/unstructured/unstructured_ppt.py @@ -1,5 +1,6 @@ import logging from typing import List + from langchain.document_loaders.base import BaseLoader from langchain.schema import Document diff --git a/api/core/embedding/cached_embedding.py b/api/core/embedding/cached_embedding.py index ba25eaa787..185b87b8b6 100644 --- a/api/core/embedding/cached_embedding.py +++ b/api/core/embedding/cached_embedding.py @@ -8,9 +8,8 @@ from core.model_manager import ModelInstance from core.model_runtime.entities.model_entities import ModelPropertyKey from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel from extensions.ext_database import db -from langchain.embeddings.base import Embeddings - from extensions.ext_redis import redis_client +from langchain.embeddings.base import Embeddings from libs import helper from models.dataset import Embedding from sqlalchemy.exc import IntegrityError diff --git a/api/core/entities/application_entities.py b/api/core/entities/application_entities.py index 56172188e4..6883a004e4 100644 --- a/api/core/entities/application_entities.py +++ b/api/core/entities/application_entities.py @@ -1,12 +1,11 @@ from enum import Enum -from typing import Optional, Any, cast, Literal, Union - -from pydantic import BaseModel +from typing import Any, Literal, Optional, Union, cast from core.entities.provider_configuration import ProviderModelBundle from core.file.file_obj import FileObj from core.model_runtime.entities.message_entities import PromptMessageRole from core.model_runtime.entities.model_entities import AIModelEntity +from pydantic import BaseModel class ModelConfigEntity(BaseModel): diff --git a/api/core/features/agent_runner.py b/api/core/features/agent_runner.py index 2a7e373dbf..66d41dace0 100644 --- a/api/core/features/agent_runner.py +++ b/api/core/features/agent_runner.py @@ -1,27 +1,26 @@ import logging -from typing import cast, Optional, List - -from langchain import WikipediaAPIWrapper -from langchain.callbacks.base import BaseCallbackHandler -from langchain.tools import BaseTool, WikipediaQueryRun, Tool -from pydantic import BaseModel, Field +from typing import List, Optional, cast from core.agent.agent.agent_llm_callback import AgentLLMCallback -from core.agent.agent_executor import PlanningStrategy, AgentConfiguration, AgentExecutor +from core.agent.agent_executor import AgentConfiguration, AgentExecutor, PlanningStrategy from core.application_queue_manager import ApplicationQueueManager from core.callback_handler.agent_loop_gather_callback_handler import AgentLoopGatherCallbackHandler from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler from core.callback_handler.std_out_callback_handler import DifyStdOutCallbackHandler -from core.entities.application_entities import ModelConfigEntity, InvokeFrom, \ - AgentEntity, AgentToolEntity, AppOrchestrationConfigEntity +from core.entities.application_entities import (AgentEntity, AgentToolEntity, AppOrchestrationConfigEntity, InvokeFrom, + ModelConfigEntity) from core.memory.token_buffer_memory import TokenBufferMemory from core.model_runtime.entities.model_entities import ModelFeature, ModelType from core.model_runtime.model_providers import model_provider_factory from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.tools.tool.dataset_retriever.dataset_retriever_tool import DatasetRetrieverTool from extensions.ext_database import db +from langchain import WikipediaAPIWrapper +from langchain.callbacks.base import BaseCallbackHandler +from langchain.tools import BaseTool, Tool, WikipediaQueryRun from models.dataset import Dataset from models.model import Message +from pydantic import BaseModel, Field logger = logging.getLogger(__name__) diff --git a/api/core/features/assistant_base_runner.py b/api/core/features/assistant_base_runner.py index e9ee59263c..5538918234 100644 --- a/api/core/features/assistant_base_runner.py +++ b/api/core/features/assistant_base_runner.py @@ -1,37 +1,32 @@ -import logging import json - -from typing import Optional, List, Tuple, Union, cast +import logging from datetime import datetime from mimetypes import guess_extension +from typing import List, Optional, Tuple, Union, cast from core.app_runner.app_runner import AppRunner -from extensions.ext_database import db - -from models.model import MessageAgentThought, Message, MessageFile -from models.tools import ToolConversationVariables - -from core.tools.entities.tool_entities import ToolInvokeMessage, ToolInvokeMessageBinary, \ - ToolRuntimeVariablePool, ToolParameter -from core.tools.tool.tool import Tool -from core.tools.tool_manager import ToolManager -from core.tools.tool_file_manager import ToolFileManager -from core.tools.tool.dataset_retriever_tool import DatasetRetrieverTool -from core.app_runner.app_runner import AppRunner +from core.application_queue_manager import ApplicationQueueManager from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler -from core.entities.application_entities import ModelConfigEntity, AgentEntity, AgentToolEntity -from core.application_queue_manager import ApplicationQueueManager -from core.memory.token_buffer_memory import TokenBufferMemory -from core.entities.application_entities import ModelConfigEntity, \ - AgentEntity, AppOrchestrationConfigEntity, ApplicationGenerateEntity, InvokeFrom -from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool -from core.model_runtime.entities.llm_entities import LLMUsage -from core.model_runtime.entities.model_entities import ModelFeature -from core.model_runtime.utils.encoders import jsonable_encoder -from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel -from core.model_manager import ModelInstance +from core.entities.application_entities import (AgentEntity, AgentToolEntity, ApplicationGenerateEntity, + AppOrchestrationConfigEntity, InvokeFrom, ModelConfigEntity) from core.file.message_file_parser import FileTransferMethod +from core.memory.token_buffer_memory import TokenBufferMemory +from core.model_manager import ModelInstance +from core.model_runtime.entities.llm_entities import LLMUsage +from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool +from core.model_runtime.entities.model_entities import ModelFeature +from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel +from core.model_runtime.utils.encoders import jsonable_encoder +from core.tools.entities.tool_entities import (ToolInvokeMessage, ToolInvokeMessageBinary, ToolParameter, + ToolRuntimeVariablePool) +from core.tools.tool.dataset_retriever_tool import DatasetRetrieverTool +from core.tools.tool.tool import Tool +from core.tools.tool_file_manager import ToolFileManager +from core.tools.tool_manager import ToolManager +from extensions.ext_database import db +from models.model import Message, MessageAgentThought, MessageFile +from models.tools import ToolConversationVariables logger = logging.getLogger(__name__) diff --git a/api/core/features/assistant_cot_runner.py b/api/core/features/assistant_cot_runner.py index f07af0e694..c9cb2ba61a 100644 --- a/api/core/features/assistant_cot_runner.py +++ b/api/core/features/assistant_cot_runner.py @@ -1,24 +1,21 @@ import json import logging import re -from typing import Literal, Union, Generator, Dict, List +from typing import Dict, Generator, List, Literal, Union -from core.entities.application_entities import AgentPromptEntity, AgentScratchpadUnit from core.application_queue_manager import PublishFrom -from core.model_runtime.utils.encoders import jsonable_encoder -from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage, \ - UserPromptMessage, SystemPromptMessage, AssistantPromptMessage -from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage, LLMResultChunk, LLMResultChunkDelta -from core.model_manager import ModelInstance - -from core.tools.errors import ToolInvokeError, ToolNotFoundError, \ - ToolNotSupportedError, ToolProviderNotFoundError, ToolParameterValidationError, \ - ToolProviderCredentialValidationError - +from core.entities.application_entities import AgentPromptEntity, AgentScratchpadUnit from core.features.assistant_base_runner import BaseAssistantApplicationRunner - +from core.model_manager import ModelInstance +from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, + SystemPromptMessage, UserPromptMessage) +from core.model_runtime.utils.encoders import jsonable_encoder +from core.tools.errors import (ToolInvokeError, ToolNotFoundError, ToolNotSupportedError, ToolParameterValidationError, + ToolProviderCredentialValidationError, ToolProviderNotFoundError) from models.model import Conversation, Message + class AssistantCotApplicationRunner(BaseAssistantApplicationRunner): def run(self, conversation: Conversation, message: Message, diff --git a/api/core/features/assistant_fc_runner.py b/api/core/features/assistant_fc_runner.py index b3340906b5..8b42244838 100644 --- a/api/core/features/assistant_fc_runner.py +++ b/api/core/features/assistant_fc_runner.py @@ -1,20 +1,15 @@ import json import logging +from typing import Any, Dict, Generator, List, Tuple, Union -from typing import Union, Generator, Dict, Any, Tuple, List - -from core.model_runtime.entities.message_entities import PromptMessage, UserPromptMessage,\ - SystemPromptMessage, AssistantPromptMessage, ToolPromptMessage, PromptMessageTool -from core.model_runtime.entities.llm_entities import LLMResultChunk, LLMResult, LLMUsage, LLMResultChunkDelta -from core.model_manager import ModelInstance from core.application_queue_manager import PublishFrom - -from core.tools.errors import ToolInvokeError, ToolNotFoundError, \ - ToolNotSupportedError, ToolProviderNotFoundError, ToolParameterValidationError, \ - ToolProviderCredentialValidationError - from core.features.assistant_base_runner import BaseAssistantApplicationRunner - +from core.model_manager import ModelInstance +from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, + SystemPromptMessage, ToolPromptMessage, UserPromptMessage) +from core.tools.errors import (ToolInvokeError, ToolNotFoundError, ToolNotSupportedError, ToolParameterValidationError, + ToolProviderCredentialValidationError, ToolProviderNotFoundError) from models.model import Conversation, Message, MessageAgentThought logger = logging.getLogger(__name__) diff --git a/api/core/file/message_file_parser.py b/api/core/file/message_file_parser.py index 1f35bb245d..c92f9e6950 100644 --- a/api/core/file/message_file_parser.py +++ b/api/core/file/message_file_parser.py @@ -1,11 +1,11 @@ from typing import Dict, List, Optional, Union import requests -from core.file.file_obj import FileObj, FileTransferMethod, FileType, FileBelongsTo -from services.file_service import IMAGE_EXTENSIONS +from core.file.file_obj import FileBelongsTo, FileObj, FileTransferMethod, FileType from extensions.ext_database import db from models.account import Account from models.model import AppModelConfig, EndUser, MessageFile, UploadFile +from services.file_service import IMAGE_EXTENSIONS class MessageFileParser: diff --git a/api/core/hosting_configuration.py b/api/core/hosting_configuration.py index 995bcb0622..a04eb0a5e8 100644 --- a/api/core/hosting_configuration.py +++ b/api/core/hosting_configuration.py @@ -2,7 +2,7 @@ from typing import Optional from core.entities.provider_entities import QuotaUnit, RestrictModel from core.model_runtime.entities.model_entities import ModelType -from flask import Flask, Config +from flask import Config, Flask from models.provider import ProviderQuotaType from pydantic import BaseModel diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 42c43a6643..2f1cf282f8 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -13,7 +13,7 @@ from core.docstore.dataset_docstore import DatasetDocumentStore from core.errors.error import ProviderTokenNotInitError from core.generator.llm_generator import LLMGenerator from core.index.index import IndexBuilder -from core.model_manager import ModelManager, ModelInstance +from core.model_manager import ModelInstance, ModelManager from core.model_runtime.entities.model_entities import ModelType, PriceType from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel diff --git a/api/core/model_manager.py b/api/core/model_manager.py index 20316510dd..8a622e4f5b 100644 --- a/api/core/model_manager.py +++ b/api/core/model_manager.py @@ -12,8 +12,8 @@ from core.model_runtime.model_providers.__base.large_language_model import Large from core.model_runtime.model_providers.__base.moderation_model import ModerationModel from core.model_runtime.model_providers.__base.rerank_model import RerankModel from core.model_runtime.model_providers.__base.speech2text_model import Speech2TextModel -from core.model_runtime.model_providers.__base.tts_model import TTSModel from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel +from core.model_runtime.model_providers.__base.tts_model import TTSModel from core.provider_manager import ProviderManager diff --git a/api/core/model_runtime/model_providers/__base/tts_model.py b/api/core/model_runtime/model_providers/__base/tts_model.py index f0edcff1a7..ff20cf7b9f 100644 --- a/api/core/model_runtime/model_providers/__base/tts_model.py +++ b/api/core/model_runtime/model_providers/__base/tts_model.py @@ -1,13 +1,12 @@ -import uuid import hashlib import subprocess +import uuid from abc import abstractmethod from typing import Optional +from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType from core.model_runtime.errors.invoke import InvokeBadRequestError -from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.model_providers.__base.ai_model import AIModel -from core.model_runtime.entities.model_entities import ModelPropertyKey class TTSModel(AIModel): diff --git a/api/core/model_runtime/model_providers/bedrock/llm/llm.py b/api/core/model_runtime/model_providers/bedrock/llm/llm.py index 19c4d25fa6..6a9c695350 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/llm.py +++ b/api/core/model_runtime/model_providers/bedrock/llm/llm.py @@ -1,15 +1,14 @@ +import json import logging from typing import Generator, List, Optional, Union import boto3 -from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError, ServiceNotInRegionError, UnknownServiceError from botocore.config import Config -import json - +from botocore.exceptions import (ClientError, EndpointConnectionError, NoRegionError, ServiceNotInRegionError, + UnknownServiceError) from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta - -from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, - PromptMessageTool, SystemPromptMessage, UserPromptMessage) +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, + SystemPromptMessage, UserPromptMessage) from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError diff --git a/api/core/model_runtime/model_providers/chatglm/llm/llm.py b/api/core/model_runtime/model_providers/chatglm/llm/llm.py index 471898fcf6..6f78f7aa88 100644 --- a/api/core/model_runtime/model_providers/chatglm/llm/llm.py +++ b/api/core/model_runtime/model_providers/chatglm/llm/llm.py @@ -5,7 +5,8 @@ from typing import Generator, List, Optional, cast from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageFunction, - PromptMessageTool, SystemPromptMessage, UserPromptMessage, ToolPromptMessage) + PromptMessageTool, SystemPromptMessage, ToolPromptMessage, + UserPromptMessage) from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError diff --git a/api/core/model_runtime/model_providers/cohere/llm/llm.py b/api/core/model_runtime/model_providers/cohere/llm/llm.py index 7b7687bc99..acff4177c3 100644 --- a/api/core/model_runtime/model_providers/cohere/llm/llm.py +++ b/api/core/model_runtime/model_providers/cohere/llm/llm.py @@ -1,19 +1,18 @@ import logging -from typing import Generator, List, Optional, Union, cast, Tuple +from typing import Generator, List, Optional, Tuple, Union, cast import cohere from cohere.responses import Chat, Generations -from cohere.responses.chat import StreamingChat, StreamTextGeneration, StreamEnd -from cohere.responses.generation import StreamingText, StreamingGenerations - +from cohere.responses.chat import StreamEnd, StreamingChat, StreamTextGeneration +from cohere.responses.generation import StreamingGenerations, StreamingText from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, - PromptMessageContentType, SystemPromptMessage, - TextPromptMessageContent, UserPromptMessage, - PromptMessageTool) + PromptMessageContentType, PromptMessageTool, + SystemPromptMessage, TextPromptMessageContent, + UserPromptMessage) from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, I18nObject, ModelType -from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeError, \ - InvokeRateLimitError, InvokeAuthorizationError, InvokeBadRequestError +from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, + InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel diff --git a/api/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py index a1c73ab6a0..a239727814 100644 --- a/api/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py @@ -4,11 +4,10 @@ from typing import Optional, Tuple import cohere import numpy as np from cohere.responses import Tokens - from core.model_runtime.entities.model_entities import PriceType from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult -from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \ - InvokeAuthorizationError, InvokeBadRequestError, InvokeError +from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, + InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel diff --git a/api/core/model_runtime/model_providers/minimax/llm/llm.py b/api/core/model_runtime/model_providers/minimax/llm/llm.py index 86a246c714..2657c85419 100644 --- a/api/core/model_runtime/model_providers/minimax/llm/llm.py +++ b/api/core/model_runtime/model_providers/minimax/llm/llm.py @@ -2,7 +2,7 @@ from typing import Generator, List from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, - SystemPromptMessage, UserPromptMessage, ToolPromptMessage) + SystemPromptMessage, ToolPromptMessage, UserPromptMessage) from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError diff --git a/api/core/model_runtime/model_providers/ollama/llm/llm.py b/api/core/model_runtime/model_providers/ollama/llm/llm.py index 083e60f35e..27c4be125a 100644 --- a/api/core/model_runtime/model_providers/ollama/llm/llm.py +++ b/api/core/model_runtime/model_providers/ollama/llm/llm.py @@ -2,21 +2,20 @@ import json import logging import re from decimal import Decimal -from typing import Optional, Generator, Union, List, cast +from typing import Generator, List, Optional, Union, cast from urllib.parse import urljoin import requests - -from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage, AssistantPromptMessage, \ - UserPromptMessage, PromptMessageContentType, ImagePromptMessageContent, \ - TextPromptMessageContent, SystemPromptMessage -from core.model_runtime.entities.model_entities import I18nObject, ModelType, \ - PriceConfig, AIModelEntity, FetchFrom, ModelPropertyKey, ParameterRule, ParameterType, DefaultParameterName, \ - ModelFeature -from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, \ - LLMResultChunk, LLMResultChunkDelta -from core.model_runtime.errors.invoke import InvokeError, InvokeAuthorizationError, InvokeBadRequestError, \ - InvokeRateLimitError, InvokeServerUnavailableError, InvokeConnectionError +from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent, + PromptMessage, PromptMessageContentType, PromptMessageTool, + SystemPromptMessage, TextPromptMessageContent, + UserPromptMessage) +from core.model_runtime.entities.model_entities import (AIModelEntity, DefaultParameterName, FetchFrom, I18nObject, + ModelFeature, ModelPropertyKey, ModelType, ParameterRule, + ParameterType, PriceConfig) +from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, + InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel diff --git a/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py index f496898180..5d96ac65ff 100644 --- a/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py @@ -1,19 +1,18 @@ +import json import logging import time from decimal import Decimal from typing import Optional from urllib.parse import urljoin -import requests -import json import numpy as np - +import requests from core.model_runtime.entities.common_entities import I18nObject -from core.model_runtime.entities.model_entities import PriceType, ModelPropertyKey, ModelType, AIModelEntity, FetchFrom, \ - PriceConfig -from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage -from core.model_runtime.errors.invoke import InvokeError, InvokeAuthorizationError, InvokeBadRequestError, \ - InvokeRateLimitError, InvokeServerUnavailableError, InvokeConnectionError +from core.model_runtime.entities.model_entities import (AIModelEntity, FetchFrom, ModelPropertyKey, ModelType, + PriceConfig, PriceType) +from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult +from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, + InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel diff --git a/api/core/model_runtime/model_providers/openai/tts/tts.py b/api/core/model_runtime/model_providers/openai/tts/tts.py index 013392eb3a..95a88e9bec 100644 --- a/api/core/model_runtime/model_providers/openai/tts/tts.py +++ b/api/core/model_runtime/model_providers/openai/tts/tts.py @@ -1,16 +1,15 @@ +import concurrent.futures +from functools import reduce from io import BytesIO from typing import Optional -from functools import reduce -from pydub import AudioSegment -from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.errors.invoke import InvokeBadRequestError +from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.tts_model import TTSModel from core.model_runtime.model_providers.openai._common import _CommonOpenAI - from flask import Response, stream_with_context from openai import OpenAI -import concurrent.futures +from pydub import AudioSegment class OpenAIText2SpeechModel(_CommonOpenAI, TTSModel): diff --git a/api/core/model_runtime/model_providers/tongyi/llm/llm.py b/api/core/model_runtime/model_providers/tongyi/llm/llm.py index 1e03bc801e..63f300fc19 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/llm.py +++ b/api/core/model_runtime/model_providers/tongyi/llm/llm.py @@ -1,14 +1,13 @@ from typing import Generator, List, Optional, Union -from dashscope import get_tokenizer - -from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMMode +from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, SystemPromptMessage, UserPromptMessage) from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel +from dashscope import get_tokenizer from dashscope.api_entities.dashscope_response import DashScopeAPIResponse from dashscope.common.error import (AuthenticationError, InvalidParameter, RequestFailure, ServiceUnavailableError, UnsupportedHTTPMethod, UnsupportedModel) diff --git a/api/core/model_runtime/model_providers/tongyi/tts/tts.py b/api/core/model_runtime/model_providers/tongyi/tts/tts.py index 1dd307e28f..a6fc201080 100644 --- a/api/core/model_runtime/model_providers/tongyi/tts/tts.py +++ b/api/core/model_runtime/model_providers/tongyi/tts/tts.py @@ -1,16 +1,15 @@ +import concurrent.futures +from functools import reduce from io import BytesIO from typing import Optional -from functools import reduce -from pydub import AudioSegment - -from core.model_runtime.errors.validate import CredentialsValidateFailedError -from core.model_runtime.errors.invoke import InvokeBadRequestError -from core.model_runtime.model_providers.__base.tts_model import TTSModel -from core.model_runtime.model_providers.tongyi._common import _CommonTongyi import dashscope +from core.model_runtime.errors.invoke import InvokeBadRequestError +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.tts_model import TTSModel +from core.model_runtime.model_providers.tongyi._common import _CommonTongyi from flask import Response, stream_with_context -import concurrent.futures +from pydub import AudioSegment class TongyiText2SpeechModel(_CommonTongyi, TTSModel): diff --git a/api/core/model_runtime/model_providers/xinference/llm/llm.py b/api/core/model_runtime/model_providers/xinference/llm/llm.py index 4f5a3b1604..58801229d2 100644 --- a/api/core/model_runtime/model_providers/xinference/llm/llm.py +++ b/api/core/model_runtime/model_providers/xinference/llm/llm.py @@ -3,15 +3,15 @@ from typing import Generator, Iterator, List, Optional, Union, cast from core.model_runtime.entities.common_entities import I18nObject from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, - SystemPromptMessage, UserPromptMessage, ToolPromptMessage) -from core.model_runtime.entities.model_entities import (AIModelEntity, FetchFrom, ModelPropertyKey, ModelType, - ParameterRule, ParameterType, ModelFeature) + SystemPromptMessage, ToolPromptMessage, UserPromptMessage) +from core.model_runtime.entities.model_entities import (AIModelEntity, FetchFrom, ModelFeature, ModelPropertyKey, + ModelType, ParameterRule, ParameterType) from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.model_runtime.model_providers.xinference.xinference_helper import (XinferenceHelper, - XinferenceModelExtraParameter) + XinferenceModelExtraParameter) from core.model_runtime.utils import helper from openai import (APIConnectionError, APITimeoutError, AuthenticationError, ConflictError, InternalServerError, NotFoundError, OpenAI, PermissionDeniedError, RateLimitError, Stream, UnprocessableEntityError) diff --git a/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py index 28389db6a4..02541e310f 100644 --- a/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py @@ -2,15 +2,15 @@ import time from typing import Optional from core.model_runtime.entities.common_entities import I18nObject -from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelType, PriceType, ModelPropertyKey +from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelPropertyKey, ModelType, PriceType from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel +from core.model_runtime.model_providers.xinference.xinference_helper import XinferenceHelper from xinference_client.client.restful.restful_client import Client, RESTfulEmbeddingModelHandle, RESTfulModelHandle -from core.model_runtime.model_providers.xinference.xinference_helper import XinferenceHelper class XinferenceTextEmbeddingModel(TextEmbeddingModel): """ diff --git a/api/core/model_runtime/model_providers/xinference/xinference_helper.py b/api/core/model_runtime/model_providers/xinference/xinference_helper.py index 1b8d1fbfd7..764ffe8b65 100644 --- a/api/core/model_runtime/model_providers/xinference/xinference_helper.py +++ b/api/core/model_runtime/model_providers/xinference/xinference_helper.py @@ -1,7 +1,7 @@ +from os import path from threading import Lock from time import time from typing import List -from os import path from requests import get from requests.adapters import HTTPAdapter diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py index eafae9ab6f..e3180ec177 100644 --- a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py +++ b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py @@ -2,16 +2,19 @@ import json from typing import Any, Dict, Generator, List, Optional, Union from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta -from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageRole, - PromptMessageTool, SystemPromptMessage, UserPromptMessage, ToolPromptMessage, - TextPromptMessageContent, ImagePromptMessageContent, PromptMessageContentType) +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent, + PromptMessage, PromptMessageContentType, PromptMessageRole, + PromptMessageTool, SystemPromptMessage, + TextPromptMessageContent, ToolPromptMessage, + UserPromptMessage) from core.model_runtime.errors.validate import CredentialsValidateFailedError -from core.model_runtime.utils import helper from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.model_runtime.model_providers.zhipuai._common import _CommonZhipuaiAI from core.model_runtime.model_providers.zhipuai.zhipuai_sdk._client import ZhipuAI -from core.model_runtime.model_providers.zhipuai.zhipuai_sdk.types.chat.chat_completion_chunk import ChatCompletionChunk from core.model_runtime.model_providers.zhipuai.zhipuai_sdk.types.chat.chat_completion import Completion +from core.model_runtime.model_providers.zhipuai.zhipuai_sdk.types.chat.chat_completion_chunk import ChatCompletionChunk +from core.model_runtime.utils import helper + class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): diff --git a/api/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py index e5ecc85c47..fd39f5a7a9 100644 --- a/api/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py @@ -5,8 +5,8 @@ from core.model_runtime.entities.model_entities import PriceType from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel -from core.model_runtime.model_providers.zhipuai.zhipuai_sdk._client import ZhipuAI from core.model_runtime.model_providers.zhipuai._common import _CommonZhipuaiAI +from core.model_runtime.model_providers.zhipuai.zhipuai_sdk._client import ZhipuAI from langchain.schema.language_model import _get_token_ids_default_method diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/__init__.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/__init__.py index 5527c48913..8a687ef47a 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/__init__.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/__init__.py @@ -1,17 +1,6 @@ -from ._client import ZhipuAI - -from .core._errors import ( - ZhipuAIError, - APIStatusError, - APIRequestFailedError, - APIAuthenticationError, - APIReachLimitError, - APIInternalError, - APIServerFlowExceedError, - APIResponseError, - APIResponseValidationError, - APITimeoutError, -) - from .__version__ import __version__ +from ._client import ZhipuAI +from .core._errors import (APIAuthenticationError, APIInternalError, APIReachLimitError, APIRequestFailedError, + APIResponseError, APIResponseValidationError, APIServerFlowExceedError, APIStatusError, + APITimeoutError, ZhipuAIError) diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/_client.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/_client.py index e169c5485c..573f0715c4 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/_client.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/_client.py @@ -1,17 +1,17 @@ from __future__ import annotations -from typing import Union, Mapping - -from typing_extensions import override - -from .core import _jwt_token -from .core._errors import ZhipuAIError -from .core._http_client import HttpClient, ZHIPUAI_DEFAULT_MAX_RETRIES -from .core._base_type import NotGiven, NOT_GIVEN -from . import api_resource import os +from typing import Mapping, Union + import httpx from httpx import Timeout +from typing_extensions import override + +from . import api_resource +from .core import _jwt_token +from .core._base_type import NOT_GIVEN, NotGiven +from .core._errors import ZhipuAIError +from .core._http_client import ZHIPUAI_DEFAULT_MAX_RETRIES, HttpClient class ZhipuAI(HttpClient): diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/__init__.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/__init__.py index b596702bc2..0a90e21e48 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/__init__.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/__init__.py @@ -1,5 +1,5 @@ from .chat import chat -from .images import Images from .embeddings import Embeddings from .files import Files from .fine_tuning import fine_tuning +from .images import Images diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/async_completions.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/async_completions.py index b926bd0131..16c4b54f1a 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/async_completions.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/async_completions.py @@ -1,14 +1,14 @@ from __future__ import annotations -from typing import Union, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Union import httpx from typing_extensions import Literal from ...core._base_api import BaseAPI -from ...core._base_type import NotGiven, NOT_GIVEN, Headers +from ...core._base_type import NOT_GIVEN, Headers, NotGiven from ...core._http_client import make_user_request_input -from ...types.chat.async_chat_completion import AsyncTaskStatus, AsyncCompletion +from ...types.chat.async_chat_completion import AsyncCompletion, AsyncTaskStatus if TYPE_CHECKING: from ..._client import ZhipuAI diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/chat.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/chat.py index 6a9fef671f..92362fc50a 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/chat.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/chat.py @@ -1,7 +1,8 @@ from typing import TYPE_CHECKING -from .completions import Completions -from .async_completions import AsyncCompletions + from ...core._base_api import BaseAPI +from .async_completions import AsyncCompletions +from .completions import Completions if TYPE_CHECKING: from ..._client import ZhipuAI diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/completions.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/completions.py index bec2755f93..e5bb8cdf68 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/completions.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/completions.py @@ -1,12 +1,12 @@ from __future__ import annotations -from typing import Union, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Union import httpx from typing_extensions import Literal from ...core._base_api import BaseAPI -from ...core._base_type import NotGiven, NOT_GIVEN, Headers +from ...core._base_type import NOT_GIVEN, Headers, NotGiven from ...core._http_client import make_user_request_input from ...core._sse_client import StreamResponse from ...types.chat.chat_completion import Completion diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/embeddings.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/embeddings.py index b12ce95645..d5db469de4 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/embeddings.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/embeddings.py @@ -1,11 +1,11 @@ from __future__ import annotations -from typing import Union, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Union import httpx from ..core._base_api import BaseAPI -from ..core._base_type import NotGiven, NOT_GIVEN, Headers +from ..core._base_type import NOT_GIVEN, Headers, NotGiven from ..core._http_client import make_user_request_input from ..types.embeddings import EmbeddingsResponded diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/files.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/files.py index 5dde40dae0..dfe52fd54c 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/files.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/files.py @@ -5,11 +5,9 @@ from typing import TYPE_CHECKING import httpx from ..core._base_api import BaseAPI -from ..core._base_type import NOT_GIVEN, Body, Query, Headers, NotGiven, FileTypes +from ..core._base_type import NOT_GIVEN, Body, FileTypes, Headers, NotGiven, Query from ..core._files import is_file_content -from ..core._http_client import ( - make_user_request_input, -) +from ..core._http_client import make_user_request_input from ..types.file_object import FileObject, ListOfFileObject if TYPE_CHECKING: diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/fine_tuning.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/fine_tuning.py index b06274c955..dc54a9ca45 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/fine_tuning.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/fine_tuning.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING -from .jobs import Jobs + from ...core._base_api import BaseAPI +from .jobs import Jobs if TYPE_CHECKING: from ..._client import ZhipuAI diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/jobs.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/jobs.py index 03b597ddbe..ead6cdae2f 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/jobs.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/jobs.py @@ -1,20 +1,13 @@ from __future__ import annotations -from typing import Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Optional import httpx from ...core._base_api import BaseAPI from ...core._base_type import NOT_GIVEN, Headers, NotGiven -from ...core._http_client import ( - make_user_request_input, -) -from ...types.fine_tuning import ( - FineTuningJob, - job_create_params, - ListOfFineTuningJob, - FineTuningJobEvent, -) +from ...core._http_client import make_user_request_input +from ...types.fine_tuning import FineTuningJob, FineTuningJobEvent, ListOfFineTuningJob, job_create_params if TYPE_CHECKING: from ..._client import ZhipuAI diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/images.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/images.py index d245fb8abc..65ce5b246f 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/images.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/images.py @@ -1,11 +1,11 @@ from __future__ import annotations -from typing import Union, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Union import httpx from ..core._base_api import BaseAPI -from ..core._base_type import NotGiven, NOT_GIVEN, Headers +from ..core._base_type import NOT_GIVEN, Headers, NotGiven from ..core._http_client import make_user_request_input from ..types.image import ImagesResponded diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_api.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_api.py index 21291b2a73..10b46ff8e3 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_api.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_api.py @@ -1,4 +1,5 @@ from __future__ import annotations + from typing import TYPE_CHECKING if TYPE_CHECKING: diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py index cc613490dd..f3dde8461c 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py @@ -1,20 +1,10 @@ from __future__ import annotations from os import PathLike -from typing import ( - TYPE_CHECKING, - Type, - Union, - Mapping, - TypeVar, IO, Tuple, Sequence, Any, List, -) +from typing import IO, TYPE_CHECKING, Any, List, Mapping, Sequence, Tuple, Type, TypeVar, Union import pydantic -from typing_extensions import ( - Literal, - override, -) - +from typing_extensions import Literal, override Query = Mapping[str, object] Body = object diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_files.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_files.py index 9ae372adce..e41ede128a 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_files.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_files.py @@ -5,12 +5,7 @@ import os from pathlib import Path from typing import Mapping, Sequence -from ._base_type import ( - FileTypes, - HttpxFileTypes, - HttpxRequestFiles, - RequestFiles, -) +from ._base_type import FileTypes, HttpxFileTypes, HttpxRequestFiles, RequestFiles def is_file_content(obj: object) -> bool: diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py index 0720c78e76..5227d20615 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py @@ -2,20 +2,14 @@ from __future__ import annotations import inspect -from typing import ( - Any, - Type, - Union, - cast, - Mapping, -) +from typing import Any, Mapping, Type, Union, cast import httpx import pydantic from httpx import URL, Timeout from . import _errors -from ._base_type import NotGiven, ResponseT, Body, Headers, NOT_GIVEN, RequestFiles, Query, Data +from ._base_type import NOT_GIVEN, Body, Data, Headers, NotGiven, Query, RequestFiles, ResponseT from ._errors import APIResponseValidationError, APIStatusError, APITimeoutError from ._files import make_httpx_files from ._request_opt import ClientRequestParam, UserRequestInput diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_request_opt.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_request_opt.py index f2281c5284..3f22731de6 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_request_opt.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_request_opt.py @@ -1,15 +1,13 @@ from __future__ import annotations -from typing import Union, Any, cast +from typing import Any, Union, cast import pydantic.generics from httpx import Timeout from pydantic import ConfigDict -from typing_extensions import ( - Unpack, ClassVar, TypedDict -) +from typing_extensions import ClassVar, TypedDict, Unpack -from ._base_type import Body, NotGiven, Headers, HttpxRequestFiles, Query +from ._base_type import Body, Headers, HttpxRequestFiles, NotGiven, Query from ._utils import remove_notgiven_indict diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_response.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_response.py index 86ce50d9f4..116246e645 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_response.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_response.py @@ -1,11 +1,11 @@ from __future__ import annotations import datetime -from typing import TypeVar, Generic, cast, Any, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast import httpx import pydantic -from typing_extensions import ParamSpec, get_origin, get_args +from typing_extensions import ParamSpec, get_args, get_origin from ._base_type import NoneType from ._sse_client import StreamResponse diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_sse_client.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_sse_client.py index 83b487e871..6efe20edcb 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_sse_client.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_sse_client.py @@ -2,7 +2,7 @@ from __future__ import annotations import json -from typing import Generic, Iterator, TYPE_CHECKING, Mapping +from typing import TYPE_CHECKING, Generic, Iterator, Mapping import httpx diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_utils.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_utils.py index 6193edcbee..78c97af65b 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_utils.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_utils.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Mapping, Iterable, TypeVar +from typing import Iterable, Mapping, TypeVar from ._base_type import NotGiven diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/embeddings.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/embeddings.py index 9f52d296d3..a8737cf8dc 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/embeddings.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/embeddings.py @@ -1,9 +1,11 @@ from __future__ import annotations -from typing import Optional, List +from typing import List, Optional from pydantic import BaseModel + from .chat.chat_completion import CompletionUsage + __all__ = ["Embedding", "EmbeddingsResponded"] diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/file_object.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/file_object.py index 39599786ea..94db046bd6 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/file_object.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/file_object.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from pydantic import BaseModel diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job.py index c41fd5f246..c841e1d756 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job.py @@ -1,7 +1,7 @@ -from typing import List, Union, Optional -from typing_extensions import Literal +from typing import List, Optional, Union from pydantic import BaseModel +from typing_extensions import Literal __all__ = ["FineTuningJob", "Error", "Hyperparameters", "ListOfFineTuningJob" ] diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job_event.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job_event.py index fd2a4138aa..1a70483a7b 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job_event.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job_event.py @@ -1,7 +1,7 @@ -from typing import List, Union, Optional -from typing_extensions import Literal +from typing import List, Optional, Union from pydantic import BaseModel +from typing_extensions import Literal __all__ = ["FineTuningJobEvent", "Metric", "JobEvent"] diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/image.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/image.py index 681942c84d..429a7e25bc 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/image.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/image.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Optional, List +from typing import List, Optional from pydantic import BaseModel diff --git a/api/core/tools/entities/tool_bundle.py b/api/core/tools/entities/tool_bundle.py index 6958ccda36..e9d677cbc1 100644 --- a/api/core/tools/entities/tool_bundle.py +++ b/api/core/tools/entities/tool_bundle.py @@ -1,7 +1,8 @@ -from pydantic import BaseModel -from typing import Dict, Optional, Any, List +from typing import Any, Dict, List, Optional + +from core.tools.entities.tool_entities import ToolParameter, ToolProviderType +from pydantic import BaseModel -from core.tools.entities.tool_entities import ToolProviderType, ToolParameter class ApiBasedToolBundle(BaseModel): """ diff --git a/api/core/tools/entities/tool_entities.py b/api/core/tools/entities/tool_entities.py index 6afc3e9615..d33c1ba8c3 100644 --- a/api/core/tools/entities/tool_entities.py +++ b/api/core/tools/entities/tool_entities.py @@ -1,8 +1,9 @@ -from pydantic import BaseModel, Field from enum import Enum -from typing import Optional, List, Dict, Any, Union, cast +from typing import Any, Dict, List, Optional, Union, cast from core.tools.entities.common_entities import I18nObject +from pydantic import BaseModel, Field + class ToolProviderType(Enum): """ diff --git a/api/core/tools/entities/user_entities.py b/api/core/tools/entities/user_entities.py index 99d02f32c1..265f856eb4 100644 --- a/api/core/tools/entities/user_entities.py +++ b/api/core/tools/entities/user_entities.py @@ -1,10 +1,11 @@ -from pydantic import BaseModel from enum import Enum -from typing import List, Dict, Optional +from typing import Dict, List, Optional from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_entities import ToolProviderCredentials from core.tools.tool.tool import ToolParameter +from pydantic import BaseModel + class UserToolProvider(BaseModel): class ProviderType(Enum): diff --git a/api/core/tools/model/tool_model_manager.py b/api/core/tools/model/tool_model_manager.py index 0a8e131344..ee17742161 100644 --- a/api/core/tools/model/tool_model_manager.py +++ b/api/core/tools/model/tool_model_manager.py @@ -4,23 +4,21 @@ Therefore, a model manager is needed to list/invoke/validate models. """ -from core.model_runtime.entities.message_entities import PromptMessage -from core.model_runtime.entities.llm_entities import LLMResult -from core.model_runtime.entities.model_entities import ModelType -from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel, ModelPropertyKey -from core.model_runtime.errors.invoke import InvokeRateLimitError, InvokeBadRequestError, \ - InvokeConnectionError, InvokeAuthorizationError, InvokeServerUnavailableError -from core.model_runtime.utils.encoders import jsonable_encoder +import json +from typing import List, cast + from core.model_manager import ModelManager - +from core.model_runtime.entities.llm_entities import LLMResult +from core.model_runtime.entities.message_entities import PromptMessage +from core.model_runtime.entities.model_entities import ModelType +from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, + InvokeRateLimitError, InvokeServerUnavailableError) +from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel, ModelPropertyKey +from core.model_runtime.utils.encoders import jsonable_encoder from core.tools.model.errors import InvokeModelError - from extensions.ext_database import db - from models.tools import ToolModelInvoke -from typing import List, cast -import json class ToolModelManager: @staticmethod diff --git a/api/core/tools/provider/api_tool_provider.py b/api/core/tools/provider/api_tool_provider.py index 6472a34dda..03edb365ce 100644 --- a/api/core/tools/provider/api_tool_provider.py +++ b/api/core/tools/provider/api_tool_provider.py @@ -1,15 +1,16 @@ from typing import Any, Dict, List -from core.tools.entities.tool_entities import ToolProviderType, ApiProviderAuthType, ToolProviderCredentials, ToolCredentialsOption + from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_bundle import ApiBasedToolBundle -from core.tools.tool.tool import Tool -from core.tools.tool.api_tool import ApiTool +from core.tools.entities.tool_entities import (ApiProviderAuthType, ToolCredentialsOption, ToolProviderCredentials, + ToolProviderType) from core.tools.provider.tool_provider import ToolProviderController - +from core.tools.tool.api_tool import ApiTool +from core.tools.tool.tool import Tool from extensions.ext_database import db - from models.tools import ApiToolProvider + class ApiBasedToolProviderController(ToolProviderController): @staticmethod def from_db(db_provider: ApiToolProvider, auth_type: ApiProviderAuthType) -> 'ApiBasedToolProviderController': diff --git a/api/core/tools/provider/app_tool_provider.py b/api/core/tools/provider/app_tool_provider.py index 1b3e391d53..963c2a1b15 100644 --- a/api/core/tools/provider/app_tool_provider.py +++ b/api/core/tools/provider/app_tool_provider.py @@ -1,14 +1,13 @@ -from typing import Any, Dict, List -from core.tools.entities.tool_entities import ToolProviderType, ToolParameter, ToolParameterOption -from core.tools.tool.tool import Tool -from core.tools.entities.common_entities import I18nObject -from core.tools.provider.tool_provider import ToolProviderController - -from extensions.ext_database import db -from models.tools import PublishedAppTool -from models.model import App, AppModelConfig - import logging +from typing import Any, Dict, List + +from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_entities import ToolParameter, ToolParameterOption, ToolProviderType +from core.tools.provider.tool_provider import ToolProviderController +from core.tools.tool.tool import Tool +from extensions.ext_database import db +from models.model import App, AppModelConfig +from models.tools import PublishedAppTool logger = logging.getLogger(__name__) diff --git a/api/core/tools/provider/builtin/_positions.py b/api/core/tools/provider/builtin/_positions.py index b8b92e3129..232b6684aa 100644 --- a/api/core/tools/provider/builtin/_positions.py +++ b/api/core/tools/provider/builtin/_positions.py @@ -1,6 +1,7 @@ -from core.tools.entities.user_entities import UserToolProvider from typing import List +from core.tools.entities.user_entities import UserToolProvider + position = { 'google': 1, 'wikipedia': 2, diff --git a/api/core/tools/provider/builtin/azuredalle/azuredalle.py b/api/core/tools/provider/builtin/azuredalle/azuredalle.py index e218d09efa..09fe5600a5 100644 --- a/api/core/tools/provider/builtin/azuredalle/azuredalle.py +++ b/api/core/tools/provider/builtin/azuredalle/azuredalle.py @@ -1,9 +1,10 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.provider.builtin.azuredalle.tools.dalle3 import DallE3Tool -from core.tools.errors import ToolProviderCredentialValidationError - from typing import Any, Dict +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.azuredalle.tools.dalle3 import DallE3Tool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + class AzureDALLEProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: try: diff --git a/api/core/tools/provider/builtin/azuredalle/tools/dalle3.py b/api/core/tools/provider/builtin/azuredalle/tools/dalle3.py index 2809247957..25fac01268 100644 --- a/api/core/tools/provider/builtin/azuredalle/tools/dalle3.py +++ b/api/core/tools/provider/builtin/azuredalle/tools/dalle3.py @@ -1,12 +1,12 @@ -from typing import Any, Dict, List, Union -from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.tool.builtin_tool import BuiltinTool - from base64 import b64decode from os.path import join +from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from openai import AzureOpenAI + class DallE3Tool(BuiltinTool): def _invoke(self, user_id: str, diff --git a/api/core/tools/provider/builtin/chart/chart.py b/api/core/tools/provider/builtin/chart/chart.py index 679c4e05cf..77773bf6bb 100644 --- a/api/core/tools/provider/builtin/chart/chart.py +++ b/api/core/tools/provider/builtin/chart/chart.py @@ -1,9 +1,8 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError - -from core.tools.provider.builtin.chart.tools.line import LinearChartTool - import matplotlib.pyplot as plt +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.chart.tools.line import LinearChartTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + # use a business theme plt.style.use('seaborn-v0_8-darkgrid') diff --git a/api/core/tools/provider/builtin/chart/tools/bar.py b/api/core/tools/provider/builtin/chart/tools/bar.py index 0403430357..bfc4936140 100644 --- a/api/core/tools/provider/builtin/chart/tools/bar.py +++ b/api/core/tools/provider/builtin/chart/tools/bar.py @@ -1,10 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage -import matplotlib.pyplot as plt import io - from typing import Any, Dict, List, Union +import matplotlib.pyplot as plt +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + class BarChartTool(BuiltinTool): def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \ -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]: diff --git a/api/core/tools/provider/builtin/chart/tools/line.py b/api/core/tools/provider/builtin/chart/tools/line.py index d77ab2e43d..a2b618315b 100644 --- a/api/core/tools/provider/builtin/chart/tools/line.py +++ b/api/core/tools/provider/builtin/chart/tools/line.py @@ -1,10 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage -import matplotlib.pyplot as plt import io - from typing import Any, Dict, List, Union +import matplotlib.pyplot as plt +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + class LinearChartTool(BuiltinTool): def _invoke(self, user_id: str, diff --git a/api/core/tools/provider/builtin/chart/tools/pie.py b/api/core/tools/provider/builtin/chart/tools/pie.py index 0635b2afb3..47c2bb1976 100644 --- a/api/core/tools/provider/builtin/chart/tools/pie.py +++ b/api/core/tools/provider/builtin/chart/tools/pie.py @@ -1,10 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage -import matplotlib.pyplot as plt import io - from typing import Any, Dict, List, Union +import matplotlib.pyplot as plt +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + class PieChartTool(BuiltinTool): def _invoke(self, user_id: str, diff --git a/api/core/tools/provider/builtin/dalle/dalle.py b/api/core/tools/provider/builtin/dalle/dalle.py index 26e09aaab6..e5d3159d93 100644 --- a/api/core/tools/provider/builtin/dalle/dalle.py +++ b/api/core/tools/provider/builtin/dalle/dalle.py @@ -1,9 +1,10 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.provider.builtin.dalle.tools.dalle2 import DallE2Tool -from core.tools.errors import ToolProviderCredentialValidationError - from typing import Any, Dict +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.dalle.tools.dalle2 import DallE2Tool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + class DALLEProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: try: diff --git a/api/core/tools/provider/builtin/dalle/tools/dalle2.py b/api/core/tools/provider/builtin/dalle/tools/dalle2.py index a542f01796..ef62faf8b8 100644 --- a/api/core/tools/provider/builtin/dalle/tools/dalle2.py +++ b/api/core/tools/provider/builtin/dalle/tools/dalle2.py @@ -1,12 +1,12 @@ -from typing import Any, Dict, List, Union -from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.tool.builtin_tool import BuiltinTool - from base64 import b64decode from os.path import join +from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from openai import OpenAI + class DallE2Tool(BuiltinTool): def _invoke(self, user_id: str, diff --git a/api/core/tools/provider/builtin/dalle/tools/dalle3.py b/api/core/tools/provider/builtin/dalle/tools/dalle3.py index 6f75719859..43e96d7780 100644 --- a/api/core/tools/provider/builtin/dalle/tools/dalle3.py +++ b/api/core/tools/provider/builtin/dalle/tools/dalle3.py @@ -1,12 +1,12 @@ -from typing import Any, Dict, List, Union -from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.tool.builtin_tool import BuiltinTool - from base64 import b64decode from os.path import join +from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from openai import OpenAI + class DallE3Tool(BuiltinTool): def _invoke(self, user_id: str, diff --git a/api/core/tools/provider/builtin/gaode/gaode.py b/api/core/tools/provider/builtin/gaode/gaode.py index 50433fbc9e..a89ea6579a 100644 --- a/api/core/tools/provider/builtin/gaode/gaode.py +++ b/api/core/tools/provider/builtin/gaode/gaode.py @@ -1,7 +1,8 @@ -import requests import urllib.parse -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + +import requests from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController class GaodeProvider(BuiltinToolProviderController): diff --git a/api/core/tools/provider/builtin/gaode/tools/gaode_weather.py b/api/core/tools/provider/builtin/gaode/tools/gaode_weather.py index dd50622394..527ee6ba25 100644 --- a/api/core/tools/provider/builtin/gaode/tools/gaode_weather.py +++ b/api/core/tools/provider/builtin/gaode/tools/gaode_weather.py @@ -1,9 +1,10 @@ import json -import requests -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage from typing import Any, Dict, List, Union +import requests +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + class GaodeRepositoriesTool(BuiltinTool): def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]: diff --git a/api/core/tools/provider/builtin/github/github.py b/api/core/tools/provider/builtin/github/github.py index 8376093bfe..b224a82a4b 100644 --- a/api/core/tools/provider/builtin/github/github.py +++ b/api/core/tools/provider/builtin/github/github.py @@ -1,6 +1,6 @@ import requests -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController class GihubProvider(BuiltinToolProviderController): diff --git a/api/core/tools/provider/builtin/github/tools/github_repositories.py b/api/core/tools/provider/builtin/github/tools/github_repositories.py index cb0099f6e9..10ca5a5cd7 100644 --- a/api/core/tools/provider/builtin/github/tools/github_repositories.py +++ b/api/core/tools/provider/builtin/github/tools/github_repositories.py @@ -1,11 +1,11 @@ import json -import requests from datetime import datetime -from urllib.parse import quote -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - from typing import Any, Dict, List, Union +from urllib.parse import quote + +import requests +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool class GihubRepositoriesTool(BuiltinTool): diff --git a/api/core/tools/provider/builtin/google/google.py b/api/core/tools/provider/builtin/google/google.py index 09a802757b..0ac34af33d 100644 --- a/api/core/tools/provider/builtin/google/google.py +++ b/api/core/tools/provider/builtin/google/google.py @@ -1,10 +1,10 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError - -from core.tools.provider.builtin.google.tools.google_search import GoogleSearchTool - from typing import Any, Dict, List +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.google.tools.google_search import GoogleSearchTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + class GoogleProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: try: diff --git a/api/core/tools/provider/builtin/google/tools/google_search.py b/api/core/tools/provider/builtin/google/tools/google_search.py index cfc1508481..81ff20fdc1 100644 --- a/api/core/tools/provider/builtin/google/tools/google_search.py +++ b/api/core/tools/provider/builtin/google/tools/google_search.py @@ -1,13 +1,12 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - -from typing import Any, Dict, List, Union - import os import sys +from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from serpapi import GoogleSearch + class HiddenPrints: """Context manager to hide prints.""" diff --git a/api/core/tools/provider/builtin/stablediffusion/stablediffusion.py b/api/core/tools/provider/builtin/stablediffusion/stablediffusion.py index 2e79df4ff3..f899abf00f 100644 --- a/api/core/tools/provider/builtin/stablediffusion/stablediffusion.py +++ b/api/core/tools/provider/builtin/stablediffusion/stablediffusion.py @@ -1,10 +1,9 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError - -from core.tools.provider.builtin.stablediffusion.tools.stable_diffusion import StableDiffusionTool - from typing import Any, Dict +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.stablediffusion.tools.stable_diffusion import StableDiffusionTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + class StableDiffusionProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: diff --git a/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py b/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py index b9e91698ec..2899235f76 100644 --- a/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py +++ b/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py @@ -1,18 +1,16 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolParameterOption -from core.tools.entities.common_entities import I18nObject -from core.tools.errors import ToolProviderCredentialValidationError - -from typing import Any, Dict, List, Union -from httpx import post, get -from os.path import join -from base64 import b64decode, b64encode -from PIL import Image - -import json import io - +import json +from base64 import b64decode, b64encode from copy import deepcopy +from os.path import join +from typing import Any, Dict, List, Union + +from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolParameterOption +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.tool.builtin_tool import BuiltinTool +from httpx import get, post +from PIL import Image DRAW_TEXT_OPTIONS = { "prompt": "", @@ -70,10 +68,10 @@ class StableDiffusionTool(BuiltinTool): base_url = self.runtime.credentials.get('base_url', None) if not base_url: return self.create_text_message('Please input base_url') - + if 'model' in tool_parameters: self.runtime.credentials['model'] = tool_parameters['model'] - + model = self.runtime.credentials.get('model', None) if not model: return self.create_text_message('Please input model') diff --git a/api/core/tools/provider/builtin/time/time.py b/api/core/tools/provider/builtin/time/time.py index e2b25238e5..214f202c91 100644 --- a/api/core/tools/provider/builtin/time/time.py +++ b/api/core/tools/provider/builtin/time/time.py @@ -1,10 +1,10 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError - -from core.tools.provider.builtin.time.tools.current_time import CurrentTimeTool - from typing import Any, Dict +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.time.tools.current_time import CurrentTimeTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + class WikiPediaProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: try: diff --git a/api/core/tools/provider/builtin/time/tools/current_time.py b/api/core/tools/provider/builtin/time/tools/current_time.py index ae4ece9aa6..07a1f94b31 100644 --- a/api/core/tools/provider/builtin/time/tools/current_time.py +++ b/api/core/tools/provider/builtin/time/tools/current_time.py @@ -1,10 +1,10 @@ +from datetime import datetime, timezone +from typing import Any, Dict, List, Union + from core.tools.entities.tool_entities import ToolInvokeMessage from core.tools.tool.builtin_tool import BuiltinTool - -from typing import Any, Dict, List, Union from pytz import timezone as pytz_timezone -from datetime import datetime, timezone class CurrentTimeTool(BuiltinTool): def _invoke(self, diff --git a/api/core/tools/provider/builtin/vectorizer/tools/vectorizer.py b/api/core/tools/provider/builtin/vectorizer/tools/vectorizer.py index 725c596e23..8051729825 100644 --- a/api/core/tools/provider/builtin/vectorizer/tools/vectorizer.py +++ b/api/core/tools/provider/builtin/vectorizer/tools/vectorizer.py @@ -1,11 +1,12 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter -from core.tools.provider.builtin.vectorizer.tools.test_data import VECTORIZER_ICON_PNG -from core.tools.errors import ToolProviderCredentialValidationError - -from typing import Any, Dict, List, Union -from httpx import post from base64 import b64decode +from typing import Any, Dict, List, Union + +from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.vectorizer.tools.test_data import VECTORIZER_ICON_PNG +from core.tools.tool.builtin_tool import BuiltinTool +from httpx import post + class VectorizerTool(BuiltinTool): def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \ diff --git a/api/core/tools/provider/builtin/vectorizer/vectorizer.py b/api/core/tools/provider/builtin/vectorizer/vectorizer.py index 6b4edd67b8..3711d2b5f1 100644 --- a/api/core/tools/provider/builtin/vectorizer/vectorizer.py +++ b/api/core/tools/provider/builtin/vectorizer/vectorizer.py @@ -1,10 +1,10 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError - -from core.tools.provider.builtin.vectorizer.tools.vectorizer import VectorizerTool - from typing import Any, Dict +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.vectorizer.tools.vectorizer import VectorizerTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + class VectorizerProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: try: diff --git a/api/core/tools/provider/builtin/webscraper/tools/webscraper.py b/api/core/tools/provider/builtin/webscraper/tools/webscraper.py index 4d5251533b..d3129c37e2 100644 --- a/api/core/tools/provider/builtin/webscraper/tools/webscraper.py +++ b/api/core/tools/provider/builtin/webscraper/tools/webscraper.py @@ -1,8 +1,9 @@ -from core.tools.tool.builtin_tool import BuiltinTool +from typing import Any, Dict, List, Union + from core.tools.entities.tool_entities import ToolInvokeMessage from core.tools.errors import ToolInvokeError +from core.tools.tool.builtin_tool import BuiltinTool -from typing import Any, Dict, List, Union class WebscraperTool(BuiltinTool): def _invoke(self, diff --git a/api/core/tools/provider/builtin/webscraper/webscraper.py b/api/core/tools/provider/builtin/webscraper/webscraper.py index edea0384d8..aef35e02a4 100644 --- a/api/core/tools/provider/builtin/webscraper/webscraper.py +++ b/api/core/tools/provider/builtin/webscraper/webscraper.py @@ -1,10 +1,10 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError - -from core.tools.provider.builtin.webscraper.tools.webscraper import WebscraperTool - from typing import Any, Dict, List +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.webscraper.tools.webscraper import WebscraperTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + class WebscraperProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: try: diff --git a/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py b/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py index 6ecd449131..3cab06fdd5 100644 --- a/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py +++ b/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py @@ -1,12 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - -from pydantic import BaseModel, Field - from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from langchain import WikipediaAPIWrapper from langchain.tools import WikipediaQueryRun +from pydantic import BaseModel, Field + class WikipediaInput(BaseModel): query: str = Field(..., description="search query.") diff --git a/api/core/tools/provider/builtin/wikipedia/wikipedia.py b/api/core/tools/provider/builtin/wikipedia/wikipedia.py index d3af7d6224..8d53852255 100644 --- a/api/core/tools/provider/builtin/wikipedia/wikipedia.py +++ b/api/core/tools/provider/builtin/wikipedia/wikipedia.py @@ -1,7 +1,7 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController from core.tools.errors import ToolProviderCredentialValidationError - from core.tools.provider.builtin.wikipedia.tools.wikipedia_search import WikiPediaSearchTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + class WikiPediaProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: dict) -> None: diff --git a/api/core/tools/provider/builtin/wolframalpha/tools/wolframalpha.py b/api/core/tools/provider/builtin/wolframalpha/tools/wolframalpha.py index 40075b3b8e..d915044d89 100644 --- a/api/core/tools/provider/builtin/wolframalpha/tools/wolframalpha.py +++ b/api/core/tools/provider/builtin/wolframalpha/tools/wolframalpha.py @@ -1,11 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.errors import ToolProviderCredentialValidationError, ToolInvokeError - from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.errors import ToolInvokeError, ToolProviderCredentialValidationError +from core.tools.tool.builtin_tool import BuiltinTool from httpx import get + class WolframAlphaTool(BuiltinTool): _base_url = 'https://api.wolframalpha.com/v2/query' diff --git a/api/core/tools/provider/builtin/wolframalpha/wolframalpha.py b/api/core/tools/provider/builtin/wolframalpha/wolframalpha.py index 9293870029..55d827093c 100644 --- a/api/core/tools/provider/builtin/wolframalpha/wolframalpha.py +++ b/api/core/tools/provider/builtin/wolframalpha/wolframalpha.py @@ -1,12 +1,12 @@ -from core.tools.entities.tool_entities import ToolInvokeMessage, ToolProviderType -from core.tools.tool.tool import Tool -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError - -from core.tools.provider.builtin.wolframalpha.tools.wolframalpha import WolframAlphaTool - from typing import Any, Dict, List +from core.tools.entities.tool_entities import ToolInvokeMessage, ToolProviderType +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.wolframalpha.tools.wolframalpha import WolframAlphaTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController +from core.tools.tool.tool import Tool + + class GoogleProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: try: diff --git a/api/core/tools/provider/builtin/yahoo/tools/analytics.py b/api/core/tools/provider/builtin/yahoo/tools/analytics.py index 557ea176fd..85608b5324 100644 --- a/api/core/tools/provider/builtin/yahoo/tools/analytics.py +++ b/api/core/tools/provider/builtin/yahoo/tools/analytics.py @@ -1,12 +1,12 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - -from typing import Any, Dict, List, Union -from requests.exceptions import HTTPError, ReadTimeout from datetime import datetime +from typing import Any, Dict, List, Union -from yfinance import download import pandas as pd +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from requests.exceptions import HTTPError, ReadTimeout +from yfinance import download + class YahooFinanceAnalyticsTool(BuiltinTool): def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \ diff --git a/api/core/tools/provider/builtin/yahoo/tools/news.py b/api/core/tools/provider/builtin/yahoo/tools/news.py index 6a8dec99d6..79de87e0f3 100644 --- a/api/core/tools/provider/builtin/yahoo/tools/news.py +++ b/api/core/tools/provider/builtin/yahoo/tools/news.py @@ -1,10 +1,10 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - from typing import Any, Dict, List, Union -from requests.exceptions import HTTPError, ReadTimeout import yfinance +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from requests.exceptions import HTTPError, ReadTimeout + class YahooFinanceSearchTickerTool(BuiltinTool): def _invoke(self,user_id: str, tool_parameters: Dict[str, Any]) \ diff --git a/api/core/tools/provider/builtin/yahoo/tools/ticker.py b/api/core/tools/provider/builtin/yahoo/tools/ticker.py index 3843619546..b28cee0374 100644 --- a/api/core/tools/provider/builtin/yahoo/tools/ticker.py +++ b/api/core/tools/provider/builtin/yahoo/tools/ticker.py @@ -1,11 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - from typing import Any, Dict, List, Union -from requests.exceptions import HTTPError, ReadTimeout +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from requests.exceptions import HTTPError, ReadTimeout from yfinance import Ticker + class YahooFinanceSearchTickerTool(BuiltinTool): def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \ -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]: diff --git a/api/core/tools/provider/builtin/yahoo/yahoo.py b/api/core/tools/provider/builtin/yahoo/yahoo.py index 4cb5d76b8b..ade33ffb63 100644 --- a/api/core/tools/provider/builtin/yahoo/yahoo.py +++ b/api/core/tools/provider/builtin/yahoo/yahoo.py @@ -1,7 +1,7 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController from core.tools.errors import ToolProviderCredentialValidationError - from core.tools.provider.builtin.yahoo.tools.ticker import YahooFinanceSearchTickerTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + class YahooFinanceProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: dict) -> None: diff --git a/api/core/tools/provider/builtin/youtube/tools/videos.py b/api/core/tools/provider/builtin/youtube/tools/videos.py index 15415f32ce..5922c37dda 100644 --- a/api/core/tools/provider/builtin/youtube/tools/videos.py +++ b/api/core/tools/provider/builtin/youtube/tools/videos.py @@ -1,11 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - -from typing import Any, Dict, List, Union from datetime import datetime +from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from googleapiclient.discovery import build + class YoutubeVideosAnalyticsTool(BuiltinTool): def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \ -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]: diff --git a/api/core/tools/provider/builtin/youtube/youtube.py b/api/core/tools/provider/builtin/youtube/youtube.py index cdca1468e6..8cca578c46 100644 --- a/api/core/tools/provider/builtin/youtube/youtube.py +++ b/api/core/tools/provider/builtin/youtube/youtube.py @@ -1,7 +1,7 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController from core.tools.errors import ToolProviderCredentialValidationError - from core.tools.provider.builtin.youtube.tools.videos import YoutubeVideosAnalyticsTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + class YahooFinanceProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: dict) -> None: diff --git a/api/core/tools/provider/builtin_tool_provider.py b/api/core/tools/provider/builtin_tool_provider.py index 09da8860de..14eefebf49 100644 --- a/api/core/tools/provider/builtin_tool_provider.py +++ b/api/core/tools/provider/builtin_tool_provider.py @@ -1,19 +1,17 @@ -from abc import abstractmethod -from typing import List, Dict, Any - -from os import path, listdir -from yaml import load, FullLoader - -from core.tools.entities.tool_entities import ToolProviderType, \ - ToolParameter, ToolProviderCredentials -from core.tools.tool.tool import Tool -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.provider.tool_provider import ToolProviderController -from core.tools.entities.user_entities import UserToolProviderCredentials -from core.tools.errors import ToolNotFoundError, ToolProviderNotFoundError, \ - ToolParameterValidationError, ToolProviderCredentialValidationError - import importlib +from abc import abstractmethod +from os import listdir, path +from typing import Any, Dict, List + +from core.tools.entities.tool_entities import ToolParameter, ToolProviderCredentials, ToolProviderType +from core.tools.entities.user_entities import UserToolProviderCredentials +from core.tools.errors import (ToolNotFoundError, ToolParameterValidationError, ToolProviderCredentialValidationError, + ToolProviderNotFoundError) +from core.tools.provider.tool_provider import ToolProviderController +from core.tools.tool.builtin_tool import BuiltinTool +from core.tools.tool.tool import Tool +from yaml import FullLoader, load + class BuiltinToolProviderController(ToolProviderController): def __init__(self, **data: Any) -> None: diff --git a/api/core/tools/provider/tool_provider.py b/api/core/tools/provider/tool_provider.py index 155f45d016..954727f774 100644 --- a/api/core/tools/provider/tool_provider.py +++ b/api/core/tools/provider/tool_provider.py @@ -1,14 +1,13 @@ from abc import ABC, abstractmethod -from typing import List, Dict, Any, Optional +from typing import Any, Dict, List, Optional +from core.tools.entities.tool_entities import (ToolParameter, ToolProviderCredentials, ToolProviderIdentity, + ToolProviderType) +from core.tools.entities.user_entities import UserToolProviderCredentials +from core.tools.errors import ToolNotFoundError, ToolParameterValidationError, ToolProviderCredentialValidationError +from core.tools.tool.tool import Tool from pydantic import BaseModel -from core.tools.entities.tool_entities import ToolProviderType, \ - ToolProviderIdentity, ToolParameter, ToolProviderCredentials -from core.tools.tool.tool import Tool -from core.tools.entities.user_entities import UserToolProviderCredentials -from core.tools.errors import ToolNotFoundError, \ - ToolParameterValidationError, ToolProviderCredentialValidationError class ToolProviderController(BaseModel, ABC): identity: Optional[ToolProviderIdentity] = None diff --git a/api/core/tools/tool/api_tool.py b/api/core/tools/tool/api_tool.py index 4d1973b1c8..2d8845e98b 100644 --- a/api/core/tools/tool/api_tool.py +++ b/api/core/tools/tool/api_tool.py @@ -1,14 +1,14 @@ -from typing import Any, Dict, List, Union +import json from json import dumps - -from core.tools.entities.tool_bundle import ApiBasedToolBundle -from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.tool.tool import Tool -from core.tools.errors import ToolProviderCredentialValidationError +from typing import Any, Dict, List, Union import httpx import requests -import json +from core.tools.entities.tool_bundle import ApiBasedToolBundle +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.tool.tool import Tool + class ApiTool(Tool): api_bundle: ApiBasedToolBundle diff --git a/api/core/tools/tool/builtin_tool.py b/api/core/tools/tool/builtin_tool.py index 11116dbad2..e862d427af 100644 --- a/api/core/tools/tool/builtin_tool.py +++ b/api/core/tools/tool/builtin_tool.py @@ -1,12 +1,11 @@ -from core.tools.tool.tool import Tool -from core.tools.model.tool_model_manager import ToolModelManager -from core.model_runtime.entities.message_entities import PromptMessage -from core.model_runtime.entities.llm_entities import LLMResult -from core.model_runtime.entities.message_entities import SystemPromptMessage, UserPromptMessage -from core.tools.utils.web_reader_tool import get_url - -from typing import List from enum import Enum +from typing import List + +from core.model_runtime.entities.llm_entities import LLMResult +from core.model_runtime.entities.message_entities import PromptMessage, SystemPromptMessage, UserPromptMessage +from core.tools.model.tool_model_manager import ToolModelManager +from core.tools.tool.tool import Tool +from core.tools.utils.web_reader_tool import get_url _SUMMARY_PROMPT = """You are a professional language researcher, you are interested in the language and you can quickly aimed at the main point of an webpage and reproduce it in your own words but diff --git a/api/core/tools/tool/dataset_retriever_tool.py b/api/core/tools/tool/dataset_retriever_tool.py index e092bcdccd..13b9a8497b 100644 --- a/api/core/tools/tool/dataset_retriever_tool.py +++ b/api/core/tools/tool/dataset_retriever_tool.py @@ -1,13 +1,14 @@ from typing import Any, Dict, List, Union -from core.features.dataset_retrieval import DatasetRetrievalFeature -from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolIdentity, ToolDescription -from core.tools.tool.tool import Tool -from core.tools.entities.common_entities import I18nObject + from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler from core.entities.application_entities import DatasetRetrieveConfigEntity, InvokeFrom - +from core.features.dataset_retrieval import DatasetRetrievalFeature +from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_entities import ToolDescription, ToolIdentity, ToolInvokeMessage, ToolParameter +from core.tools.tool.tool import Tool from langchain.tools import BaseTool + class DatasetRetrieverTool(Tool): langchain_tool: BaseTool diff --git a/api/core/tools/tool/tool.py b/api/core/tools/tool/tool.py index fdb25b8635..95a8f8578f 100644 --- a/api/core/tools/tool/tool.py +++ b/api/core/tools/tool/tool.py @@ -1,13 +1,13 @@ +from abc import ABC, abstractmethod +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler +from core.tools.entities.tool_entities import (ToolDescription, ToolIdentity, ToolInvokeMessage, ToolParameter, + ToolRuntimeImageVariable, ToolRuntimeVariable, ToolRuntimeVariablePool) +from core.tools.tool_file_manager import ToolFileManager from pydantic import BaseModel -from typing import List, Dict, Any, Union, Optional -from abc import abstractmethod, ABC -from enum import Enum - -from core.tools.entities.tool_entities import ToolIdentity, ToolInvokeMessage,\ - ToolParameter, ToolDescription, ToolRuntimeVariablePool, ToolRuntimeVariable, ToolRuntimeImageVariable -from core.tools.tool_file_manager import ToolFileManager -from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler class Tool(BaseModel, ABC): identity: ToolIdentity = None diff --git a/api/core/tools/tool_file_manager.py b/api/core/tools/tool_file_manager.py index 43d74622ac..c373399606 100644 --- a/api/core/tools/tool_file_manager.py +++ b/api/core/tools/tool_file_manager.py @@ -1,22 +1,19 @@ -import logging -import time -import os -import hmac import base64 import hashlib - -from typing import Union, Tuple, Generator -from uuid import uuid4 +import hmac +import logging +import os +import time from mimetypes import guess_extension, guess_type -from httpx import get - -from flask import current_app - -from models.tools import ToolFile -from models.model import MessageFile +from typing import Generator, Tuple, Union +from uuid import uuid4 from extensions.ext_database import db from extensions.ext_storage import storage +from flask import current_app +from httpx import get +from models.model import MessageFile +from models.tools import ToolFile logger = logging.getLogger(__name__) @@ -194,4 +191,5 @@ class ToolFileManager: # init tool_file_parser from core.file.tool_file_parser import tool_file_manager + tool_file_manager['manager'] = ToolFileManager diff --git a/api/core/tools/tool_manager.py b/api/core/tools/tool_manager.py index 6250f0e5f1..ee1f38a813 100644 --- a/api/core/tools/tool_manager.py +++ b/api/core/tools/tool_manager.py @@ -1,33 +1,29 @@ -from typing import List, Dict, Any, Tuple, Union +import importlib +import json +import logging +import mimetypes from os import listdir, path +from typing import Any, Dict, List, Tuple, Union -from core.tools.entities.tool_entities import ToolInvokeMessage, ApiProviderAuthType, ToolProviderCredentials -from core.tools.provider.tool_provider import ToolProviderController -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.tool.api_tool import ApiTool -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.entities.constant import DEFAULT_PROVIDERS +from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler +from core.model_runtime.entities.message_entities import PromptMessage from core.tools.entities.common_entities import I18nObject +from core.tools.entities.constant import DEFAULT_PROVIDERS +from core.tools.entities.tool_entities import ApiProviderAuthType, ToolInvokeMessage, ToolProviderCredentials +from core.tools.entities.user_entities import UserToolProvider from core.tools.errors import ToolProviderNotFoundError from core.tools.provider.api_tool_provider import ApiBasedToolProviderController from core.tools.provider.app_tool_provider import AppBasedToolProviderEntity -from core.tools.entities.user_entities import UserToolProvider +from core.tools.provider.builtin._positions import BuiltinToolProviderSort +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController +from core.tools.provider.tool_provider import ToolProviderController +from core.tools.tool.api_tool import ApiTool +from core.tools.tool.builtin_tool import BuiltinTool from core.tools.utils.configuration import ToolConfiguration from core.tools.utils.encoder import serialize_base_model_dict -from core.tools.provider.builtin._positions import BuiltinToolProviderSort - -from core.model_runtime.entities.message_entities import PromptMessage -from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler - from extensions.ext_database import db - from models.tools import ApiToolProvider, BuiltinToolProvider -import importlib -import logging -import json -import mimetypes - logger = logging.getLogger(__name__) _builtin_providers = {} diff --git a/api/core/tools/utils/configuration.py b/api/core/tools/utils/configuration.py index 47583b7cae..21609ce302 100644 --- a/api/core/tools/utils/configuration.py +++ b/api/core/tools/utils/configuration.py @@ -1,9 +1,10 @@ -from typing import Dict, Any -from pydantic import BaseModel +from typing import Any, Dict +from core.helper import encrypter from core.tools.entities.tool_entities import ToolProviderCredentials from core.tools.provider.tool_provider import ToolProviderController -from core.helper import encrypter +from pydantic import BaseModel + class ToolConfiguration(BaseModel): tenant_id: str diff --git a/api/core/tools/utils/encoder.py b/api/core/tools/utils/encoder.py index eaf9b6bedc..cce50fb0ac 100644 --- a/api/core/tools/utils/encoder.py +++ b/api/core/tools/utils/encoder.py @@ -1,7 +1,9 @@ -from pydantic import BaseModel from enum import Enum from typing import List +from pydantic import BaseModel + + def serialize_base_model_array(l: List[BaseModel]) -> str: class _BaseModel(BaseModel): __root__: List[BaseModel] diff --git a/api/core/tools/utils/parser.py b/api/core/tools/utils/parser.py index ec4f3237ad..48b84d630c 100644 --- a/api/core/tools/utils/parser.py +++ b/api/core/tools/utils/parser.py @@ -1,15 +1,15 @@ -from core.tools.entities.tool_bundle import ApiBasedToolBundle -from core.tools.entities.tool_entities import ToolParameter, ToolParameterOption, ApiProviderSchemaType -from core.tools.entities.common_entities import I18nObject -from core.tools.errors import ToolProviderNotFoundError, ToolNotSupportedError, \ - ToolApiSchemaError - +from json import dumps as json_dumps +from json import loads as json_loads from typing import List, Tuple -from yaml import FullLoader, load -from json import loads as json_loads, dumps as json_dumps +from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_bundle import ApiBasedToolBundle +from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolParameter, ToolParameterOption +from core.tools.errors import ToolApiSchemaError, ToolNotSupportedError, ToolProviderNotFoundError from requests import get +from yaml import FullLoader, load + class ApiBasedToolSchemaParser: @staticmethod diff --git a/api/core/tools/utils/web_reader_tool.py b/api/core/tools/utils/web_reader_tool.py index b4cfc19871..5a769f6f2d 100644 --- a/api/core/tools/utils/web_reader_tool.py +++ b/api/core/tools/utils/web_reader_tool.py @@ -7,10 +7,14 @@ import subprocess import tempfile import unicodedata from contextlib import contextmanager -from typing import Type, Any +from typing import Any, Type import requests -from bs4 import BeautifulSoup, NavigableString, Comment, CData +from bs4 import BeautifulSoup, CData, Comment, NavigableString +from core.chain.llm_chain import LLMChain +from core.data_loader import file_extractor +from core.data_loader.file_extractor import FileExtractor +from core.entities.application_entities import ModelConfigEntity from langchain.chains import RefineDocumentsChain from langchain.chains.summarize import refine_prompts from langchain.schema import Document @@ -20,11 +24,6 @@ from newspaper import Article from pydantic import BaseModel, Field from regex import regex -from core.chain.llm_chain import LLMChain -from core.data_loader import file_extractor -from core.data_loader.file_extractor import FileExtractor -from core.entities.application_entities import ModelConfigEntity - FULL_TEMPLATE = """ TITLE: {title} AUTHORS: {authors} diff --git a/api/libs/gmpy2_pkcs10aep_cipher.py b/api/libs/gmpy2_pkcs10aep_cipher.py index d2557f251a..e4af4309ab 100644 --- a/api/libs/gmpy2_pkcs10aep_cipher.py +++ b/api/libs/gmpy2_pkcs10aep_cipher.py @@ -20,16 +20,17 @@ # SOFTWARE. # =================================================================== -from Crypto.Signature.pss import MGF1 -import Crypto.Hash.SHA1 - -from Crypto.Util.py3compat import bord, _copy_bytes -import Crypto.Util.number -from Crypto.Util.number import ceil_div, bytes_to_long, long_to_bytes -from Crypto.Util.strxor import strxor -from Crypto import Random from hashlib import sha1 + +import Crypto.Hash.SHA1 +import Crypto.Util.number import gmpy2 +from Crypto import Random +from Crypto.Signature.pss import MGF1 +from Crypto.Util.number import bytes_to_long, ceil_div, long_to_bytes +from Crypto.Util.py3compat import _copy_bytes, bord +from Crypto.Util.strxor import strxor + class PKCS1OAEP_Cipher: """Cipher object for PKCS#1 v1.5 OAEP. diff --git a/api/libs/rsa.py b/api/libs/rsa.py index f45fb44f56..e2ffed8d05 100644 --- a/api/libs/rsa.py +++ b/api/libs/rsa.py @@ -1,13 +1,13 @@ # -*- coding:utf-8 -*- import hashlib +import libs.gmpy2_pkcs10aep_cipher as gmpy2_pkcs10aep_cipher from Crypto.Cipher import AES from Crypto.PublicKey import RSA from Crypto.Random import get_random_bytes from extensions.ext_redis import redis_client from extensions.ext_storage import storage -import libs.gmpy2_pkcs10aep_cipher as gmpy2_pkcs10aep_cipher def generate_key_pair(tenant_id): private_key = RSA.generate(2048) diff --git a/api/migrations/versions/00bacef91f18_rename_api_provider_description.py b/api/migrations/versions/00bacef91f18_rename_api_provider_description.py index a498c90460..5ae9e8769a 100644 --- a/api/migrations/versions/00bacef91f18_rename_api_provider_description.py +++ b/api/migrations/versions/00bacef91f18_rename_api_provider_description.py @@ -5,9 +5,8 @@ Revises: 8ec536f3c800 Create Date: 2024-01-07 04:07:34.482983 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '00bacef91f18' diff --git a/api/migrations/versions/053da0c1d756_add_api_tool_privacy.py b/api/migrations/versions/053da0c1d756_add_api_tool_privacy.py index 8d304ebfb9..a589f1f08b 100644 --- a/api/migrations/versions/053da0c1d756_add_api_tool_privacy.py +++ b/api/migrations/versions/053da0c1d756_add_api_tool_privacy.py @@ -5,8 +5,8 @@ Revises: 4829e54d2fee Create Date: 2024-01-12 06:47:21.656262 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/114eed84c228_remove_tool_id_from_model_invoke.py b/api/migrations/versions/114eed84c228_remove_tool_id_from_model_invoke.py index 1b1d77055a..58863fe3a7 100644 --- a/api/migrations/versions/114eed84c228_remove_tool_id_from_model_invoke.py +++ b/api/migrations/versions/114eed84c228_remove_tool_id_from_model_invoke.py @@ -5,8 +5,8 @@ Revises: c71211c8f604 Create Date: 2024-01-10 04:40:57.257824 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/23db93619b9d_add_message_files_into_agent_thought.py b/api/migrations/versions/23db93619b9d_add_message_files_into_agent_thought.py index 25363ca947..f3eef4681e 100644 --- a/api/migrations/versions/23db93619b9d_add_message_files_into_agent_thought.py +++ b/api/migrations/versions/23db93619b9d_add_message_files_into_agent_thought.py @@ -5,9 +5,8 @@ Revises: 8ae9bc661daa Create Date: 2024-01-18 08:46:37.302657 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '23db93619b9d' diff --git a/api/migrations/versions/380c6aa5a70d_add_tool_labels_to_agent_thought.py b/api/migrations/versions/380c6aa5a70d_add_tool_labels_to_agent_thought.py index 2031dc2ad3..6c13818463 100644 --- a/api/migrations/versions/380c6aa5a70d_add_tool_labels_to_agent_thought.py +++ b/api/migrations/versions/380c6aa5a70d_add_tool_labels_to_agent_thought.py @@ -5,9 +5,8 @@ Revises: dfb3b7f477da Create Date: 2024-01-24 10:58:15.644445 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '380c6aa5a70d' diff --git a/api/migrations/versions/3ef9b2b6bee6_add_assistant_app.py b/api/migrations/versions/3ef9b2b6bee6_add_assistant_app.py index 4a4a497993..4fbc570303 100644 --- a/api/migrations/versions/3ef9b2b6bee6_add_assistant_app.py +++ b/api/migrations/versions/3ef9b2b6bee6_add_assistant_app.py @@ -5,8 +5,8 @@ Revises: 89c7899ca936 Create Date: 2024-01-05 15:26:25.117551 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/4823da1d26cf_add_tool_file.py b/api/migrations/versions/4823da1d26cf_add_tool_file.py index 797a9539b7..1a473a10fe 100644 --- a/api/migrations/versions/4823da1d26cf_add_tool_file.py +++ b/api/migrations/versions/4823da1d26cf_add_tool_file.py @@ -5,8 +5,8 @@ Revises: 053da0c1d756 Create Date: 2024-01-15 11:37:16.782718 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/4829e54d2fee_change_message_chain_id_to_nullable.py b/api/migrations/versions/4829e54d2fee_change_message_chain_id_to_nullable.py index f67a18cb2b..790ade64ec 100644 --- a/api/migrations/versions/4829e54d2fee_change_message_chain_id_to_nullable.py +++ b/api/migrations/versions/4829e54d2fee_change_message_chain_id_to_nullable.py @@ -5,8 +5,8 @@ Revises: 114eed84c228 Create Date: 2024-01-12 03:42:27.362415 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/89c7899ca936_.py b/api/migrations/versions/89c7899ca936_.py index 75cbb5c2b4..0fad39fa57 100644 --- a/api/migrations/versions/89c7899ca936_.py +++ b/api/migrations/versions/89c7899ca936_.py @@ -5,9 +5,8 @@ Revises: 187385f442fc Create Date: 2024-01-21 04:10:23.192853 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '89c7899ca936' diff --git a/api/migrations/versions/8ae9bc661daa_add_tool_conversation_variables_idx.py b/api/migrations/versions/8ae9bc661daa_add_tool_conversation_variables_idx.py index c65372419e..430a8c78c2 100644 --- a/api/migrations/versions/8ae9bc661daa_add_tool_conversation_variables_idx.py +++ b/api/migrations/versions/8ae9bc661daa_add_tool_conversation_variables_idx.py @@ -5,9 +5,8 @@ Revises: 9fafbd60eca1 Create Date: 2024-01-15 14:22:03.597692 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '8ae9bc661daa' diff --git a/api/migrations/versions/8ec536f3c800_rename_api_provider_credentails.py b/api/migrations/versions/8ec536f3c800_rename_api_provider_credentails.py index 6cab1f2092..6cafc198aa 100644 --- a/api/migrations/versions/8ec536f3c800_rename_api_provider_credentails.py +++ b/api/migrations/versions/8ec536f3c800_rename_api_provider_credentails.py @@ -5,9 +5,8 @@ Revises: ad472b61a054 Create Date: 2024-01-07 03:57:35.257545 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '8ec536f3c800' diff --git a/api/migrations/versions/9fafbd60eca1_add_message_file_belongs_to.py b/api/migrations/versions/9fafbd60eca1_add_message_file_belongs_to.py index 367c2e731f..968906bdd7 100644 --- a/api/migrations/versions/9fafbd60eca1_add_message_file_belongs_to.py +++ b/api/migrations/versions/9fafbd60eca1_add_message_file_belongs_to.py @@ -5,9 +5,8 @@ Revises: 4823da1d26cf Create Date: 2024-01-15 13:07:20.340896 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '9fafbd60eca1' diff --git a/api/migrations/versions/ad472b61a054_add_api_provider_icon.py b/api/migrations/versions/ad472b61a054_add_api_provider_icon.py index 1328326c2d..0ddaf1eb0a 100644 --- a/api/migrations/versions/ad472b61a054_add_api_provider_icon.py +++ b/api/migrations/versions/ad472b61a054_add_api_provider_icon.py @@ -5,9 +5,8 @@ Revises: 3ef9b2b6bee6 Create Date: 2024-01-07 02:21:23.114790 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = 'ad472b61a054' diff --git a/api/migrations/versions/b24be59fbb04_.py b/api/migrations/versions/b24be59fbb04_.py index e19ea09e86..29ba859f2b 100644 --- a/api/migrations/versions/b24be59fbb04_.py +++ b/api/migrations/versions/b24be59fbb04_.py @@ -5,9 +5,8 @@ Revises: 187385f442fc Create Date: 2024-01-17 01:31:12.670556 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = 'b24be59fbb04' diff --git a/api/migrations/versions/c71211c8f604_add_tool_invoke_model_log.py b/api/migrations/versions/c71211c8f604_add_tool_invoke_model_log.py index dc96672b5e..95fb8f5d0e 100644 --- a/api/migrations/versions/c71211c8f604_add_tool_invoke_model_log.py +++ b/api/migrations/versions/c71211c8f604_add_tool_invoke_model_log.py @@ -5,8 +5,8 @@ Revises: f25003750af4 Create Date: 2024-01-09 11:42:50.664797 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/de95f5c77138_migration_serpapi_api_key.py b/api/migrations/versions/de95f5c77138_migration_serpapi_api_key.py index cbb3662851..f1236df316 100644 --- a/api/migrations/versions/de95f5c77138_migration_serpapi_api_key.py +++ b/api/migrations/versions/de95f5c77138_migration_serpapi_api_key.py @@ -5,10 +5,10 @@ Revises: 23db93619b9d Create Date: 2024-01-21 12:09:04.651394 """ -from alembic import op -import sqlalchemy as sa from json import dumps, loads +import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = 'de95f5c77138' diff --git a/api/migrations/versions/dfb3b7f477da_add_tool_index.py b/api/migrations/versions/dfb3b7f477da_add_tool_index.py index 66e2a4f863..3ef03595fe 100644 --- a/api/migrations/versions/dfb3b7f477da_add_tool_index.py +++ b/api/migrations/versions/dfb3b7f477da_add_tool_index.py @@ -5,9 +5,8 @@ Revises: b24be59fbb04 Create Date: 2024-01-24 02:17:01.631635 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = 'dfb3b7f477da' diff --git a/api/migrations/versions/f25003750af4_add_created_updated_at.py b/api/migrations/versions/f25003750af4_add_created_updated_at.py index 8bdb6a0ff7..178eaf2380 100644 --- a/api/migrations/versions/f25003750af4_add_created_updated_at.py +++ b/api/migrations/versions/f25003750af4_add_created_updated_at.py @@ -5,9 +5,8 @@ Revises: 00bacef91f18 Create Date: 2024-01-07 04:53:24.441861 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = 'f25003750af4' diff --git a/api/models/model.py b/api/models/model.py index c45b0e636b..afe2bc1628 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -1,8 +1,8 @@ import json import uuid -from core.file.upload_file_parser import UploadFileParser from core.file.tool_file_parser import ToolFileParser +from core.file.upload_file_parser import UploadFileParser from extensions.ext_database import db from flask import current_app, request from flask_login import UserMixin diff --git a/api/models/tools.py b/api/models/tools.py index 68668e2343..6b074786a2 100644 --- a/api/models/tools.py +++ b/api/models/tools.py @@ -2,16 +2,14 @@ import json from enum import Enum from typing import List -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy import ForeignKey - -from extensions.ext_database import db - -from core.tools.entities.tool_bundle import ApiBasedToolBundle from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_bundle import ApiBasedToolBundle from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolRuntimeVariablePool +from extensions.ext_database import db +from models.model import Account, App, Tenant +from sqlalchemy import ForeignKey +from sqlalchemy.dialects.postgresql import UUID -from models.model import Tenant, Account, App class BuiltinToolProvider(db.Model): """ diff --git a/api/services/account_service.py b/api/services/account_service.py index 18f72f9ef7..9a97d19889 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -8,7 +8,7 @@ from datetime import datetime, timedelta from hashlib import sha256 from typing import Any, Dict, Optional -from constants.languages import languages, language_timezone_mapping +from constants.languages import language_timezone_mapping, languages from events.tenant_event import tenant_was_created from extensions.ext_redis import redis_client from flask import current_app, session diff --git a/api/services/audio_service.py b/api/services/audio_service.py index bbd5bfb52b..4cc965bacf 100644 --- a/api/services/audio_service.py +++ b/api/services/audio_service.py @@ -3,11 +3,9 @@ from typing import Optional from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelType -from services.errors.audio import (AudioTooLargeServiceError, - NoAudioUploadedServiceError, - ProviderNotSupportTextToSpeechServiceError, +from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, - UnsupportedAudioTypeServiceError) + ProviderNotSupportTextToSpeechServiceError, UnsupportedAudioTypeServiceError) from werkzeug.datastructures import FileStorage FILE_SIZE = 15 diff --git a/api/services/tools_manage_service.py b/api/services/tools_manage_service.py index 6fb1853871..2a97b3e8b5 100644 --- a/api/services/tools_manage_service.py +++ b/api/services/tools_manage_service.py @@ -1,26 +1,23 @@ +import json from typing import List, Tuple -from flask import current_app - -from core.tools.tool_manager import ToolManager -from core.tools.entities.user_entities import UserToolProvider, UserTool -from core.tools.entities.tool_entities import ApiProviderSchemaType, ApiProviderAuthType, ToolProviderCredentials, \ - ToolCredentialsOption from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_bundle import ApiBasedToolBundle -from core.tools.provider.tool_provider import ToolProviderController +from core.tools.entities.tool_entities import (ApiProviderAuthType, ApiProviderSchemaType, ToolCredentialsOption, + ToolProviderCredentials) +from core.tools.entities.user_entities import UserTool, UserToolProvider +from core.tools.errors import ToolNotFoundError, ToolProviderCredentialValidationError, ToolProviderNotFoundError from core.tools.provider.api_tool_provider import ApiBasedToolProviderController -from core.tools.utils.parser import ApiBasedToolSchemaParser -from core.tools.utils.encoder import serialize_base_model_array, serialize_base_model_dict +from core.tools.provider.tool_provider import ToolProviderController +from core.tools.tool_manager import ToolManager from core.tools.utils.configuration import ToolConfiguration -from core.tools.errors import ToolProviderCredentialValidationError, ToolProviderNotFoundError, ToolNotFoundError - +from core.tools.utils.encoder import serialize_base_model_array, serialize_base_model_dict +from core.tools.utils.parser import ApiBasedToolSchemaParser from extensions.ext_database import db -from models.tools import BuiltinToolProvider, ApiToolProvider - +from flask import current_app from httpx import get +from models.tools import ApiToolProvider, BuiltinToolProvider -import json class ToolManageService: @staticmethod diff --git a/api/tasks/mail_invite_member_task.py b/api/tasks/mail_invite_member_task.py index bdb6dbacec..a7df7f9298 100644 --- a/api/tasks/mail_invite_member_task.py +++ b/api/tasks/mail_invite_member_task.py @@ -3,9 +3,10 @@ import time import click from celery import shared_task +from constants.languages import languages from extensions.ext_mail import mail from flask import current_app, render_template -from constants.languages import languages + @shared_task(queue='mail') def send_invite_member_mail_task(language: str, to: str, token: str, inviter_name: str, workspace_name: str): diff --git a/api/tests/integration_tests/model_runtime/bedrock/test_llm.py b/api/tests/integration_tests/model_runtime/bedrock/test_llm.py index c395a08fe2..750c049614 100644 --- a/api/tests/integration_tests/model_runtime/bedrock/test_llm.py +++ b/api/tests/integration_tests/model_runtime/bedrock/test_llm.py @@ -7,6 +7,7 @@ from core.model_runtime.entities.message_entities import AssistantPromptMessage, from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.bedrock.llm.llm import BedrockLargeLanguageModel + def test_validate_credentials(): model = BedrockLargeLanguageModel() diff --git a/api/tests/integration_tests/model_runtime/bedrock/test_provider.py b/api/tests/integration_tests/model_runtime/bedrock/test_provider.py index ce3d61c0f7..6819f8c9a1 100644 --- a/api/tests/integration_tests/model_runtime/bedrock/test_provider.py +++ b/api/tests/integration_tests/model_runtime/bedrock/test_provider.py @@ -4,6 +4,7 @@ import pytest from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.bedrock.bedrock import BedrockProvider + def test_validate_provider_credentials(): provider = BedrockProvider() diff --git a/api/tests/integration_tests/model_runtime/cohere/test_llm.py b/api/tests/integration_tests/model_runtime/cohere/test_llm.py index 613675c1fd..a3d054cacf 100644 --- a/api/tests/integration_tests/model_runtime/cohere/test_llm.py +++ b/api/tests/integration_tests/model_runtime/cohere/test_llm.py @@ -3,8 +3,7 @@ from typing import Generator import pytest from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta -from core.model_runtime.entities.message_entities import (AssistantPromptMessage, SystemPromptMessage, - UserPromptMessage) +from core.model_runtime.entities.message_entities import AssistantPromptMessage, SystemPromptMessage, UserPromptMessage from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.cohere.llm.llm import CohereLargeLanguageModel diff --git a/api/tests/integration_tests/model_runtime/ollama/test_llm.py b/api/tests/integration_tests/model_runtime/ollama/test_llm.py index 5543085a54..4265190f58 100644 --- a/api/tests/integration_tests/model_runtime/ollama/test_llm.py +++ b/api/tests/integration_tests/model_runtime/ollama/test_llm.py @@ -2,11 +2,10 @@ import os from typing import Generator import pytest - -from core.model_runtime.entities.message_entities import AssistantPromptMessage, UserPromptMessage, \ - SystemPromptMessage, TextPromptMessageContent, ImagePromptMessageContent -from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunkDelta, \ - LLMResultChunk +from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent, + SystemPromptMessage, TextPromptMessageContent, + UserPromptMessage) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.ollama.llm.llm import OllamaLargeLanguageModel diff --git a/api/tests/integration_tests/model_runtime/ollama/test_text_embedding.py b/api/tests/integration_tests/model_runtime/ollama/test_text_embedding.py index c5f5918235..e305226b85 100644 --- a/api/tests/integration_tests/model_runtime/ollama/test_text_embedding.py +++ b/api/tests/integration_tests/model_runtime/ollama/test_text_embedding.py @@ -1,7 +1,6 @@ import os import pytest - from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.ollama.text_embedding.text_embedding import OllamaEmbeddingModel diff --git a/api/tests/integration_tests/model_runtime/zhipuai/test_llm.py b/api/tests/integration_tests/model_runtime/zhipuai/test_llm.py index a04e607a5d..5ca1ee44b8 100644 --- a/api/tests/integration_tests/model_runtime/zhipuai/test_llm.py +++ b/api/tests/integration_tests/model_runtime/zhipuai/test_llm.py @@ -3,8 +3,8 @@ from typing import Generator import pytest from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta -from core.model_runtime.entities.message_entities import (AssistantPromptMessage, SystemPromptMessage, - UserPromptMessage, PromptMessageTool) +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessageTool, + SystemPromptMessage, UserPromptMessage) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.zhipuai.llm.llm import ZhipuAILargeLanguageModel diff --git a/api/tests/integration_tests/tools/test_all_provider.py b/api/tests/integration_tests/tools/test_all_provider.py index 83eccb1b11..c846ddecfb 100644 --- a/api/tests/integration_tests/tools/test_all_provider.py +++ b/api/tests/integration_tests/tools/test_all_provider.py @@ -1,5 +1,6 @@ from core.tools.tool_manager import ToolManager + def test_tool_providers(): """ Test that all tool providers can be loaded diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index d9d8ba8d52..cd22a73cac 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3.1' services: # API service api: - image: langgenius/dify-api:0.5.2 + image: langgenius/dify-api:0.5.3 restart: always environment: # Startup mode, 'api' starts the API server. @@ -130,7 +130,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.5.2 + image: langgenius/dify-api:0.5.3 restart: always environment: # Startup mode, 'worker' starts the Celery worker for processing the queue. @@ -201,7 +201,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.5.2 + image: langgenius/dify-web:0.5.3 restart: always environment: EDITION: SELF_HOSTED diff --git a/web/package.json b/web/package.json index 4ff7ad11d4..60a59b5061 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "dify-web", - "version": "0.5.2", + "version": "0.5.3", "private": true, "scripts": { "dev": "next dev",