feat: add the audio tool (#10695)

api/core/tools/provider/builtin/audio/_assets/icon.svg

@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="200" height="200" viewBox="0 0 200 200" fill="none">
+<path d="M167.358 102.395C167.358 117.174 157.246 129.18 144.61 131.027H137.861C125.225 129.18 115.113 117.174 115.113 102.395H100.792C100.792 123.637 115.118 142.106 133.653 145.801V164.276H147.139V145.801C165.674 142.106 180 124.558 180 102.4H167.358V102.395ZM154.717 62.677C154.717 53.4397 147.979 46.9765 140.396 46.9765C138.523 46.9446 136.663 47.3273 134.924 48.1024C133.185 48.8775 131.603 50.0294 130.27 51.4909C128.936 52.9524 127.878 54.6943 127.157 56.6148C126.436 58.5354 126.066 60.5962 126.07 62.677V78.3775H154.717V70.4478V62.677ZM126.07 102.395C126.07 111.632 132.813 118.095 140.396 118.095C142.269 118.127 144.13 117.744 145.868 116.969C147.607 116.194 149.189 115.042 150.523 113.581C151.856 112.119 152.914 110.377 153.635 108.457C154.356 106.536 154.726 104.475 154.722 102.395V86.694H126.07V102.395ZM92.1297 45.8938L70.4796 21.7595L69.4235 20.5865L59.604 20L68.3674 20.5865L67.3113 21.7654L64.1429 25.2961L63.6149 25.8826L64.1429 27.0614L66.2552 29.4133L77.8723 42.3631H54.1099C35.1 43.5361 20.3146 61.1896 20.3146 81.7874V83.5527H28.2354V81.7932C28.2354 65.8992 39.8525 52.3628 54.1099 51.1899H77.8723L66.2552 64.1338L64.671 65.8992L64.1429 67.0722L63.6149 67.6645L64.1429 68.251L68.3674 72.9606L68.8954 73.5471L69.4235 72.9606L74.1759 67.6645L92.1297 47.6591L92.6578 47.0727L92.1297 45.8938ZM20 95.8496V118.213H30.033V107.034H50.099V168.821H40.066V180H70.165V168.821H60.132V107.034H80.198V118.213H90.231V95.8496H20Z" fill="#FF0099"/>
+</svg>

api/core/tools/provider/builtin/audio/audio.py

@@ -0,0 +1,6 @@
+from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController
+
+
+class AudioToolProvider(BuiltinToolProviderController):
+    def _validate_credentials(self, credentials: dict) -> None:
+        pass

api/core/tools/provider/builtin/audio/audio.yaml

@@ -0,0 +1,11 @@
+identity:
+  author: hjlarry
+  name: audio
+  label:
+    en_US: Audio
+  description:
+    en_US: A tool for tts and asr.
+    zh_Hans: 一个用于文本转语音和语音转文本的工具。
+  icon: icon.svg
+  tags:
+    - utilities

api/core/tools/provider/builtin/audio/tools/asr.py

@@ -0,0 +1,70 @@
+import io
+from typing import Any
+
+from core.file.enums import FileType
+from core.file.file_manager import download
+from core.model_manager import ModelManager
+from core.model_runtime.entities.model_entities import ModelType
+from core.tools.entities.common_entities import I18nObject
+from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolParameterOption
+from core.tools.tool.builtin_tool import BuiltinTool
+from services.model_provider_service import ModelProviderService
+
+
+class ASRTool(BuiltinTool):
+    def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> list[ToolInvokeMessage]:
+        file = tool_parameters.get("audio_file")
+        if file.type != FileType.AUDIO:
+            return [self.create_text_message("not a valid audio file")]
+        audio_binary = io.BytesIO(download(file))
+        audio_binary.name = "temp.mp3"
+        provider, model = tool_parameters.get("model").split("#")
+        model_manager = ModelManager()
+        model_instance = model_manager.get_model_instance(
+            tenant_id=self.runtime.tenant_id,
+            provider=provider,
+            model_type=ModelType.SPEECH2TEXT,
+            model=model,
+        )
+        text = model_instance.invoke_speech2text(
+            file=audio_binary,
+            user=user_id,
+        )
+        return [self.create_text_message(text)]
+
+    def get_available_models(self) -> list[tuple[str, str]]:
+        model_provider_service = ModelProviderService()
+        models = model_provider_service.get_models_by_model_type(
+            tenant_id=self.runtime.tenant_id, model_type="speech2text"
+        )
+        items = []
+        for provider_model in models:
+            provider = provider_model.provider
+            for model in provider_model.models:
+                items.append((provider, model.model))
+        return items
+
+    def get_runtime_parameters(self) -> list[ToolParameter]:
+        parameters = []
+
+        options = []
+        for provider, model in self.get_available_models():
+            option = ToolParameterOption(value=f"{provider}#{model}", label=I18nObject(en_US=f"{model}({provider})"))
+            options.append(option)
+
+        parameters.append(
+            ToolParameter(
+                name="model",
+                label=I18nObject(en_US="Model", zh_Hans="Model"),
+                human_description=I18nObject(
+                    en_US="All available ASR models",
+                    zh_Hans="所有可用的 ASR 模型",
+                ),
+                type=ToolParameter.ToolParameterType.SELECT,
+                form=ToolParameter.ToolParameterForm.FORM,
+                required=True,
+                default=options[0].value,
+                options=options,
+            )
+        )
+        return parameters

api/core/tools/provider/builtin/audio/tools/asr.yaml

@@ -0,0 +1,22 @@
+identity:
+  name: asr
+  author: hjlarry
+  label:
+    en_US: Speech To Text
+description:
+  human:
+    en_US: Convert audio file to text.
+    zh_Hans: 将音频文件转换为文本。
+  llm: Convert audio file to text.
+parameters:
+  - name: audio_file
+    type: file
+    required: true
+    label:
+      en_US: Audio File
+      zh_Hans: 音频文件
+    human_description:
+      en_US: The audio file to be converted.
+      zh_Hans: 要转换的音频文件。
+    llm_description: The audio file to be converted.
+    form: llm

api/core/tools/provider/builtin/audio/tools/tts.py

@@ -0,0 +1,90 @@
+import io
+from typing import Any
+
+from core.model_manager import ModelManager
+from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
+from core.tools.entities.common_entities import I18nObject
+from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolParameterOption
+from core.tools.tool.builtin_tool import BuiltinTool
+from services.model_provider_service import ModelProviderService
+
+
+class TTSTool(BuiltinTool):
+    def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> list[ToolInvokeMessage]:
+        provider, model = tool_parameters.get("model").split("#")
+        voice = tool_parameters.get(f"voice#{provider}#{model}")
+        model_manager = ModelManager()
+        model_instance = model_manager.get_model_instance(
+            tenant_id=self.runtime.tenant_id,
+            provider=provider,
+            model_type=ModelType.TTS,
+            model=model,
+        )
+        tts = model_instance.invoke_tts(
+            content_text=tool_parameters.get("text"),
+            user=user_id,
+            tenant_id=self.runtime.tenant_id,
+            voice=voice,
+        )
+        buffer = io.BytesIO()
+        for chunk in tts:
+            buffer.write(chunk)
+
+        wav_bytes = buffer.getvalue()
+        return [
+            self.create_text_message("Audio generated successfully"),
+            self.create_blob_message(
+                blob=wav_bytes,
+                meta={"mime_type": "audio/x-wav"},
+                save_as=self.VariableKey.AUDIO,
+            ),
+        ]
+
+    def get_available_models(self) -> list[tuple[str, str, list[Any]]]:
+        model_provider_service = ModelProviderService()
+        models = model_provider_service.get_models_by_model_type(tenant_id=self.runtime.tenant_id, model_type="tts")
+        items = []
+        for provider_model in models:
+            provider = provider_model.provider
+            for model in provider_model.models:
+                voices = model.model_properties.get(ModelPropertyKey.VOICES, [])
+                items.append((provider, model.model, voices))
+        return items
+
+    def get_runtime_parameters(self) -> list[ToolParameter]:
+        parameters = []
+
+        options = []
+        for provider, model, voices in self.get_available_models():
+            option = ToolParameterOption(value=f"{provider}#{model}", label=I18nObject(en_US=f"{model}({provider})"))
+            options.append(option)
+            parameters.append(
+                ToolParameter(
+                    name=f"voice#{provider}#{model}",
+                    label=I18nObject(en_US=f"Voice of {model}({provider})"),
+                    type=ToolParameter.ToolParameterType.SELECT,
+                    form=ToolParameter.ToolParameterForm.FORM,
+                    options=[
+                        ToolParameterOption(value=voice.get("mode"), label=I18nObject(en_US=voice.get("name")))
+                        for voice in voices
+                    ],
+                )
+            )
+
+        parameters.insert(
+            0,
+            ToolParameter(
+                name="model",
+                label=I18nObject(en_US="Model", zh_Hans="Model"),
+                human_description=I18nObject(
+                    en_US="All available TTS models",
+                    zh_Hans="所有可用的 TTS 模型",
+                ),
+                type=ToolParameter.ToolParameterType.SELECT,
+                form=ToolParameter.ToolParameterForm.FORM,
+                required=True,
+                default=options[0].value,
+                options=options,
+            ),
+        )
+        return parameters

api/core/tools/provider/builtin/audio/tools/tts.yaml

@@ -0,0 +1,22 @@
+identity:
+  name: tts
+  author: hjlarry
+  label:
+    en_US: Text To Speech
+description:
+  human:
+    en_US: Convert text to audio file.
+    zh_Hans: 将文本转换为音频文件。
+  llm: Convert text to audio file.
+parameters:
+  - name: text
+    type: string
+    required: true
+    label:
+      en_US: Text
+      zh_Hans: 文本
+    human_description:
+      en_US: The text to be converted.
+      zh_Hans: 要转换的文本。
+    llm_description: The text to be converted.
+    form: llm