diff --git a/web/app/components/app/annotation/index.tsx b/web/app/components/app/annotation/index.tsx
index e287f6970f..46ecdd5480 100644
--- a/web/app/components/app/annotation/index.tsx
+++ b/web/app/components/app/annotation/index.tsx
@@ -27,7 +27,7 @@ import AnnotationFullModal from '@/app/components/billing/annotation-full/modal'
 import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
 import type { App } from '@/types/app'
 
-type Props = {
+interface Props {
   appDetail: App
 }
 
@@ -283,7 +283,6 @@ const Annotation: FC<Props> = ({
               if (
                 embeddingModel.embedding_model_name !== annotationConfig?.embedding_model?.embedding_model_name
                 || embeddingModel.embedding_provider_name !== annotationConfig?.embedding_model?.embedding_provider_name
-                || embeddingModel.plugin_id !== annotationConfig?.embedding_model?.plugin_id
               ) {
                 const { job_id: jobId }: any = await updateAnnotationStatus(appDetail.id, AnnotationEnableStatus.enable, embeddingModel, score)
                 await ensureJobCompleted(jobId, AnnotationEnableStatus.enable)
diff --git a/web/app/components/app/annotation/type.ts b/web/app/components/app/annotation/type.ts
index 28811f4617..5df6f51ace 100644
--- a/web/app/components/app/annotation/type.ts
+++ b/web/app/components/app/annotation/type.ts
@@ -23,9 +23,8 @@ export type HitHistoryItem = {
 }
 
 export type EmbeddingModelConfig = {
-  plugin_id: string
-  embedding_model_name: string
   embedding_provider_name: string
+  embedding_model_name: string
 }
 
 export enum AnnotationEnableStatus {
diff --git a/web/app/components/app/configuration/toolbox/annotation/config-param.tsx b/web/app/components/app/configuration/toolbox/annotation/config-param.tsx
new file mode 100644
index 0000000000..e418a76c34
--- /dev/null
+++ b/web/app/components/app/configuration/toolbox/annotation/config-param.tsx
@@ -0,0 +1,124 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import { useContext } from 'use-context-selector'
+import { usePathname, useRouter } from 'next/navigation'
+import ConfigParamModal from './config-param-modal'
+import Panel from '@/app/components/app/configuration/base/feature-panel'
+import { MessageFast } from '@/app/components/base/icons/src/vender/solid/communication'
+import Tooltip from '@/app/components/base/tooltip'
+import { LinkExternal02, Settings04 } from '@/app/components/base/icons/src/vender/line/general'
+import ConfigContext from '@/context/debug-configuration'
+import type { EmbeddingModelConfig } from '@/app/components/app/annotation/type'
+import { fetchAnnotationConfig, updateAnnotationScore } from '@/service/annotation'
+import type { AnnotationReplyConfig as AnnotationReplyConfigType } from '@/models/debug'
+
+type Props = {
+  onEmbeddingChange: (embeddingModel: EmbeddingModelConfig) => void
+  onScoreChange: (score: number, embeddingModel?: EmbeddingModelConfig) => void
+}
+
+export const Item: FC<{ title: string; tooltip: string; children: JSX.Element }> = ({
+  title,
+  tooltip,
+  children,
+}) => {
+  return (
+    <div>
+      <div className='flex items-center space-x-1'>
+        <div>{title}</div>
+        <Tooltip
+          popupContent={
+            <div className='max-w-[200px] leading-[18px] text-[13px] font-medium text-gray-800'>{tooltip}</div>
+          }
+        />
+      </div>
+      <div>{children}</div>
+    </div>
+  )
+}
+
+const AnnotationReplyConfig: FC<Props> = ({
+  onEmbeddingChange,
+  onScoreChange,
+}) => {
+  const { t } = useTranslation()
+  const router = useRouter()
+  const pathname = usePathname()
+  const matched = pathname.match(/\/app\/([^/]+)/)
+  const appId = (matched?.length && matched[1]) ? matched[1] : ''
+  const {
+    annotationConfig,
+  } = useContext(ConfigContext)
+
+  const [isShowEdit, setIsShowEdit] = React.useState(false)
+
+  return (
+    <>
+      <Panel
+        className="mt-4"
+        headerIcon={
+          <MessageFast className='w-4 h-4 text-[#444CE7]' />
+        }
+        title={t('appDebug.feature.annotation.title')}
+        headerRight={
+          <div className='flex items-center'>
+            <div
+              className='flex items-center rounded-md h-7 px-3 space-x-1 text-gray-700 cursor-pointer hover:bg-gray-200'
+              onClick={() => { setIsShowEdit(true) }}
+            >
+              <Settings04 className="w-[14px] h-[14px]" />
+              <div className='text-xs font-medium'>
+
+                {t('common.operation.params')}
+              </div>
+            </div>
+            <div
+              className='ml-1 flex items-center h-7 px-3 space-x-1 leading-[18px] text-xs font-medium text-gray-700 rounded-md cursor-pointer hover:bg-gray-200'
+              onClick={() => {
+                router.push(`/app/${appId}/annotations`)
+              }}>
+              <div>{t('appDebug.feature.annotation.cacheManagement')}</div>
+              <LinkExternal02 className='w-3.5 h-3.5' />
+            </div>
+          </div>
+        }
+        noBodySpacing
+      />
+      {isShowEdit && (
+        <ConfigParamModal
+          appId={appId}
+          isShow
+          onHide={() => {
+            setIsShowEdit(false)
+          }}
+          onSave={async (embeddingModel, score) => {
+            const annotationConfig = await fetchAnnotationConfig(appId) as AnnotationReplyConfigType
+            let isEmbeddingModelChanged = false
+            if (
+              embeddingModel.embedding_model_name !== annotationConfig.embedding_model.embedding_model_name
+              || embeddingModel.embedding_provider_name !== annotationConfig.embedding_model.embedding_provider_name
+            ) {
+              await onEmbeddingChange(embeddingModel)
+              isEmbeddingModelChanged = true
+            }
+
+            if (score !== annotationConfig.score_threshold) {
+              await updateAnnotationScore(appId, annotationConfig.id, score)
+              if (isEmbeddingModelChanged)
+                onScoreChange(score, embeddingModel)
+
+              else
+                onScoreChange(score)
+            }
+
+            setIsShowEdit(false)
+          }}
+          annotationConfig={annotationConfig}
+        />
+      )}
+    </>
+  )
+}
+export default React.memo(AnnotationReplyConfig)
diff --git a/web/app/components/app/configuration/toolbox/index.tsx b/web/app/components/app/configuration/toolbox/index.tsx
new file mode 100644
index 0000000000..00ea301a42
--- /dev/null
+++ b/web/app/components/app/configuration/toolbox/index.tsx
@@ -0,0 +1,45 @@
+'use client'
+
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import GroupName from '../base/group-name'
+import Moderation from './moderation'
+import Annotation from './annotation/config-param'
+import type { EmbeddingModelConfig } from '@/app/components/app/annotation/type'
+
+export type ToolboxProps = {
+  showModerationSettings: boolean
+  showAnnotation: boolean
+  onEmbeddingChange: (embeddingModel: EmbeddingModelConfig) => void
+  onScoreChange: (score: number, embeddingModel?: EmbeddingModelConfig) => void
+}
+
+const Toolbox: FC<ToolboxProps> = ({
+  showModerationSettings,
+  showAnnotation,
+  onEmbeddingChange,
+  onScoreChange,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='mt-7'>
+      <GroupName name={t('appDebug.feature.toolbox.title')} />
+      {
+        showModerationSettings && (
+          <Moderation />
+        )
+      }
+      {
+        showAnnotation && (
+          <Annotation
+            onEmbeddingChange={onEmbeddingChange}
+            onScoreChange={onScoreChange}
+          />
+        )
+      }
+    </div>
+  )
+}
+export default React.memo(Toolbox)
diff --git a/web/app/components/base/features/new-feature-panel/annotation-reply/config-param-modal.tsx b/web/app/components/base/features/new-feature-panel/annotation-reply/config-param-modal.tsx
index 6309498d20..801f1348ee 100644
--- a/web/app/components/base/features/new-feature-panel/annotation-reply/config-param-modal.tsx
+++ b/web/app/components/base/features/new-feature-panel/annotation-reply/config-param-modal.tsx
@@ -18,7 +18,6 @@ type Props = {
   isShow: boolean
   onHide: () => void
   onSave: (embeddingModel: {
-    plugin_id: string
     embedding_provider_name: string
     embedding_model_name: string
   }, score: number) => void
@@ -44,13 +43,11 @@ const ConfigParamModal: FC<Props> = ({
   const [isLoading, setLoading] = useState(false)
   const [embeddingModel, setEmbeddingModel] = useState(oldAnnotationConfig.embedding_model
     ? {
-      plugin_id: oldAnnotationConfig.embedding_model.plugin_id,
       providerName: oldAnnotationConfig.embedding_model.embedding_provider_name,
       modelName: oldAnnotationConfig.embedding_model.embedding_model_name,
     }
     : (embeddingsDefaultModel
       ? {
-        plugin_id: embeddingsDefaultModel.provider.plugin_id,
         providerName: embeddingsDefaultModel.provider.provider,
         modelName: embeddingsDefaultModel.model,
       }
@@ -70,7 +67,6 @@ const ConfigParamModal: FC<Props> = ({
     }
     setLoading(true)
     await onSave({
-      plugin_id: embeddingModel.plugin_id,
       embedding_provider_name: embeddingModel.providerName,
       embedding_model_name: embeddingModel.modelName,
     }, annotationConfig.score_threshold)
@@ -111,14 +107,12 @@ const ConfigParamModal: FC<Props> = ({
           <div className='pt-1'>
             <ModelSelector
               defaultModel={embeddingModel && {
-                plugin_id: '',
                 provider: embeddingModel.providerName,
                 model: embeddingModel.modelName,
               }}
               modelList={embeddingsModelList}
               onSelect={(val) => {
                 setEmbeddingModel({
-                  plugin_id: val.plugin_id,
                   providerName: val.provider,
                   modelName: val.model,
                 })
diff --git a/web/models/debug.ts b/web/models/debug.ts
index fe85544fa7..64a6cb7f84 100644
--- a/web/models/debug.ts
+++ b/web/models/debug.ts
@@ -93,7 +93,6 @@ export type AnnotationReplyConfig = {
   enabled: boolean
   score_threshold: number
   embedding_model: {
-    plugin_id: string
     embedding_provider_name: string
     embedding_model_name: string
   }
diff --git a/web/service/annotation.ts b/web/service/annotation.ts
index 868f82bedc..5096a4f58a 100644
--- a/web/service/annotation.ts
+++ b/web/service/annotation.ts
@@ -13,9 +13,7 @@ export const updateAnnotationStatus = (appId: string, action: AnnotationEnableSt
   if (embeddingModel) {
     body = {
       ...body,
-      embedding_model_plugin_id: embeddingModel.plugin_id,
-      embedding_provider_name: embeddingModel.embedding_provider_name,
-      embedding_model_name: embeddingModel.embedding_model_name,
+      ...embeddingModel,
     }
   }