mirror of
https://github.com/langgenius/dify.git
synced 2024-11-16 19:59:50 +08:00
447 lines
17 KiB
Python
447 lines
17 KiB
Python
import datetime
|
|
import uuid
|
|
|
|
import pandas as pd
|
|
from flask_login import current_user
|
|
from sqlalchemy import or_
|
|
from werkzeug.datastructures import FileStorage
|
|
from werkzeug.exceptions import NotFound
|
|
|
|
from extensions.ext_database import db
|
|
from extensions.ext_redis import redis_client
|
|
from models.model import App, AppAnnotationHitHistory, AppAnnotationSetting, Message, MessageAnnotation
|
|
from services.feature_service import FeatureService
|
|
from tasks.annotation.add_annotation_to_index_task import add_annotation_to_index_task
|
|
from tasks.annotation.batch_import_annotations_task import batch_import_annotations_task
|
|
from tasks.annotation.delete_annotation_index_task import delete_annotation_index_task
|
|
from tasks.annotation.disable_annotation_reply_task import disable_annotation_reply_task
|
|
from tasks.annotation.enable_annotation_reply_task import enable_annotation_reply_task
|
|
from tasks.annotation.update_annotation_to_index_task import update_annotation_to_index_task
|
|
|
|
|
|
class AppAnnotationService:
|
|
@classmethod
|
|
def up_insert_app_annotation_from_message(cls, args: dict, app_id: str) -> MessageAnnotation:
|
|
# get app info
|
|
app = (
|
|
db.session.query(App)
|
|
.filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
|
|
.first()
|
|
)
|
|
|
|
if not app:
|
|
raise NotFound("App not found")
|
|
if args.get("message_id"):
|
|
message_id = str(args["message_id"])
|
|
# get message info
|
|
message = db.session.query(Message).filter(Message.id == message_id, Message.app_id == app.id).first()
|
|
|
|
if not message:
|
|
raise NotFound("Message Not Exists.")
|
|
|
|
annotation = message.annotation
|
|
# save the message annotation
|
|
if annotation:
|
|
annotation.content = args["answer"]
|
|
annotation.question = args["question"]
|
|
else:
|
|
annotation = MessageAnnotation(
|
|
app_id=app.id,
|
|
conversation_id=message.conversation_id,
|
|
message_id=message.id,
|
|
content=args["answer"],
|
|
question=args["question"],
|
|
account_id=current_user.id,
|
|
)
|
|
else:
|
|
annotation = MessageAnnotation(
|
|
app_id=app.id, content=args["answer"], question=args["question"], account_id=current_user.id
|
|
)
|
|
db.session.add(annotation)
|
|
db.session.commit()
|
|
# if annotation reply is enabled , add annotation to index
|
|
annotation_setting = (
|
|
db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app_id).first()
|
|
)
|
|
if annotation_setting:
|
|
add_annotation_to_index_task.delay(
|
|
annotation.id,
|
|
args["question"],
|
|
current_user.current_tenant_id,
|
|
app_id,
|
|
annotation_setting.collection_binding_id,
|
|
)
|
|
return annotation
|
|
|
|
@classmethod
|
|
def enable_app_annotation(cls, args: dict, app_id: str) -> dict:
|
|
enable_app_annotation_key = "enable_app_annotation_{}".format(str(app_id))
|
|
cache_result = redis_client.get(enable_app_annotation_key)
|
|
if cache_result is not None:
|
|
return {"job_id": cache_result, "job_status": "processing"}
|
|
|
|
# async job
|
|
job_id = str(uuid.uuid4())
|
|
enable_app_annotation_job_key = "enable_app_annotation_job_{}".format(str(job_id))
|
|
# send batch add segments task
|
|
redis_client.setnx(enable_app_annotation_job_key, "waiting")
|
|
enable_annotation_reply_task.delay(
|
|
str(job_id),
|
|
app_id,
|
|
current_user.id,
|
|
current_user.current_tenant_id,
|
|
args["score_threshold"],
|
|
args["embedding_provider_name"],
|
|
args["embedding_model_name"],
|
|
)
|
|
return {"job_id": job_id, "job_status": "waiting"}
|
|
|
|
@classmethod
|
|
def disable_app_annotation(cls, app_id: str) -> dict:
|
|
disable_app_annotation_key = "disable_app_annotation_{}".format(str(app_id))
|
|
cache_result = redis_client.get(disable_app_annotation_key)
|
|
if cache_result is not None:
|
|
return {"job_id": cache_result, "job_status": "processing"}
|
|
|
|
# async job
|
|
job_id = str(uuid.uuid4())
|
|
disable_app_annotation_job_key = "disable_app_annotation_job_{}".format(str(job_id))
|
|
# send batch add segments task
|
|
redis_client.setnx(disable_app_annotation_job_key, "waiting")
|
|
disable_annotation_reply_task.delay(str(job_id), app_id, current_user.current_tenant_id)
|
|
return {"job_id": job_id, "job_status": "waiting"}
|
|
|
|
@classmethod
|
|
def get_annotation_list_by_app_id(cls, app_id: str, page: int, limit: int, keyword: str):
|
|
# get app info
|
|
app = (
|
|
db.session.query(App)
|
|
.filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
|
|
.first()
|
|
)
|
|
|
|
if not app:
|
|
raise NotFound("App not found")
|
|
if keyword:
|
|
annotations = (
|
|
db.session.query(MessageAnnotation)
|
|
.filter(MessageAnnotation.app_id == app_id)
|
|
.filter(
|
|
or_(
|
|
MessageAnnotation.question.ilike("%{}%".format(keyword)),
|
|
MessageAnnotation.content.ilike("%{}%".format(keyword)),
|
|
)
|
|
)
|
|
.order_by(MessageAnnotation.created_at.desc())
|
|
.paginate(page=page, per_page=limit, max_per_page=100, error_out=False)
|
|
)
|
|
else:
|
|
annotations = (
|
|
db.session.query(MessageAnnotation)
|
|
.filter(MessageAnnotation.app_id == app_id)
|
|
.order_by(MessageAnnotation.created_at.desc())
|
|
.paginate(page=page, per_page=limit, max_per_page=100, error_out=False)
|
|
)
|
|
return annotations.items, annotations.total
|
|
|
|
@classmethod
|
|
def export_annotation_list_by_app_id(cls, app_id: str):
|
|
# get app info
|
|
app = (
|
|
db.session.query(App)
|
|
.filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
|
|
.first()
|
|
)
|
|
|
|
if not app:
|
|
raise NotFound("App not found")
|
|
annotations = (
|
|
db.session.query(MessageAnnotation)
|
|
.filter(MessageAnnotation.app_id == app_id)
|
|
.order_by(MessageAnnotation.created_at.desc())
|
|
.all()
|
|
)
|
|
return annotations
|
|
|
|
@classmethod
|
|
def insert_app_annotation_directly(cls, args: dict, app_id: str) -> MessageAnnotation:
|
|
# get app info
|
|
app = (
|
|
db.session.query(App)
|
|
.filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
|
|
.first()
|
|
)
|
|
|
|
if not app:
|
|
raise NotFound("App not found")
|
|
|
|
annotation = MessageAnnotation(
|
|
app_id=app.id, content=args["answer"], question=args["question"], account_id=current_user.id
|
|
)
|
|
db.session.add(annotation)
|
|
db.session.commit()
|
|
# if annotation reply is enabled , add annotation to index
|
|
annotation_setting = (
|
|
db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app_id).first()
|
|
)
|
|
if annotation_setting:
|
|
add_annotation_to_index_task.delay(
|
|
annotation.id,
|
|
args["question"],
|
|
current_user.current_tenant_id,
|
|
app_id,
|
|
annotation_setting.collection_binding_id,
|
|
)
|
|
return annotation
|
|
|
|
@classmethod
|
|
def update_app_annotation_directly(cls, args: dict, app_id: str, annotation_id: str):
|
|
# get app info
|
|
app = (
|
|
db.session.query(App)
|
|
.filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
|
|
.first()
|
|
)
|
|
|
|
if not app:
|
|
raise NotFound("App not found")
|
|
|
|
annotation = db.session.query(MessageAnnotation).filter(MessageAnnotation.id == annotation_id).first()
|
|
|
|
if not annotation:
|
|
raise NotFound("Annotation not found")
|
|
|
|
annotation.content = args["answer"]
|
|
annotation.question = args["question"]
|
|
|
|
db.session.commit()
|
|
# if annotation reply is enabled , add annotation to index
|
|
app_annotation_setting = (
|
|
db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app_id).first()
|
|
)
|
|
|
|
if app_annotation_setting:
|
|
update_annotation_to_index_task.delay(
|
|
annotation.id,
|
|
annotation.question,
|
|
current_user.current_tenant_id,
|
|
app_id,
|
|
app_annotation_setting.collection_binding_id,
|
|
)
|
|
|
|
return annotation
|
|
|
|
@classmethod
|
|
def delete_app_annotation(cls, app_id: str, annotation_id: str):
|
|
# get app info
|
|
app = (
|
|
db.session.query(App)
|
|
.filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
|
|
.first()
|
|
)
|
|
|
|
if not app:
|
|
raise NotFound("App not found")
|
|
|
|
annotation = db.session.query(MessageAnnotation).filter(MessageAnnotation.id == annotation_id).first()
|
|
|
|
if not annotation:
|
|
raise NotFound("Annotation not found")
|
|
|
|
db.session.delete(annotation)
|
|
|
|
annotation_hit_histories = (
|
|
db.session.query(AppAnnotationHitHistory)
|
|
.filter(AppAnnotationHitHistory.annotation_id == annotation_id)
|
|
.all()
|
|
)
|
|
if annotation_hit_histories:
|
|
for annotation_hit_history in annotation_hit_histories:
|
|
db.session.delete(annotation_hit_history)
|
|
|
|
db.session.commit()
|
|
# if annotation reply is enabled , delete annotation index
|
|
app_annotation_setting = (
|
|
db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app_id).first()
|
|
)
|
|
|
|
if app_annotation_setting:
|
|
delete_annotation_index_task.delay(
|
|
annotation.id, app_id, current_user.current_tenant_id, app_annotation_setting.collection_binding_id
|
|
)
|
|
|
|
@classmethod
|
|
def batch_import_app_annotations(cls, app_id, file: FileStorage) -> dict:
|
|
# get app info
|
|
app = (
|
|
db.session.query(App)
|
|
.filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
|
|
.first()
|
|
)
|
|
|
|
if not app:
|
|
raise NotFound("App not found")
|
|
|
|
try:
|
|
# Skip the first row
|
|
df = pd.read_csv(file)
|
|
result = []
|
|
for index, row in df.iterrows():
|
|
content = {"question": row[0], "answer": row[1]}
|
|
result.append(content)
|
|
if len(result) == 0:
|
|
raise ValueError("The CSV file is empty.")
|
|
# check annotation limit
|
|
features = FeatureService.get_features(current_user.current_tenant_id)
|
|
if features.billing.enabled:
|
|
annotation_quota_limit = features.annotation_quota_limit
|
|
if annotation_quota_limit.limit < len(result) + annotation_quota_limit.size:
|
|
raise ValueError("The number of annotations exceeds the limit of your subscription.")
|
|
# async job
|
|
job_id = str(uuid.uuid4())
|
|
indexing_cache_key = "app_annotation_batch_import_{}".format(str(job_id))
|
|
# send batch add segments task
|
|
redis_client.setnx(indexing_cache_key, "waiting")
|
|
batch_import_annotations_task.delay(
|
|
str(job_id), result, app_id, current_user.current_tenant_id, current_user.id
|
|
)
|
|
except Exception as e:
|
|
return {"error_msg": str(e)}
|
|
return {"job_id": job_id, "job_status": "waiting"}
|
|
|
|
@classmethod
|
|
def get_annotation_hit_histories(cls, app_id: str, annotation_id: str, page, limit):
|
|
# get app info
|
|
app = (
|
|
db.session.query(App)
|
|
.filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
|
|
.first()
|
|
)
|
|
|
|
if not app:
|
|
raise NotFound("App not found")
|
|
|
|
annotation = db.session.query(MessageAnnotation).filter(MessageAnnotation.id == annotation_id).first()
|
|
|
|
if not annotation:
|
|
raise NotFound("Annotation not found")
|
|
|
|
annotation_hit_histories = (
|
|
db.session.query(AppAnnotationHitHistory)
|
|
.filter(
|
|
AppAnnotationHitHistory.app_id == app_id,
|
|
AppAnnotationHitHistory.annotation_id == annotation_id,
|
|
)
|
|
.order_by(AppAnnotationHitHistory.created_at.desc())
|
|
.paginate(page=page, per_page=limit, max_per_page=100, error_out=False)
|
|
)
|
|
return annotation_hit_histories.items, annotation_hit_histories.total
|
|
|
|
@classmethod
|
|
def get_annotation_by_id(cls, annotation_id: str) -> MessageAnnotation | None:
|
|
annotation = db.session.query(MessageAnnotation).filter(MessageAnnotation.id == annotation_id).first()
|
|
|
|
if not annotation:
|
|
return None
|
|
return annotation
|
|
|
|
@classmethod
|
|
def add_annotation_history(
|
|
cls,
|
|
annotation_id: str,
|
|
app_id: str,
|
|
annotation_question: str,
|
|
annotation_content: str,
|
|
query: str,
|
|
user_id: str,
|
|
message_id: str,
|
|
from_source: str,
|
|
score: float,
|
|
):
|
|
# add hit count to annotation
|
|
db.session.query(MessageAnnotation).filter(MessageAnnotation.id == annotation_id).update(
|
|
{MessageAnnotation.hit_count: MessageAnnotation.hit_count + 1}, synchronize_session=False
|
|
)
|
|
|
|
annotation_hit_history = AppAnnotationHitHistory(
|
|
annotation_id=annotation_id,
|
|
app_id=app_id,
|
|
account_id=user_id,
|
|
question=query,
|
|
source=from_source,
|
|
score=score,
|
|
message_id=message_id,
|
|
annotation_question=annotation_question,
|
|
annotation_content=annotation_content,
|
|
)
|
|
db.session.add(annotation_hit_history)
|
|
db.session.commit()
|
|
|
|
@classmethod
|
|
def get_app_annotation_setting_by_app_id(cls, app_id: str):
|
|
# get app info
|
|
app = (
|
|
db.session.query(App)
|
|
.filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
|
|
.first()
|
|
)
|
|
|
|
if not app:
|
|
raise NotFound("App not found")
|
|
|
|
annotation_setting = (
|
|
db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app_id).first()
|
|
)
|
|
if annotation_setting:
|
|
collection_binding_detail = annotation_setting.collection_binding_detail
|
|
return {
|
|
"id": annotation_setting.id,
|
|
"enabled": True,
|
|
"score_threshold": annotation_setting.score_threshold,
|
|
"embedding_model": {
|
|
"embedding_provider_name": collection_binding_detail.provider_name,
|
|
"embedding_model_name": collection_binding_detail.model_name,
|
|
},
|
|
}
|
|
return {"enabled": False}
|
|
|
|
@classmethod
|
|
def update_app_annotation_setting(cls, app_id: str, annotation_setting_id: str, args: dict):
|
|
# get app info
|
|
app = (
|
|
db.session.query(App)
|
|
.filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id, App.status == "normal")
|
|
.first()
|
|
)
|
|
|
|
if not app:
|
|
raise NotFound("App not found")
|
|
|
|
annotation_setting = (
|
|
db.session.query(AppAnnotationSetting)
|
|
.filter(
|
|
AppAnnotationSetting.app_id == app_id,
|
|
AppAnnotationSetting.id == annotation_setting_id,
|
|
)
|
|
.first()
|
|
)
|
|
if not annotation_setting:
|
|
raise NotFound("App annotation not found")
|
|
annotation_setting.score_threshold = args["score_threshold"]
|
|
annotation_setting.updated_user_id = current_user.id
|
|
annotation_setting.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
|
|
db.session.add(annotation_setting)
|
|
db.session.commit()
|
|
|
|
collection_binding_detail = annotation_setting.collection_binding_detail
|
|
|
|
return {
|
|
"id": annotation_setting.id,
|
|
"enabled": True,
|
|
"score_threshold": annotation_setting.score_threshold,
|
|
"embedding_model": {
|
|
"embedding_provider_name": collection_binding_detail.provider_name,
|
|
"embedding_model_name": collection_binding_detail.model_name,
|
|
},
|
|
}
|