mirror of
https://github.com/langgenius/dify.git
synced 2024-11-16 11:42:29 +08:00
Merge branch 'main' into deploy/dev
# Conflicts: # web/app/(commonLayout)/datasets/template/template.en.mdx # web/app/(commonLayout)/datasets/template/template.zh.mdx
This commit is contained in:
jyong
commit
70073a3e65
|
|
@ -55,7 +55,12 @@ RUN apt-get update \
|
|||
&& echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
|
||||
&& apt-get update \
|
||||
# For Security
|
||||
&& apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.3-1 libldap-2.5-0=2.5.18+dfsg-3+b1 perl=5.40.0-6 libsqlite3-0=3.46.1-1 \
|
||||
&& apt-get install -y --no-install-recommends expat=2.6.3-2 libldap-2.5-0=2.5.18+dfsg-3+b1 perl=5.40.0-6 libsqlite3-0=3.46.1-1 \
|
||||
&& if [ "$(dpkg --print-architecture)" = "amd64" ]; then \
|
||||
apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1+b1; \
|
||||
else \
|
||||
apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1; \
|
||||
fi \
|
||||
# install a chinese font to support the use of tools like matplotlib
|
||||
&& apt-get install -y fonts-noto-cjk \
|
||||
&& apt-get autoremove -y \
|
||||
|
|
|
|||
|
|
@ -27,18 +27,17 @@ class RerankModelRunner(BaseRerankRunner):
|
|||
:return:
|
||||
"""
|
||||
docs = []
|
||||
doc_id = []
|
||||
doc_id = set()
|
||||
unique_documents = []
|
||||
dify_documents = [item for item in documents if item.provider == "dify"]
|
||||
external_documents = [item for item in documents if item.provider == "external"]
|
||||
for document in dify_documents:
|
||||
if document.metadata["doc_id"] not in doc_id:
|
||||
doc_id.append(document.metadata["doc_id"])
|
||||
for document in documents:
|
||||
if document.provider == "dify" and document.metadata["doc_id"] not in doc_id:
|
||||
doc_id.add(document.metadata["doc_id"])
|
||||
docs.append(document.page_content)
|
||||
unique_documents.append(document)
|
||||
for document in external_documents:
|
||||
docs.append(document.page_content)
|
||||
unique_documents.append(document)
|
||||
elif document.provider == "external":
|
||||
if document not in unique_documents:
|
||||
docs.append(document.page_content)
|
||||
unique_documents.append(document)
|
||||
|
||||
documents = unique_documents
|
||||
|
||||
|
|
|
|||
|
|
@ -736,11 +736,12 @@ class DocumentService:
|
|||
dataset.retrieval_model = document_data.get("retrieval_model") or default_retrieval_model
|
||||
|
||||
documents = []
|
||||
batch = time.strftime("%Y%m%d%H%M%S") + str(random.randint(100000, 999999))
|
||||
if document_data.get("original_document_id"):
|
||||
document = DocumentService.update_document_with_dataset_id(dataset, document_data, account)
|
||||
documents.append(document)
|
||||
batch = document.batch
|
||||
else:
|
||||
batch = time.strftime("%Y%m%d%H%M%S") + str(random.randint(100000, 999999))
|
||||
# save process rule
|
||||
if not dataset_process_rule:
|
||||
process_rule = document_data["process_rule"]
|
||||
|
|
@ -921,7 +922,7 @@ class DocumentService:
|
|||
if duplicate_document_ids:
|
||||
duplicate_document_indexing_task.delay(dataset.id, duplicate_document_ids)
|
||||
|
||||
return documents, batch
|
||||
return documents, batch
|
||||
|
||||
@staticmethod
|
||||
def check_documents_upload_quota(count: int, features: FeatureModel):
|
||||
|
|
|
|||
|
|
@ -195,8 +195,8 @@ function DetailPanel({ detail, onFeedback }: IDetailPanel) {
|
|||
conversation_id: detail.id,
|
||||
limit: 10,
|
||||
}
|
||||
if (allChatItems.at(-1)?.id)
|
||||
params.first_id = allChatItems.at(-1)?.id.replace('question-', '')
|
||||
if (allChatItems[0]?.id)
|
||||
params.first_id = allChatItems[0]?.id.replace('question-', '')
|
||||
const messageRes = await fetchChatMessages({
|
||||
url: `/apps/${appDetail?.id}/chat-messages`,
|
||||
params,
|
||||
|
|
|
|||
|
|
@ -1804,6 +1804,280 @@ exports[`build chat item tree and get thread messages should get thread messages
|
|||
]
|
||||
`;
|
||||
|
||||
exports[`build chat item tree and get thread messages should work with partial messages 1`] = `
|
||||
[
|
||||
{
|
||||
"children": [
|
||||
{
|
||||
"agent_thoughts": [
|
||||
{
|
||||
"chain_id": null,
|
||||
"created_at": 1726105809,
|
||||
"files": [],
|
||||
"id": "1019cd79-d141-4f9f-880a-fc1441cfd802",
|
||||
"message_id": "cd5affb0-7bc2-4a6f-be7e-25e74595c9dd",
|
||||
"observation": "",
|
||||
"position": 1,
|
||||
"thought": "Sure! My number is 54. Your turn!",
|
||||
"tool": "",
|
||||
"tool_input": "",
|
||||
"tool_labels": {},
|
||||
},
|
||||
],
|
||||
"children": [
|
||||
{
|
||||
"children": [
|
||||
{
|
||||
"agent_thoughts": [
|
||||
{
|
||||
"chain_id": null,
|
||||
"created_at": 1726105822,
|
||||
"files": [],
|
||||
"id": "0773bec7-b992-4a53-92b2-20ebaeae8798",
|
||||
"message_id": "324bce32-c98c-435d-a66b-bac974ebb5ed",
|
||||
"observation": "",
|
||||
"position": 1,
|
||||
"thought": "My number is 4729. Your turn!",
|
||||
"tool": "",
|
||||
"tool_input": "",
|
||||
"tool_labels": {},
|
||||
},
|
||||
],
|
||||
"children": [],
|
||||
"content": "My number is 4729. Your turn!",
|
||||
"conversationId": "dd6c9cfd-2656-48ec-bd51-2139c1790d80",
|
||||
"feedbackDisabled": false,
|
||||
"id": "324bce32-c98c-435d-a66b-bac974ebb5ed",
|
||||
"input": {
|
||||
"inputs": {},
|
||||
"query": "3306",
|
||||
},
|
||||
"isAnswer": true,
|
||||
"log": [
|
||||
{
|
||||
"files": [],
|
||||
"role": "user",
|
||||
"text": "Let's play a game, I say a number , and you response me with another bigger, yet randomly number. I'll start first, 38",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "assistant",
|
||||
"text": "Sure! My number is 54. Your turn!",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "user",
|
||||
"text": "3306",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "assistant",
|
||||
"text": "My number is 4729. Your turn!",
|
||||
},
|
||||
],
|
||||
"message_files": [],
|
||||
"more": {
|
||||
"latency": "1.30",
|
||||
"time": "09/11/2024 09:50 PM",
|
||||
"tokens": 66,
|
||||
},
|
||||
"parentMessageId": "question-324bce32-c98c-435d-a66b-bac974ebb5ed",
|
||||
"siblingIndex": 0,
|
||||
"workflow_run_id": null,
|
||||
},
|
||||
],
|
||||
"content": "3306",
|
||||
"id": "question-324bce32-c98c-435d-a66b-bac974ebb5ed",
|
||||
"isAnswer": false,
|
||||
"message_files": [],
|
||||
"parentMessageId": "cd5affb0-7bc2-4a6f-be7e-25e74595c9dd",
|
||||
},
|
||||
{
|
||||
"children": [
|
||||
{
|
||||
"agent_thoughts": [
|
||||
{
|
||||
"chain_id": null,
|
||||
"created_at": 1726107812,
|
||||
"files": [],
|
||||
"id": "5ca650f3-982c-4399-8b95-9ea241c76707",
|
||||
"message_id": "684b5396-4e91-4043-88e9-aabe48b21acc",
|
||||
"observation": "",
|
||||
"position": 1,
|
||||
"thought": "My number is 4821. Your turn!",
|
||||
"tool": "",
|
||||
"tool_input": "",
|
||||
"tool_labels": {},
|
||||
},
|
||||
],
|
||||
"children": [
|
||||
{
|
||||
"children": [
|
||||
{
|
||||
"agent_thoughts": [
|
||||
{
|
||||
"chain_id": null,
|
||||
"created_at": 1726111024,
|
||||
"files": [],
|
||||
"id": "095cacab-afad-4387-a41d-1662578b8b13",
|
||||
"message_id": "19904a7b-7494-4ed8-b72c-1d18668cea8c",
|
||||
"observation": "",
|
||||
"position": 1,
|
||||
"thought": "My number is 1456. Your turn!",
|
||||
"tool": "",
|
||||
"tool_input": "",
|
||||
"tool_labels": {},
|
||||
},
|
||||
],
|
||||
"children": [],
|
||||
"content": "My number is 1456. Your turn!",
|
||||
"conversationId": "dd6c9cfd-2656-48ec-bd51-2139c1790d80",
|
||||
"feedbackDisabled": false,
|
||||
"id": "19904a7b-7494-4ed8-b72c-1d18668cea8c",
|
||||
"input": {
|
||||
"inputs": {},
|
||||
"query": "1003",
|
||||
},
|
||||
"isAnswer": true,
|
||||
"log": [
|
||||
{
|
||||
"files": [],
|
||||
"role": "user",
|
||||
"text": "Let's play a game, I say a number , and you response me with another bigger, yet randomly number. I'll start first, 38",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "assistant",
|
||||
"text": "Sure! My number is 54. Your turn!",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "user",
|
||||
"text": "3306",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "assistant",
|
||||
"text": "My number is 4821. Your turn!",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "user",
|
||||
"text": "1003",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "assistant",
|
||||
"text": "My number is 1456. Your turn!",
|
||||
},
|
||||
],
|
||||
"message_files": [],
|
||||
"more": {
|
||||
"latency": "1.38",
|
||||
"time": "09/11/2024 11:17 PM",
|
||||
"tokens": 86,
|
||||
},
|
||||
"parentMessageId": "question-19904a7b-7494-4ed8-b72c-1d18668cea8c",
|
||||
"siblingIndex": 0,
|
||||
"workflow_run_id": null,
|
||||
},
|
||||
],
|
||||
"content": "1003",
|
||||
"id": "question-19904a7b-7494-4ed8-b72c-1d18668cea8c",
|
||||
"isAnswer": false,
|
||||
"message_files": [],
|
||||
"parentMessageId": "684b5396-4e91-4043-88e9-aabe48b21acc",
|
||||
},
|
||||
],
|
||||
"content": "My number is 4821. Your turn!",
|
||||
"conversationId": "dd6c9cfd-2656-48ec-bd51-2139c1790d80",
|
||||
"feedbackDisabled": false,
|
||||
"id": "684b5396-4e91-4043-88e9-aabe48b21acc",
|
||||
"input": {
|
||||
"inputs": {},
|
||||
"query": "3306",
|
||||
},
|
||||
"isAnswer": true,
|
||||
"log": [
|
||||
{
|
||||
"files": [],
|
||||
"role": "user",
|
||||
"text": "Let's play a game, I say a number , and you response me with another bigger, yet randomly number. I'll start first, 38",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "assistant",
|
||||
"text": "Sure! My number is 54. Your turn!",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "user",
|
||||
"text": "3306",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "assistant",
|
||||
"text": "My number is 4821. Your turn!",
|
||||
},
|
||||
],
|
||||
"message_files": [],
|
||||
"more": {
|
||||
"latency": "1.48",
|
||||
"time": "09/11/2024 10:23 PM",
|
||||
"tokens": 66,
|
||||
},
|
||||
"parentMessageId": "question-684b5396-4e91-4043-88e9-aabe48b21acc",
|
||||
"siblingIndex": 1,
|
||||
"workflow_run_id": null,
|
||||
},
|
||||
],
|
||||
"content": "3306",
|
||||
"id": "question-684b5396-4e91-4043-88e9-aabe48b21acc",
|
||||
"isAnswer": false,
|
||||
"message_files": [],
|
||||
"parentMessageId": "cd5affb0-7bc2-4a6f-be7e-25e74595c9dd",
|
||||
},
|
||||
],
|
||||
"content": "Sure! My number is 54. Your turn!",
|
||||
"conversationId": "dd6c9cfd-2656-48ec-bd51-2139c1790d80",
|
||||
"feedbackDisabled": false,
|
||||
"id": "cd5affb0-7bc2-4a6f-be7e-25e74595c9dd",
|
||||
"input": {
|
||||
"inputs": {},
|
||||
"query": "Let's play a game, I say a number , and you response me with another bigger, yet randomly number. I'll start first, 38",
|
||||
},
|
||||
"isAnswer": true,
|
||||
"log": [
|
||||
{
|
||||
"files": [],
|
||||
"role": "user",
|
||||
"text": "Let's play a game, I say a number , and you response me with another bigger, yet randomly number. I'll start first, 38",
|
||||
},
|
||||
{
|
||||
"files": [],
|
||||
"role": "assistant",
|
||||
"text": "Sure! My number is 54. Your turn!",
|
||||
},
|
||||
],
|
||||
"message_files": [],
|
||||
"more": {
|
||||
"latency": "1.52",
|
||||
"time": "09/11/2024 09:50 PM",
|
||||
"tokens": 46,
|
||||
},
|
||||
"parentMessageId": "question-cd5affb0-7bc2-4a6f-be7e-25e74595c9dd",
|
||||
"siblingIndex": 0,
|
||||
"workflow_run_id": null,
|
||||
},
|
||||
],
|
||||
"content": "Let's play a game, I say a number , and you response me with another bigger, yet randomly number. I'll start first, 38",
|
||||
"id": "question-cd5affb0-7bc2-4a6f-be7e-25e74595c9dd",
|
||||
"isAnswer": false,
|
||||
"message_files": [],
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`build chat item tree and get thread messages should work with real world messages 1`] = `
|
||||
[
|
||||
{
|
||||
|
|
|
|||
|
|
@ -255,4 +255,10 @@ describe('build chat item tree and get thread messages', () => {
|
|||
const threadMessages6_2 = getThreadMessages(tree6, 'ff4c2b43-48a5-47ad-9dc5-08b34ddba61b')
|
||||
expect(threadMessages6_2).toMatchSnapshot()
|
||||
})
|
||||
|
||||
const partialMessages = (realWorldMessages as ChatItemInTree[]).slice(-10)
|
||||
const tree7 = buildChatItemTree(partialMessages)
|
||||
it('should work with partial messages', () => {
|
||||
expect(tree7).toMatchSnapshot()
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -134,6 +134,12 @@ function buildChatItemTree(allMessages: IChatItem[]): ChatItemInTree[] {
|
|||
}
|
||||
}
|
||||
|
||||
// If no messages have parentMessageId=null (indicating a root node),
|
||||
// then we likely have a partial chat history. In this case,
|
||||
// use the first available message as the root node.
|
||||
if (rootNodes.length === 0 && allMessages.length > 0)
|
||||
rootNodes.push(map[allMessages[0]!.id]!)
|
||||
|
||||
return rootNodes
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user