Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

convert webhook call to v1

Browse Source
This commit is contained in:
Gergo Moricz 2024-08-25 14:05:46 +02:00
parent 064ebfc54d
commit 96e91ab9ec
4 changed files with 18 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/controllers/v1/crawl.ts
View File

@ -81,6 +81,7 @@ export async function crawlController(
origin: "api", origin: "api",
crawl_id: id, crawl_id: id,
sitemapped: true, sitemapped: true,
v1: true,
}, },
opts: { opts: {
jobId: uuid, jobId: uuid,
@ -110,6 +111,7 @@ export async function crawlController(
origin: "api", origin: "api",
crawl_id: id, crawl_id: id,
webhook: req.body.webhook, webhook: req.body.webhook,
v1: true,
}, },
{ {
priority: 15, priority: 15,

7
apps/api/src/services/queue-worker.ts
View File

@ -215,7 +215,7 @@ async function processJob(job: Job, token: string) {
}; };
if (job.data.mode === "crawl") { if (job.data.mode === "crawl") {
await callWebhook(job.data.team_id, job.id as string, data, job.data.webhook); await callWebhook(job.data.team_id, job.id as string, data, job.data.webhook, job.data.v1);
} }
if (job.data.crawl_id) { if (job.data.crawl_id) {
@ -259,6 +259,7 @@ async function processJob(job: Job, token: string) {
pageOptions: sc.pageOptions, pageOptions: sc.pageOptions,
origin: job.data.origin, origin: job.data.origin,
crawl_id: job.data.crawl_id, crawl_id: job.data.crawl_id,
v1: job.data.v1,
}); });
await addCrawlJob(job.data.crawl_id, newJob.id); await addCrawlJob(job.data.crawl_id, newJob.id);
@ -328,7 +329,7 @@ async function processJob(job: Job, token: string) {
docs: fullDocs, docs: fullDocs,
}; };
await callWebhook(job.data.team_id, job.data.crawl_id, data); await callWebhook(job.data.team_id, job.data.crawl_id, data, job.data.webhook, job.data.v1);
} }
} }
@ -372,7 +373,7 @@ async function processJob(job: Job, token: string) {
}; };
if (job.data.mode === "crawl" || job.data.crawl_id) { if (job.data.mode === "crawl" || job.data.crawl_id) {
await callWebhook(job.data.team_id, job.data.crawl_id ?? job.id as string, data); await callWebhook(job.data.team_id, job.data.crawl_id ?? job.id as string, data, job.data.webhook, job.data.v1);
} }
if (job.data.crawl_id) { if (job.data.crawl_id) {

17
apps/api/src/services/webhook.ts
View File

@ -1,7 +1,8 @@
import { legacyDocumentConverter } from "../../src/controllers/v1/types";
import { Logger } from "../../src/lib/logger"; import { Logger } from "../../src/lib/logger";
import { supabase_service } from "./supabase"; import { supabase_service } from "./supabase";
export const callWebhook = async (teamId: string, jobId: string, data: any, specified?: string) => { export const callWebhook = async (teamId: string, jobId: string, data: any, specified?: string, v1 = false) => {
try { try {
const selfHostedUrl = process.env.SELF_HOSTED_WEBHOOK_URL?.replace("{{JOB_ID}}", jobId); const selfHostedUrl = process.env.SELF_HOSTED_WEBHOOK_URL?.replace("{{JOB_ID}}", jobId);
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true'; const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
@ -30,11 +31,15 @@ export const callWebhook = async (teamId: string, jobId: string, data: any, spec
let dataToSend = []; let dataToSend = [];
if (data.result.links && data.result.links.length !== 0) { if (data.result.links && data.result.links.length !== 0) {
for (let i = 0; i < data.result.links.length; i++) { for (let i = 0; i < data.result.links.length; i++) {
dataToSend.push({ if (v1) {
content: data.result.links[i].content.content, dataToSend.push(legacyDocumentConverter(data.result.links[i].content))
markdown: data.result.links[i].content.markdown, } else {
metadata: data.result.links[i].content.metadata, dataToSend.push({
}); content: data.result.links[i].content.content,
markdown: data.result.links[i].content.markdown,
metadata: data.result.links[i].content.metadata,
});
}
} }
} }

1
apps/api/src/types.ts
View File

@ -31,6 +31,7 @@ export interface WebScraperOptions {
crawl_id?: string; crawl_id?: string;
sitemapped?: boolean; sitemapped?: boolean;
webhook?: string; webhook?: string;
v1?: boolean;
} }
export interface RunWebScraperParams { export interface RunWebScraperParams {