Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Update queue-worker.ts

Browse Source
This commit is contained in:
Nicolas 2024-06-24 16:51:32 -03:00 committed by GitHub
parent 3ebdf93342
commit 08c1fa799b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/services/queue-worker.ts
View File

@ -14,7 +14,7 @@ if(process.env.ENV === 'production') {
getWebScraperQueue().process(
Math.floor(Number(process.env.NUM_WORKERS_PER_QUEUE ?? 8)),
async function (job, done) {
console.log('getWebScraperQueue - start')
try {
job.progress({
current: 1,
@ -23,13 +23,10 @@ getWebScraperQueue().process(
current_url: "",
});
const start = Date.now();
console.log('getWebScraperQueue - startWebScraperPipeline')
const { success, message, docs } = await startWebScraperPipeline({ job });
console.log('getWebScraperQueue - startWebScraperPipeline - done')
const end = Date.now();
const timeTakenInSeconds = (end - start) / 1000;
console.log('docs.length:', docs.length)
const data = {
success: success,
result: {