Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

added check job and cancel to fire-engine requests

Browse Source
This commit is contained in:
rafaelsideguide 2024-08-20 14:26:42 -03:00
parent e1c9cbf709
commit e326249a57
1 changed files with 23 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
View File

@ -86,7 +86,12 @@ export async function scrapWithFireEngine({
pageOptions.atsv = false;
}
const response = await axios.post(
const axiosInstance = axios.create({
headers: { "Content-Type": "application/json" }
});
const startTime = Date.now();
const response = await axiosInstance.post(
process.env.FIRE_ENGINE_BETA_URL + endpoint,
{
url: url,
@ -98,16 +103,31 @@ export async function scrapWithFireEngine({
disableJsDom: pageOptions?.disableJsDom ?? false,
priority,
engine,
instantReturn: true,
...fireEngineOptionsParam,
},
{
headers: {
"Content-Type": "application/json",
},
timeout: universalTimeout + waitParam,
}
}
);
let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${response.data.jobId}`);
while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitParam) {
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${response.data.jobId}`);
}
if (checkStatusResponse.data.processing) {
axiosInstance.delete(
process.env.FIRE_ENGINE_BETA_URL + `/scrape/${response.data.jobId}`,
);
Logger.debug(`⛏️ Fire-Engine (${engine}): Request timed out for ${url}`);
logParams.error_message = "Request timed out";
return { html: "", screenshot: "", pageStatusCode: null, pageError: "" };
}
if (response.status !== 200) {
Logger.debug(
`⛏️ Fire-Engine (${engine}): Failed to fetch url: ${url} \t status: ${response.status}`