Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Update single_url.ts

Browse Source
This commit is contained in:
Nicolas 2024-06-28 15:45:16 -03:00
parent dbfae2d9bf
commit 7e17498bcf
1 changed files with 9 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -46,12 +46,8 @@ export async function generateRequestParams(
}
}
export async function scrapWithFireEngine(
url: string,
waitFor: number = 0,
screenshot: boolean = false,
pageOptions: { scrollXPaths?: string[], parsePDF?: boolean } = { parsePDF: true },
headers?: Record<string, string>,
options?: any
{ url, waitFor = 0, screenshot = false, pageOptions = { parsePDF: true }, headers, options }:
{ url: string, waitFor?: number, screenshot?: boolean, pageOptions?: { scrollXPaths?: string[], parsePDF?: boolean }, headers?: Record<string, string>, options?: any }
): Promise<FireEngineResponse> {
try {
const reqParams = await generateRequestParams(url);
@ -321,11 +317,13 @@ export async function scrapSingleUrl(
case "fire-engine":
if (process.env.FIRE_ENGINE_BETA_URL) {
console.log(`Scraping ${url} with Fire Engine`);
const response = await scrapWithFireEngine(
const response = await scrapWithFireEngine({
url,
pageOptions.waitFor,
pageOptions.screenshot,
pageOptions.headers
waitFor: pageOptions.waitFor,
screenshot: pageOptions.screenshot,
pageOptions: pageOptions,
headers: pageOptions.headers
}
);
scraperResponse.text = response.html;
scraperResponse.screenshot = response.screenshot;
@ -377,7 +375,7 @@ export async function scrapSingleUrl(
if (customScraperResult){
switch (customScraperResult.scraper) {
case "fire-engine":
customScrapedContent = await scrapWithFireEngine(customScraperResult.url, customScraperResult.waitAfterLoad, false, customScraperResult.pageOptions)
customScrapedContent = await scrapWithFireEngine({url: customScraperResult.url, waitFor: customScraperResult.waitAfterLoad, screenshot: false, pageOptions: customScraperResult.pageOptions})
if (screenshot) {
customScrapedContent.screenshot = screenshot;
}