Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Nick: llm extract in batch scrape

Browse Source
This commit is contained in:
Nicolas 2024-10-28 16:51:49 -03:00
parent 298a343a90
commit 726430c2e6
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/controllers/v1/batch-scrape.ts
View File

@ -4,6 +4,7 @@ import {
BatchScrapeRequest, BatchScrapeRequest,
batchScrapeRequestSchema, batchScrapeRequestSchema,
CrawlResponse, CrawlResponse,
legacyExtractorOptions,
legacyScrapeOptions, legacyScrapeOptions,
RequestWithAuth, RequestWithAuth,
} from "./types"; } from "./types";
@ -34,6 +35,8 @@ export async function batchScrapeController(
} }
const pageOptions = legacyScrapeOptions(req.body); const pageOptions = legacyScrapeOptions(req.body);
const extractorOptions = req.body.extract ? legacyExtractorOptions(req.body.extract) : undefined;
const sc: StoredCrawl = { const sc: StoredCrawl = {
crawlerOptions: null, crawlerOptions: null,
@ -65,6 +68,7 @@ export async function batchScrapeController(
plan: req.auth.plan, plan: req.auth.plan,
crawlerOptions: null, crawlerOptions: null,
pageOptions, pageOptions,
extractorOptions,
origin: "api", origin: "api",
crawl_id: id, crawl_id: id,
sitemapped: true, sitemapped: true,