diff --git a/apps/js-sdk/example.ts b/apps/js-sdk/example.ts index 9fa823ae..f314c080 100644 --- a/apps/js-sdk/example.ts +++ b/apps/js-sdk/example.ts @@ -1,11 +1,14 @@ -import FirecrawlApp, { JobStatusResponse } from '@mendable/firecrawl-js'; +import FirecrawlApp, { JobStatusResponse } from './firecrawl/src/index' //'@mendable/firecrawl-js'; import { z } from "zod"; const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"}); // Scrape a website: const scrapeResult = await app.scrapeUrl('firecrawl.dev'); -console.log(scrapeResult.data.content) + +if (scrapeResult.data) { + console.log(scrapeResult.data.content) +} // Crawl a website: const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false); @@ -23,12 +26,13 @@ while (true) { await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second } -console.log(job.data[0].content); +if (job.data) { + console.log(job.data[0].content); +} // Search for a query: const query = 'what is mendable?' const searchResult = await app.search(query) -console.log(searchResult) // LLM Extraction: // Define schema to extract contents into using zod schema @@ -50,7 +54,9 @@ let llmExtractionResult = await app.scrapeUrl("https://news.ycombinator.com", { extractorOptions: { extractionSchema: zodSchema }, }); -console.log(llmExtractionResult.data.llm_extraction); +if (llmExtractionResult.data) { + console.log(llmExtractionResult.data.llm_extraction); +} // Define schema to extract contents into using json schema const jsonSchema = { @@ -80,4 +86,7 @@ llmExtractionResult = await app.scrapeUrl("https://news.ycombinator.com", { extractorOptions: { extractionSchema: jsonSchema }, }); -console.log(llmExtractionResult.data.llm_extraction); \ No newline at end of file +if (llmExtractionResult.data) { + console.log(llmExtractionResult.data.llm_extraction); +} + diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 4ab793c9..15d8034e 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "0.0.26", + "version": "0.0.27", "description": "JavaScript SDK for Firecrawl API", "main": "build/index.js", "types": "types/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index f8841257..5028a206 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -10,56 +10,112 @@ export interface FirecrawlAppConfig { } /** - * Generic parameter interface. + * Metadata for a Firecrawl document. */ -export interface Params { +export interface FirecrawlDocumentMetadata { + title?: string; + description?: string; + language?: string; + keywords?: string; + robots?: string; + ogTitle?: string; + ogDescription?: string; + ogUrl?: string; + ogImage?: string; + ogAudio?: string; + ogDeterminer?: string; + ogLocale?: string; + ogLocaleAlternate?: string[]; + ogSiteName?: string; + ogVideo?: string; + dctermsCreated?: string; + dcDateCreated?: string; + dcDate?: string; + dctermsType?: string; + dcType?: string; + dctermsAudience?: string; + dctermsSubject?: string; + dcSubject?: string; + dcDescription?: string; + dctermsKeywords?: string; + modifiedTime?: string; + publishedTime?: string; + articleTag?: string; + articleSection?: string; + sourceURL?: string; + pageStatusCode?: number; + pageError?: string; [key: string]: any; - extractorOptions?: { - extractionSchema: z.ZodSchema | any; - mode?: "llm-extraction"; - extractionPrompt?: string; - }; +} + +/** + * Document interface for Firecrawl. + */ +export interface FirecrawlDocument { + id?: string; + url?: string; + content: string; + markdown?: string; + html?: string; + llm_extraction?: Record; + createdAt?: Date; + updatedAt?: Date; + type?: string; + metadata: FirecrawlDocumentMetadata; + childrenLinks?: string[]; + provider?: string; + warning?: string; + + index?: number; } /** * Response interface for scraping operations. */ export interface ScrapeResponse { - success: boolean; - data?: any; - error?: string; + success: boolean; + data?: FirecrawlDocument; + error?: string; } - /** - * Response interface for searching operations. - */ +* Response interface for searching operations. +*/ export interface SearchResponse { - success: boolean; - data?: any; - error?: string; + success: boolean; + data?: FirecrawlDocument[]; + error?: string; } /** - * Response interface for crawling operations. - */ +* Response interface for crawling operations. +*/ export interface CrawlResponse { - success: boolean; - jobId?: string; - data?: any; - error?: string; + success: boolean; + jobId?: string; + data?: FirecrawlDocument[]; + error?: string; } - /** - * Response interface for job status checks. - */ +* Response interface for job status checks. +*/ export interface JobStatusResponse { - success: boolean; - status: string; - jobId?: string; - data?: any; - partial_data?: any, - error?: string; + success: boolean; + status: string; + jobId?: string; + data?: FirecrawlDocument[]; + partial_data?: FirecrawlDocument[]; + error?: string; +} +/** + * Generic parameter interface. + */ +export interface Params { + [key: string]: any; + extractorOptions?: { + extractionSchema: z.ZodSchema | any; + mode?: "llm-extraction"; + extractionPrompt?: string; + }; } - /** * Main class for interacting with the Firecrawl API. */ diff --git a/apps/js-sdk/firecrawl/types/index.d.ts b/apps/js-sdk/firecrawl/types/index.d.ts index 52a7d1ee..1a7cd1f2 100644 --- a/apps/js-sdk/firecrawl/types/index.d.ts +++ b/apps/js-sdk/firecrawl/types/index.d.ts @@ -8,8 +8,101 @@ export interface FirecrawlAppConfig { apiUrl?: string | null; } /** - * Generic parameter interface. + * Metadata for a Firecrawl document. */ +export interface FirecrawlDocumentMetadata { + title?: string; + description?: string; + language?: string; + keywords?: string; + robots?: string; + ogTitle?: string; + ogDescription?: string; + ogUrl?: string; + ogImage?: string; + ogAudio?: string; + ogDeterminer?: string; + ogLocale?: string; + ogLocaleAlternate?: string[]; + ogSiteName?: string; + ogVideo?: string; + dctermsCreated?: string; + dcDateCreated?: string; + dcDate?: string; + dctermsType?: string; + dcType?: string; + dctermsAudience?: string; + dctermsSubject?: string; + dcSubject?: string; + dcDescription?: string; + dctermsKeywords?: string; + modifiedTime?: string; + publishedTime?: string; + articleTag?: string; + articleSection?: string; + sourceURL?: string; + pageStatusCode?: number; + pageError?: string; + [key: string]: any; +} +/** + * Document interface for Firecrawl. + */ +export interface FirecrawlDocument { + id?: string; + url?: string; + content: string; + markdown?: string; + html?: string; + llm_extraction?: Record; + createdAt?: Date; + updatedAt?: Date; + type?: string; + metadata: FirecrawlDocumentMetadata; + childrenLinks?: string[]; + provider?: string; + warning?: string; + index?: number; +} +/** + * Response interface for scraping operations. + */ +export interface ScrapeResponse { + success: boolean; + data?: FirecrawlDocument; + error?: string; +} +/** +* Response interface for searching operations. +*/ +export interface SearchResponse { + success: boolean; + data?: FirecrawlDocument[]; + error?: string; +} +/** +* Response interface for crawling operations. +*/ +export interface CrawlResponse { + success: boolean; + jobId?: string; + data?: FirecrawlDocument[]; + error?: string; +} +/** +* Response interface for job status checks. +*/ +export interface JobStatusResponse { + success: boolean; + status: string; + jobId?: string; + data?: FirecrawlDocument[]; + partial_data?: FirecrawlDocument[]; + error?: string; +} +/** + * Generic parameter interface. + */ export interface Params { [key: string]: any; extractorOptions?: { @@ -18,42 +111,6 @@ export interface Params { extractionPrompt?: string; }; } -/** - * Response interface for scraping operations. - */ -export interface ScrapeResponse { - success: boolean; - data?: any; - error?: string; -} -/** - * Response interface for searching operations. - */ -export interface SearchResponse { - success: boolean; - data?: any; - error?: string; -} -/** - * Response interface for crawling operations. - */ -export interface CrawlResponse { - success: boolean; - jobId?: string; - data?: any; - error?: string; -} -/** - * Response interface for job status checks. - */ -export interface JobStatusResponse { - success: boolean; - status: string; - jobId?: string; - data?: any; - partial_data?: any; - error?: string; -} /** * Main class for interacting with the Firecrawl API. */