Nick:

apps/js-sdk/example.ts

@@ -17,4 +17,7 @@ while (true) {
   await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
 }
 
-console.log(job.data[0].content);
+console.log(job.data);
+
+
+const a  =await app.scrapeUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}});

apps/js-sdk/firecrawl/build/index.js

@@ -8,8 +8,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import axios from 'axios';
-import dotenv from 'dotenv';
-dotenv.config();
 /**
  * Main class for interacting with the Firecrawl API.
  */
@@ -19,7 +17,7 @@ export default class FirecrawlApp {
      * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
      */
     constructor({ apiKey = null }) {
-        this.apiKey = apiKey || process.env.FIRECRAWL_API_KEY || '';
+        this.apiKey = apiKey || '';
         if (!this.apiKey) {
             throw new Error('No API key provided');
         }

apps/js-sdk/firecrawl/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "0.0.16",
+  "version": "0.0.17-beta.1",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "build/index.js",
   "types": "types/index.d.ts",
@@ -8,6 +8,7 @@
   "scripts": {
     "build": "tsc",
     "publish": "npm run build && npm publish --access public",
+    "publish:beta": "npm run build && npm publish --access public --tag beta",
     "test": "jest src/**/*.test.ts"
   },
   "repository": {

apps/js-sdk/firecrawl/src/index.ts

@@ -19,7 +19,7 @@ export interface Params {
  */
 export interface ScrapeResponse {
   success: boolean;
-  data?: any;
+  data?: Document;
   error?: string;
 }
 
@@ -28,7 +28,7 @@ export interface ScrapeResponse {
  */
 export interface SearchResponse {
   success: boolean;
-  data?: any;
+  data?: Document[] | { url: string, title: string, description: string }[];
   error?: string;
 }
 /**
@@ -37,7 +37,7 @@ export interface SearchResponse {
 export interface CrawlResponse {
   success: boolean;
   jobId?: string;
-  data?: any;
+  data?: Document[];
   error?: string;
 }
 
@@ -52,6 +52,28 @@ export interface JobStatusResponse {
   error?: string;
 }
 
+/**
+ * Return type for scraping, crawling and search operations.
+ */
+export interface Document{
+  id?: string;
+  content: string;
+  markdown?: string;
+  html?: string;
+  llm_extraction?: Record<string, any>;
+  createdAt?: Date;
+  updatedAt?: Date;
+  type?: string;
+  metadata: {
+    sourceURL?: string;
+    [key: string]: any;
+  };
+  childrenLinks?: string[];
+  provider?: string;
+  url?: string; // Used only in /search for now
+}
+
+
 /**
  * Main class for interacting with the Firecrawl API.
  */

apps/js-sdk/firecrawl/types/index.d.ts

@@ -16,7 +16,7 @@ export interface Params {
  */
 export interface ScrapeResponse {
     success: boolean;
-    data?: any;
+    data?: Document;
     error?: string;
 }
 /**
@@ -24,7 +24,11 @@ export interface ScrapeResponse {
  */
 export interface SearchResponse {
     success: boolean;
-    data?: any;
+    data?: Document[] | {
+        url: string;
+        title: string;
+        description: string;
+    }[];
     error?: string;
 }
 /**
@@ -33,7 +37,7 @@ export interface SearchResponse {
 export interface CrawlResponse {
     success: boolean;
     jobId?: string;
-    data?: any;
+    data?: Document[];
     error?: string;
 }
 /**
@@ -46,6 +50,26 @@ export interface JobStatusResponse {
     data?: any;
     error?: string;
 }
+/**
+ * Return type for scraping, crawling and search operations.
+ */
+export interface Document {
+    id?: string;
+    content: string;
+    markdown?: string;
+    html?: string;
+    llm_extraction?: Record<string, any>;
+    createdAt?: Date;
+    updatedAt?: Date;
+    type?: string;
+    metadata: {
+        sourceURL?: string;
+        [key: string]: any;
+    };
+    childrenLinks?: string[];
+    provider?: string;
+    url?: string;
+}
 /**
  * Main class for interacting with the Firecrawl API.
  */

apps/js-sdk/package-lock.json

@@ -9,17 +9,16 @@
       "version": "1.0.0",
       "license": "ISC",
       "dependencies": {
-        "@mendable/firecrawl-js": "^0.0.15",
+        "@mendable/firecrawl-js": "^0.0.17-beta.1",
         "axios": "^1.6.8"
       }
     },
     "node_modules/@mendable/firecrawl-js": {
-      "version": "0.0.15",
-      "resolved": "https://registry.npmjs.org/@mendable/firecrawl-js/-/firecrawl-js-0.0.15.tgz",
-      "integrity": "sha512-e3iCCrLIiEh+jEDerGV9Uhdkn8ymo+sG+k3osCwPg51xW1xUdAnmlcHrcJoR43RvKXdvD/lqoxg8odUEsqyH+w==",
+      "version": "0.0.17-beta.1",
+      "resolved": "https://registry.npmjs.org/@mendable/firecrawl-js/-/firecrawl-js-0.0.17-beta.1.tgz",
+      "integrity": "sha512-GxBiYIJoFn4ddKTiN6WaFfCapRp3qNDNTWw90sV3kY+YGs5s40oIJ7bkDcI1Am59QhntiKzKB4IcjpV1LVfQTA==",
       "dependencies": {
-        "axios": "^1.6.8",
-        "dotenv": "^16.4.5"
+        "axios": "^1.6.8"
       }
     },
     "node_modules/asynckit": {
@@ -56,17 +55,6 @@
         "node": ">=0.4.0"
       }
     },
-    "node_modules/dotenv": {
-      "version": "16.4.5",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz",
-      "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==",
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://dotenvx.com"
-      }
-    },
     "node_modules/follow-redirects": {
       "version": "1.15.6",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",

apps/js-sdk/package.json

@@ -11,7 +11,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@mendable/firecrawl-js": "^0.0.15",
+    "@mendable/firecrawl-js": "^0.0.17-beta.1",
     "axios": "^1.6.8"
   }
 }