Add SearchApi to search

Add support for engines, improve status code error

Remove changes in package, add engine to env params

Improve description in env example

Remove unnecessary empty line

Improve text

apps/api/.env.example

@@ -1,5 +1,5 @@
 # ===== Required ENVS ======
-NUM_WORKERS_PER_QUEUE=8 
+NUM_WORKERS_PER_QUEUE=8
 PORT=3002
 HOST=0.0.0.0
 REDIS_URL=redis://redis:6379 #for self-hosting using docker, use redis://redis:6379. For running locally, use redis://localhost:6379
@@ -11,9 +11,14 @@ USE_DB_AUTHENTICATION=true
 
 # ===== Optional ENVS ======
 
+# SearchApi key. Head to https://searchapi.com/ to get your API key
+SEARCHAPI_API_KEY=
+# SearchApi engine, defaults to google. Available options: google, bing, baidu, google_news, etc. Head to https://searchapi.com/ to explore more engines
+SEARCHAPI_ENGINE=
+
 # Supabase Setup (used to support DB authentication, advanced logging, etc.)
-SUPABASE_ANON_TOKEN= 
-SUPABASE_URL= 
+SUPABASE_ANON_TOKEN=
+SUPABASE_URL=
 SUPABASE_SERVICE_TOKEN=
 
 # Other Optionals

apps/api/.env.local

@@ -12,4 +12,4 @@ ANTHROPIC_API_KEY=
 BULL_AUTH_KEY=
 LOGTAIL_KEY=
 PLAYWRIGHT_MICROSERVICE_URL=
-
+SEARCHAPI_API_KEY=

apps/api/src/search/index.ts

@@ -2,6 +2,7 @@ import { Logger } from "../../src/lib/logger";
 import { SearchResult } from "../../src/lib/entities";
 import { googleSearch } from "./googlesearch";
 import { fireEngineMap } from "./fireEngine";
+import { searchapi_search } from "./searchapi";
 import { serper_search } from "./serper";
 
 export async function search({
@@ -30,7 +31,16 @@ export async function search({
   timeout?: number;
 }): Promise<SearchResult[]> {
   try {
-    
+    if (process.env.SEARCHAPI_API_KEY) {
+      return await searchapi_search(query, {
+        num_results,
+        tbs,
+        filter,
+        lang,
+        country,
+        location
+      });
+    }
     if (process.env.SERPER_API_KEY) {
       return await serper_search(query, {
         num_results,

apps/api/src/search/searchapi.ts

@@ -0,0 +1,60 @@
+import axios from "axios";
+import dotenv from "dotenv";
+import { SearchResult } from "../../src/lib/entities";
+
+dotenv.config();
+
+interface SearchOptions {
+  tbs?: string;
+  filter?: string;
+  lang?: string;
+  country?: string;
+  location?: string;
+  num_results: number;
+  page?: number;
+}
+
+export async function searchapi_search(q: string, options: SearchOptions): Promise<SearchResult[]> {
+  const params = {
+    q: q,
+    hl: options.lang,
+    gl: options.country,
+    location: options.location,
+    num: options.num_results,
+    page: options.page ?? 1,
+    engine: process.env.SEARCHAPI_ENGINE || "google",
+  };
+
+  const url = `https://www.searchapi.io/api/v1/search`;
+
+  try {
+    const response = await axios.get(url, {
+      headers: {
+        "Authorization": `Bearer ${process.env.SEARCHAPI_API_KEY}`,
+        "Content-Type": "application/json",
+        "X-SearchApi-Source": "Firecrawl",
+      },
+      params: params,
+    });
+
+
+    if (response.status === 401) {
+      throw new Error("Unauthorized. Please check your API key.");
+    }
+
+    const data = response.data;
+
+    if (data && Array.isArray(data.organic_results)) {
+      return data.organic_results.map((a: any) => ({
+        url: a.link,
+        title: a.title,
+        description: a.snippet,
+      }));
+    } else {
+      return [];
+    }
+  } catch (error) {
+    console.error(`There was an error searching for content: ${error.message}`);
+    return [];
+  }
+}