Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #77 from tractorjuice/patch-1

Browse Source 
Add additional file extensions to crawler.ts
This commit is contained in:
Nicolas 2024-05-22 10:16:49 -07:00 committed by GitHub
commit 0d187f0425
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -324,6 +324,12 @@ export class WebCrawler {
// ".docx",
".xlsx",
".xml",
".avi",
".flv",
".woff",
".ttf",
".woff2",
".webp"
];
return fileExtensions.some((ext) => url.endsWith(ext));
}