Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #449 from mendableai/bugfix/malformed-url-sitemap

Browse Source 
Added regex for links in sitemap
This commit is contained in:
Nicolas 2024-07-24 20:37:35 -04:00 committed by GitHub
commit 2c1221750b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -64,7 +64,7 @@ export class WebCrawler {
private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] {
return sitemapLinks
.filter((link) => {
const url = new URL(link);
const url = new URL(link.trim(), this.baseUrl);
const path = url.pathname;
const depth = getURLDepth(url.toString());