Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #491 from mendableai/bugfix/issue-477

Browse Source 
[Bug] Nested sitemaps
This commit is contained in:
Nicolas 2024-08-02 13:03:28 -04:00 committed by GitHub
commit 893113a39e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -476,7 +476,7 @@ export class WebCrawler {
try {
const response = await axios.get(baseUrlSitemap, { timeout: axiosTimeout });
if (response.status === 200) {
sitemapLinks = await getLinksFromSitemap({ sitemapUrl: baseUrlSitemap });
sitemapLinks = await getLinksFromSitemap({ sitemapUrl: baseUrlSitemap, mode: 'fire-engine' });
}
} catch (error) {
Logger.debug(`Failed to fetch sitemap from ${baseUrlSitemap}: ${error}`);

2
apps/api/src/scraper/WebScraper/sitemap.ts
View File

@ -19,7 +19,7 @@ export async function getLinksFromSitemap(
try {
let content: string;
try {
if (mode === 'axios') {
if (mode === 'axios' || process.env.FIRE_ENGINE_BETA_URL === '') {
const response = await axios.get(sitemapUrl, { timeout: axiosTimeout });
content = response.data;
} else if (mode === 'fire-engine') {