Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'nsc/improved-blocklist'

Browse Source
This commit is contained in:
Nicolas 2024-06-03 16:44:33 -07:00
commit 93bb53271e
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts
View File

@ -40,6 +40,7 @@ describe('isUrlBlocked', () => {
'https://whatsapp.com/press',
'https://wechat.com/careers',
'https://telegram.org/conditions',
'https://wix.com/careers',
];
allowedUrls.forEach(url => {
@ -54,6 +55,8 @@ describe('isUrlBlocked', () => {
'https://subdomain.example.com',
'firecrawl.dev',
'amazon.com',
'wix.com',
'https://wix.com'
];
nonBlockedUrls.forEach(url => {