Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Update blocklist.ts

Browse Source
This commit is contained in:
Nicolas 2024-10-09 14:41:23 -03:00
parent 075b63b57b
commit 064ce482c2
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -40,6 +40,7 @@ const allowedKeywords = [
'tos', 'tos',
'://ads.tiktok.com', '://ads.tiktok.com',
'://tiktok.com/business', '://tiktok.com/business',
'://developers.facebook.com'
]; ];
export function isUrlBlocked(url: string): boolean { export function isUrlBlocked(url: string): boolean {