Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'v1-webscraper' of https://github.com/mendableai/firecrawl into v1-webscraper

Browse Source
This commit is contained in:
Nicolas 2024-08-26 18:17:17 -03:00
commit 8c37ea6d96
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/controllers/v1/types.ts
View File

@ -27,7 +27,10 @@ const url = z.preprocess(
.string()
.url()
.regex(/^https?:\/\//, "URL uses unsupported protocol")
.regex(/\.[a-z]{2,}$/i, "URL must have a valid top-level domain")
.refine(
(x) => /\.[a-z]{2,}(\/|$)/i.test(x),
"URL must have a valid top-level domain or be a valid path"
)
.refine(
(x) => !isUrlBlocked(x),
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it."