Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix(scrapeURL): includeTags/excludeTags

Browse Source
This commit is contained in:
Gergő Móricz 2024-11-07 21:10:27 +01:00
parent 8d467c8ca7
commit 552d55c8fc
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
View File

@ -56,7 +56,7 @@ export const removeUnwantedElements = (
) => {
const soup = load(html);
if (scrapeOptions.includeTags && scrapeOptions.includeTags.length > 0) {
if (scrapeOptions.includeTags && scrapeOptions.includeTags.filter(x => x.trim().length !== 0).length > 0) {
// Create a new root element to hold the tags to keep
const newRoot = load("<div></div>")("div");
scrapeOptions.includeTags.forEach((tag) => {
@ -69,7 +69,7 @@ export const removeUnwantedElements = (
soup("script, style, noscript, meta, head").remove();
if (scrapeOptions.excludeTags && scrapeOptions.excludeTags.length > 0) {
if (scrapeOptions.excludeTags && scrapeOptions.excludeTags.filter(x => x.trim().length !== 0).length > 0) {
scrapeOptions.excludeTags.forEach((tag) => {
let elementsToRemove: Cheerio<AnyNode>;
if (tag.startsWith("*") && tag.endsWith("*")) {