From 33555742a79fc99eb72b11652fb70a145a84fb2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Fri, 4 Oct 2024 21:47:16 +0200 Subject: [PATCH] feat(scrapeURL): more intense tests --- .../src/scraper/scrapeURL/scrapeURL.test.ts | 465 +++++++++--------- 1 file changed, 246 insertions(+), 219 deletions(-) diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts index 2643bb17..64028af2 100644 --- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts +++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts @@ -2,232 +2,258 @@ import "dotenv/config"; import { scrapeURL } from "."; import { scrapeOptions } from "../../controllers/v1/types"; +import { Engine } from "./engines"; + +const testEngines: (Engine | undefined)[] = [ + undefined, + "fire-engine;chrome-cdp", + "fire-engine;playwright", + "fire-engine;tlsclient", + "scrapingbee", + "scrapingbeeLoad", +]; + +const testEnginesScreenshot: (Engine | undefined)[] = [ + undefined, + "fire-engine;chrome-cdp", + "fire-engine;playwright", + "scrapingbee", + "scrapingbeeLoad", +]; describe("Standalone scrapeURL tests", () => { - it("Basic scrape", async () => { - const out = await scrapeURL("test:scrape-basic", "https://www.roastmywebsite.ai/", scrapeOptions.parse({})); + describe.each(testEngines)("Engine %s", (forceEngine: Engine | undefined) => { + it("Basic scrape", async () => { + const out = await scrapeURL("test:scrape-basic", "https://www.roastmywebsite.ai/", scrapeOptions.parse({}), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).not.toHaveProperty("content"); + expect(out.document).toHaveProperty("markdown"); + expect(out.document).toHaveProperty("metadata"); + expect(out.document).not.toHaveProperty("html"); + expect(out.document.markdown).toContain("_Roast_"); + expect(out.document.metadata.error).toBeUndefined(); + expect(out.document.metadata.title).toBe("Roast My Website"); + expect(out.document.metadata.description).toBe( + "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️" + ); + expect(out.document.metadata.keywords).toBe( + "Roast My Website,Roast,Website,GitHub,Firecrawl" + ); + expect(out.document.metadata.robots).toBe("follow, index"); + expect(out.document.metadata.ogTitle).toBe("Roast My Website"); + expect(out.document.metadata.ogDescription).toBe( + "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️" + ); + expect(out.document.metadata.ogUrl).toBe( + "https://www.roastmywebsite.ai" + ); + expect(out.document.metadata.ogImage).toBe( + "https://www.roastmywebsite.ai/og.png" + ); + expect(out.document.metadata.ogLocaleAlternate).toStrictEqual([]); + expect(out.document.metadata.ogSiteName).toBe("Roast My Website"); + expect(out.document.metadata.sourceURL).toBe( + "https://www.roastmywebsite.ai/" + ); + expect(out.document.metadata.statusCode).toBe(200); + } + + }, 30000); - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).not.toHaveProperty("content"); - expect(out.document).toHaveProperty("markdown"); - expect(out.document).toHaveProperty("metadata"); - expect(out.document).not.toHaveProperty("html"); - expect(out.document.markdown).toContain("_Roast_"); - expect(out.document.metadata.error).toBeUndefined(); - expect(out.document.metadata.title).toBe("Roast My Website"); - expect(out.document.metadata.description).toBe( - "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️" - ); - expect(out.document.metadata.keywords).toBe( - "Roast My Website,Roast,Website,GitHub,Firecrawl" - ); - expect(out.document.metadata.robots).toBe("follow, index"); - expect(out.document.metadata.ogTitle).toBe("Roast My Website"); - expect(out.document.metadata.ogDescription).toBe( - "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️" - ); - expect(out.document.metadata.ogUrl).toBe( - "https://www.roastmywebsite.ai" - ); - expect(out.document.metadata.ogImage).toBe( - "https://www.roastmywebsite.ai/og.png" - ); - expect(out.document.metadata.ogLocaleAlternate).toStrictEqual([]); - expect(out.document.metadata.ogSiteName).toBe("Roast My Website"); - expect(out.document.metadata.sourceURL).toBe( - "https://www.roastmywebsite.ai/" - ); - expect(out.document.metadata.statusCode).toBe(200); - } + it("Scrape with formats markdown and html", async () => { + const out = await scrapeURL("test:scrape-formats-markdown-html", "https://roastmywebsite.ai", scrapeOptions.parse({ + formats: ["markdown", "html"], + }), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).toHaveProperty("markdown"); + expect(out.document).toHaveProperty("html"); + expect(out.document).toHaveProperty("metadata"); + expect(out.document.markdown).toContain("_Roast_"); + expect(out.document.html).toContain(" { - const out = await scrapeURL("test:scrape-formats-markdown-html", "https://roastmywebsite.ai", scrapeOptions.parse({ - formats: ["markdown", "html"], - })); + it("Scrape with onlyMainContent disabled", async () => { + const out = await scrapeURL("test:scrape-onlyMainContent-false", "https://www.scrapethissite.com/", scrapeOptions.parse({ + onlyMainContent: false, + }), { forceEngine }); - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty("markdown"); - expect(out.document).toHaveProperty("html"); - expect(out.document).toHaveProperty("metadata"); - expect(out.document.markdown).toContain("_Roast_"); - expect(out.document.html).toContain(" { + const out = await scrapeURL("test:scrape-excludeTags", "https://www.scrapethissite.com/", scrapeOptions.parse({ + onlyMainContent: false, + excludeTags: ['.nav', '#footer', 'strong'], + }), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).toHaveProperty("markdown"); + expect(out.document).toHaveProperty("metadata"); + expect(out.document).not.toHaveProperty("html"); + expect(out.document.markdown).not.toContain("Hartley Brody 2023"); + expect(out.document.markdown).not.toContain("[FAQ](/faq/)"); + } + }, 30000); + + it("Scrape of a page with 400 status code", async () => { + const out = await scrapeURL("test:scrape-400", "https://httpstat.us/400", scrapeOptions.parse({}), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).toHaveProperty('markdown'); + expect(out.document).toHaveProperty('metadata'); + expect(out.document.metadata.statusCode).toBe(400); + } + }, 30000); + + it("Scrape of a page with 401 status code", async () => { + const out = await scrapeURL("test:scrape-401", "https://httpstat.us/401", scrapeOptions.parse({}), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).toHaveProperty('markdown'); + expect(out.document).toHaveProperty('metadata'); + expect(out.document.metadata.statusCode).toBe(401); + } + }, 30000); + + it("Scrape of a page with 403 status code", async () => { + const out = await scrapeURL("test:scrape-403", "https://httpstat.us/403", scrapeOptions.parse({}), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).toHaveProperty('markdown'); + expect(out.document).toHaveProperty('metadata'); + expect(out.document.metadata.statusCode).toBe(403); + } + }, 30000); + + it("Scrape of a page with 404 status code", async () => { + const out = await scrapeURL("test:scrape-404", "https://httpstat.us/404", scrapeOptions.parse({}), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).toHaveProperty('markdown'); + expect(out.document).toHaveProperty('metadata'); + expect(out.document.metadata.statusCode).toBe(404); + } + }, 30000); + + it("Scrape of a page with 405 status code", async () => { + const out = await scrapeURL("test:scrape-405", "https://httpstat.us/405", scrapeOptions.parse({}), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).toHaveProperty('markdown'); + expect(out.document).toHaveProperty('metadata'); + expect(out.document.metadata.statusCode).toBe(405); + } + }, 30000); + + it("Scrape of a page with 500 status code", async () => { + const out = await scrapeURL("test:scrape-500", "https://httpstat.us/500", scrapeOptions.parse({}), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).toHaveProperty('markdown'); + expect(out.document).toHaveProperty('metadata'); + expect(out.document.metadata.statusCode).toBe(500); + } + }, 30000); - it("Scrape with onlyMainContent disabled", async () => { - const out = await scrapeURL("test:scrape-onlyMainContent-false", "https://www.scrapethissite.com/", scrapeOptions.parse({ - onlyMainContent: false, - })); + it("Scrape a redirected page", async () => { + const out = await scrapeURL("test:scrape-redirect", "https://scrapethissite.com/", scrapeOptions.parse({}), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).toHaveProperty('markdown'); + expect(out.document.markdown).toContain("Explore Sandbox"); + expect(out.document).toHaveProperty('metadata'); + expect(out.document.metadata.sourceURL).toBe("https://scrapethissite.com/"); + expect(out.document.metadata.url).toBe("https://www.scrapethissite.com/"); + expect(out.document.metadata.statusCode).toBe(200); + expect(out.document.metadata.error).toBeUndefined(); + } + }, 30000); + }); - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty("markdown"); - expect(out.document).toHaveProperty("metadata"); - expect(out.document).not.toHaveProperty("html"); - expect(out.document.markdown).toContain("[FAQ](/faq/)"); // .nav - expect(out.document.markdown).toContain("Hartley Brody 2023"); // #footer - } - }, 30000); - - it("Scrape with excludeTags", async () => { - const out = await scrapeURL("test:scrape-excludeTags", "https://www.scrapethissite.com/", scrapeOptions.parse({ - onlyMainContent: false, - excludeTags: ['.nav', '#footer', 'strong'], - })); - - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty("markdown"); - expect(out.document).toHaveProperty("metadata"); - expect(out.document).not.toHaveProperty("html"); - expect(out.document.markdown).not.toContain("Hartley Brody 2023"); - expect(out.document.markdown).not.toContain("[FAQ](/faq/)"); - } - }, 30000); - - it("Scrape of a page with 400 status code", async () => { - const out = await scrapeURL("test:scrape-400", "https://httpstat.us/400", scrapeOptions.parse({})); - - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty('markdown'); - expect(out.document).toHaveProperty('metadata'); - expect(out.document.metadata.statusCode).toBe(400); - } - }, 30000); - - it("Scrape of a page with 401 status code", async () => { - const out = await scrapeURL("test:scrape-401", "https://httpstat.us/401", scrapeOptions.parse({})); - - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty('markdown'); - expect(out.document).toHaveProperty('metadata'); - expect(out.document.metadata.statusCode).toBe(401); - } - }, 30000); - - it("Scrape of a page with 403 status code", async () => { - const out = await scrapeURL("test:scrape-403", "https://httpstat.us/403", scrapeOptions.parse({})); - - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty('markdown'); - expect(out.document).toHaveProperty('metadata'); - expect(out.document.metadata.statusCode).toBe(403); - } - }, 30000); - - it("Scrape of a page with 404 status code", async () => { - const out = await scrapeURL("test:scrape-404", "https://httpstat.us/404", scrapeOptions.parse({})); - - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty('markdown'); - expect(out.document).toHaveProperty('metadata'); - expect(out.document.metadata.statusCode).toBe(404); - } - }, 30000); - - it("Scrape of a page with 405 status code", async () => { - const out = await scrapeURL("test:scrape-405", "https://httpstat.us/405", scrapeOptions.parse({})); - - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty('markdown'); - expect(out.document).toHaveProperty('metadata'); - expect(out.document.metadata.statusCode).toBe(405); - } - }, 30000); - - it("Scrape of a page with 500 status code", async () => { - const out = await scrapeURL("test:scrape-500", "https://httpstat.us/500", scrapeOptions.parse({})); - - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty('markdown'); - expect(out.document).toHaveProperty('metadata'); - expect(out.document.metadata.statusCode).toBe(500); - } - }, 30000); - - it("Scrape with screenshot", async () => { - const out = await scrapeURL("test:scrape-screenshot", "https://www.scrapethissite.com/", scrapeOptions.parse({ - formats: ["screenshot"], - })); - - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty('screenshot'); - expect(typeof out.document.screenshot).toBe("string"); - expect(out.document).toHaveProperty('metadata'); - expect(out.document.metadata.statusCode).toBe(200); - expect(out.document.metadata.error).toBeUndefined(); - } - }, 30000); - - it("Scrape with full-page screenshot", async () => { - const out = await scrapeURL("test:scrape-screenshot-fullPage", "https://www.scrapethissite.com/", scrapeOptions.parse({ - formats: ["screenshot@fullPage"], - })); - - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty('screenshot'); - expect(typeof out.document.screenshot).toBe("string"); - expect(out.document).toHaveProperty('metadata'); - expect(out.document.metadata.statusCode).toBe(200); - expect(out.document.metadata.error).toBeUndefined(); - } - }, 30000); - - it("Scrape a redirected page", async () => { - const out = await scrapeURL("test:scrape-screenshot-fullPage", "https://scrapethissite.com/", scrapeOptions.parse({})); - - expect(out.logs.length).toBeGreaterThan(0); - expect(out.success).toBe(true); - if (out.success) { - expect(out.document.warning).toBeUndefined(); - expect(out.document).toHaveProperty('markdown'); - expect(out.document.markdown).toContain("Explore Sandbox"); - expect(out.document).toHaveProperty('metadata'); - expect(out.document.metadata.sourceURL).toBe("https://scrapethissite.com/"); - expect(out.document.metadata.url).toBe("https://www.scrapethissite.com/"); - expect(out.document.metadata.statusCode).toBe(200); - expect(out.document.metadata.error).toBeUndefined(); - } - }, 30000); + describe.each(testEnginesScreenshot)("Screenshot on engine %s", (forceEngine: Engine | undefined) => { + it("Scrape with screenshot", async () => { + const out = await scrapeURL("test:scrape-screenshot", "https://www.scrapethissite.com/", scrapeOptions.parse({ + formats: ["screenshot"], + }), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).toHaveProperty('screenshot'); + expect(typeof out.document.screenshot).toBe("string"); + expect(out.document.screenshot!.startsWith("https://service.firecrawl.dev/storage/v1/object/public/media/")); + // TODO: attempt to fetch screenshot + expect(out.document).toHaveProperty('metadata'); + expect(out.document.metadata.statusCode).toBe(200); + expect(out.document.metadata.error).toBeUndefined(); + } + }, 30000); + + it("Scrape with full-page screenshot", async () => { + const out = await scrapeURL("test:scrape-screenshot-fullPage", "https://www.scrapethissite.com/", scrapeOptions.parse({ + formats: ["screenshot@fullPage"], + }), { forceEngine }); + + expect(out.logs.length).toBeGreaterThan(0); + expect(out.success).toBe(true); + if (out.success) { + expect(out.document.warning).toBeUndefined(); + expect(out.document).toHaveProperty('screenshot'); + expect(typeof out.document.screenshot).toBe("string"); + expect(out.document.screenshot!.startsWith("https://service.firecrawl.dev/storage/v1/object/public/media/")); + // TODO: attempt to fetch screenshot + expect(out.document).toHaveProperty('metadata'); + expect(out.document.metadata.statusCode).toBe(200); + expect(out.document.metadata.error).toBeUndefined(); + } + }, 30000); + }); it("Scrape of a PDF file", async () => { const out = await scrapeURL("test:scrape-pdf", "https://arxiv.org/pdf/astro-ph/9301001.pdf", scrapeOptions.parse({})); @@ -320,7 +346,8 @@ describe("Standalone scrapeURL tests", () => { } }, 120000) - test.concurrent.each(new Array(100).fill(0).map((_, i) => "https://www.scrapethissite.com/?i=" + i))("Concurrent scrapes", async (url) => { + test.concurrent.each(new Array(100).fill(0).map((_, i) => i))("Concurrent scrape #%i", async (i) => { + const url = "https://www.scrapethissite.com/?i=" + i; const id = "test:concurrent:" + url; const out = await scrapeURL(id, url, scrapeOptions.parse({}));