mirror of
https://github.com/mendableai/firecrawl.git
synced 2024-11-16 11:42:24 +08:00
Merge pull request #25 from mendableai/feat/replace-all-paths-to-absolute-paths
Added option to replace all relative paths with absolute paths
This commit is contained in:
Rafael Miller
GitHub
commit
3c14b02f8b
|
|
@ -22,6 +22,7 @@ export type WebScraperOptions = {
|
||||||
maxCrawledLinks?: number;
|
maxCrawledLinks?: number;
|
||||||
limit?: number;
|
limit?: number;
|
||||||
generateImgAltText?: boolean;
|
generateImgAltText?: boolean;
|
||||||
|
replaceAllPathsWithAbsolutePaths?: boolean;
|
||||||
};
|
};
|
||||||
pageOptions?: PageOptions;
|
pageOptions?: PageOptions;
|
||||||
concurrentRequests?: number;
|
concurrentRequests?: number;
|
||||||
|
|
|
||||||
|
|
@ -1,179 +0,0 @@
|
||||||
import { WebScraperDataProvider } from "../index";
|
|
||||||
|
|
||||||
describe("WebScraperDataProvider", () => {
|
|
||||||
describe("replaceImgPathsWithAbsolutePaths", () => {
|
|
||||||
it("should replace image paths with absolute paths", () => {
|
|
||||||
const webScraperDataProvider = new WebScraperDataProvider();
|
|
||||||
const documents = [
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/page" },
|
|
||||||
content: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page" },
|
|
||||||
content: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page" },
|
|
||||||
content: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/data-image" },
|
|
||||||
content: "",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const expectedDocuments = [
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/page" },
|
|
||||||
content: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page" },
|
|
||||||
content: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page" },
|
|
||||||
content: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/data-image" },
|
|
||||||
content: "",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const result =
|
|
||||||
webScraperDataProvider.replaceImgPathsWithAbsolutePaths(documents);
|
|
||||||
expect(result).toEqual(expectedDocuments);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should handle absolute URLs without modification", () => {
|
|
||||||
const webScraperDataProvider = new WebScraperDataProvider();
|
|
||||||
const documents = [
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/page" },
|
|
||||||
content: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page" },
|
|
||||||
content:
|
|
||||||
"",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const expectedDocuments = [
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/page" },
|
|
||||||
content: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page" },
|
|
||||||
content:
|
|
||||||
"",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const result =
|
|
||||||
webScraperDataProvider.replaceImgPathsWithAbsolutePaths(documents);
|
|
||||||
expect(result).toEqual(expectedDocuments);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should not replace non-image content within the documents", () => {
|
|
||||||
const webScraperDataProvider = new WebScraperDataProvider();
|
|
||||||
const documents = [
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/page" },
|
|
||||||
content:
|
|
||||||
"This is a test.  Here is a link: [Example](https://example.com).",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page" },
|
|
||||||
content:
|
|
||||||
"Another test.  Here is some **bold text**.",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const expectedDocuments = [
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/page" },
|
|
||||||
content:
|
|
||||||
"This is a test.  Here is a link: [Example](https://example.com).",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page" },
|
|
||||||
content:
|
|
||||||
"Another test.  Here is some **bold text**.",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const result =
|
|
||||||
webScraperDataProvider.replaceImgPathsWithAbsolutePaths(documents);
|
|
||||||
expect(result).toEqual(expectedDocuments);
|
|
||||||
});
|
|
||||||
it("should replace multiple image paths within the documents", () => {
|
|
||||||
const webScraperDataProvider = new WebScraperDataProvider();
|
|
||||||
const documents = [
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/page" },
|
|
||||||
content:
|
|
||||||
"This is a test.  Here is a link: [Example](https://example.com). ",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page" },
|
|
||||||
content:
|
|
||||||
"Another test.  Here is some **bold text**. ",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const expectedDocuments = [
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/page" },
|
|
||||||
content:
|
|
||||||
"This is a test.  Here is a link: [Example](https://example.com). ",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page" },
|
|
||||||
content:
|
|
||||||
"Another test.  Here is some **bold text**. ",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const result =
|
|
||||||
webScraperDataProvider.replaceImgPathsWithAbsolutePaths(documents);
|
|
||||||
expect(result).toEqual(expectedDocuments);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should replace image paths within the documents with complex URLs", () => {
|
|
||||||
const webScraperDataProvider = new WebScraperDataProvider();
|
|
||||||
const documents = [
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/page/subpage" },
|
|
||||||
content:
|
|
||||||
"This is a test.  Here is a link: [Example](https://example.com). ",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page/subpage" },
|
|
||||||
content:
|
|
||||||
"Another test.  Here is some **bold text**. ",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const expectedDocuments = [
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/page/subpage" },
|
|
||||||
content:
|
|
||||||
"This is a test.  Here is a link: [Example](https://example.com). ",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
metadata: { sourceURL: "https://example.com/another-page/subpage" },
|
|
||||||
content:
|
|
||||||
"Another test.  Here is some **bold text**. ",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const result =
|
|
||||||
webScraperDataProvider.replaceImgPathsWithAbsolutePaths(documents);
|
|
||||||
expect(result).toEqual(expectedDocuments);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
@ -6,6 +6,7 @@ import { WebCrawler } from "./crawler";
|
||||||
import { getValue, setValue } from "../../services/redis";
|
import { getValue, setValue } from "../../services/redis";
|
||||||
import { getImageDescription } from "./utils/gptVision";
|
import { getImageDescription } from "./utils/gptVision";
|
||||||
import { fetchAndProcessPdf } from "./utils/pdfProcessor";
|
import { fetchAndProcessPdf } from "./utils/pdfProcessor";
|
||||||
|
import { replaceImgPathsWithAbsolutePaths, replacePathsWithAbsolutePaths } from "./utils/replacePaths";
|
||||||
|
|
||||||
|
|
||||||
export class WebScraperDataProvider {
|
export class WebScraperDataProvider {
|
||||||
|
|
@ -19,6 +20,7 @@ export class WebScraperDataProvider {
|
||||||
private concurrentRequests: number = 20;
|
private concurrentRequests: number = 20;
|
||||||
private generateImgAltText: boolean = false;
|
private generateImgAltText: boolean = false;
|
||||||
private pageOptions?: PageOptions;
|
private pageOptions?: PageOptions;
|
||||||
|
private replaceAllPathsWithAbsolutePaths?: boolean = false;
|
||||||
|
|
||||||
authorize(): void {
|
authorize(): void {
|
||||||
throw new Error("Method not implemented.");
|
throw new Error("Method not implemented.");
|
||||||
|
|
@ -100,7 +102,13 @@ export class WebScraperDataProvider {
|
||||||
|
|
||||||
let documents = await this.convertUrlsToDocuments(links, inProgress);
|
let documents = await this.convertUrlsToDocuments(links, inProgress);
|
||||||
documents = await this.getSitemapData(this.urls[0], documents);
|
documents = await this.getSitemapData(this.urls[0], documents);
|
||||||
documents = this.replaceImgPathsWithAbsolutePaths(documents);
|
|
||||||
|
if (this.replaceAllPathsWithAbsolutePaths) {
|
||||||
|
documents = replacePathsWithAbsolutePaths(documents);
|
||||||
|
} else {
|
||||||
|
documents = replaceImgPathsWithAbsolutePaths(documents);
|
||||||
|
}
|
||||||
|
|
||||||
if (this.generateImgAltText) {
|
if (this.generateImgAltText) {
|
||||||
documents = await this.generatesImgAltText(documents);
|
documents = await this.generatesImgAltText(documents);
|
||||||
}
|
}
|
||||||
|
|
@ -164,7 +172,13 @@ export class WebScraperDataProvider {
|
||||||
this.urls.filter((link) => !link.endsWith(".pdf")),
|
this.urls.filter((link) => !link.endsWith(".pdf")),
|
||||||
inProgress
|
inProgress
|
||||||
);
|
);
|
||||||
documents = this.replaceImgPathsWithAbsolutePaths(documents);
|
|
||||||
|
if (this.replaceAllPathsWithAbsolutePaths) {
|
||||||
|
documents = replacePathsWithAbsolutePaths(documents);
|
||||||
|
} else {
|
||||||
|
documents = replaceImgPathsWithAbsolutePaths(documents);
|
||||||
|
}
|
||||||
|
|
||||||
if (this.generateImgAltText) {
|
if (this.generateImgAltText) {
|
||||||
documents = await this.generatesImgAltText(documents);
|
documents = await this.generatesImgAltText(documents);
|
||||||
}
|
}
|
||||||
|
|
@ -197,7 +211,13 @@ export class WebScraperDataProvider {
|
||||||
);
|
);
|
||||||
|
|
||||||
documents = await this.getSitemapData(this.urls[0], documents);
|
documents = await this.getSitemapData(this.urls[0], documents);
|
||||||
documents = this.replaceImgPathsWithAbsolutePaths(documents);
|
|
||||||
|
if (this.replaceAllPathsWithAbsolutePaths) {
|
||||||
|
documents = replacePathsWithAbsolutePaths(documents);
|
||||||
|
} else {
|
||||||
|
documents = replaceImgPathsWithAbsolutePaths(documents);
|
||||||
|
}
|
||||||
|
|
||||||
if (this.generateImgAltText) {
|
if (this.generateImgAltText) {
|
||||||
documents = await this.generatesImgAltText(documents);
|
documents = await this.generatesImgAltText(documents);
|
||||||
}
|
}
|
||||||
|
|
@ -351,6 +371,7 @@ export class WebScraperDataProvider {
|
||||||
this.generateImgAltText =
|
this.generateImgAltText =
|
||||||
options.crawlerOptions?.generateImgAltText ?? false;
|
options.crawlerOptions?.generateImgAltText ?? false;
|
||||||
this.pageOptions = options.pageOptions ?? {onlyMainContent: false};
|
this.pageOptions = options.pageOptions ?? {onlyMainContent: false};
|
||||||
|
this.replaceAllPathsWithAbsolutePaths = options.crawlerOptions?.replaceAllPathsWithAbsolutePaths ?? false;
|
||||||
|
|
||||||
//! @nicolas, for some reason this was being injected and breakign everything. Don't have time to find source of the issue so adding this check
|
//! @nicolas, for some reason this was being injected and breakign everything. Don't have time to find source of the issue so adding this check
|
||||||
this.excludes = this.excludes.filter((item) => item !== "");
|
this.excludes = this.excludes.filter((item) => item !== "");
|
||||||
|
|
@ -436,40 +457,4 @@ export class WebScraperDataProvider {
|
||||||
|
|
||||||
return documents;
|
return documents;
|
||||||
};
|
};
|
||||||
|
|
||||||
replaceImgPathsWithAbsolutePaths = (documents: Document[]): Document[] => {
|
|
||||||
try {
|
|
||||||
documents.forEach((document) => {
|
|
||||||
const baseUrl = new URL(document.metadata.sourceURL).origin;
|
|
||||||
const images =
|
|
||||||
document.content.match(
|
|
||||||
/!\[.*?\]\(((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\)/g
|
|
||||||
) || [];
|
|
||||||
|
|
||||||
images.forEach((image: string) => {
|
|
||||||
let imageUrl = image.match(/\(([^)]+)\)/)[1];
|
|
||||||
let altText = image.match(/\[(.*?)\]/)[1];
|
|
||||||
|
|
||||||
if (!imageUrl.startsWith("data:image")) {
|
|
||||||
if (!imageUrl.startsWith("http")) {
|
|
||||||
if (imageUrl.startsWith("/")) {
|
|
||||||
imageUrl = imageUrl.substring(1);
|
|
||||||
}
|
|
||||||
imageUrl = new URL(imageUrl, baseUrl).toString();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
document.content = document.content.replace(
|
|
||||||
image,
|
|
||||||
``
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
return documents;
|
|
||||||
} catch (error) {
|
|
||||||
console.error("Error replacing img paths with absolute paths", error);
|
|
||||||
return documents;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,40 +1,47 @@
|
||||||
import * as pdfProcessor from '../pdfProcessor';
|
import * as pdfProcessor from '../pdfProcessor';
|
||||||
|
|
||||||
describe('PDF Processing Module - Integration Test', () => {
|
describe('PDF Processing Module - Integration Test', () => {
|
||||||
it('should download and read a simple PDF file by URL', async () => {
|
it('should correctly process a simple PDF file without the LLAMAPARSE_API_KEY', async () => {
|
||||||
|
delete process.env.LLAMAPARSE_API_KEY;
|
||||||
const pdfContent = await pdfProcessor.fetchAndProcessPdf('https://s3.us-east-1.amazonaws.com/storage.mendable.ai/rafa-testing/test%20%281%29.pdf');
|
const pdfContent = await pdfProcessor.fetchAndProcessPdf('https://s3.us-east-1.amazonaws.com/storage.mendable.ai/rafa-testing/test%20%281%29.pdf');
|
||||||
expect(pdfContent).toEqual("Dummy PDF file");
|
expect(pdfContent.trim()).toEqual("Dummy PDF file");
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should download and read a complex PDF file by URL', async () => {
|
// We're hitting the LLAMAPARSE rate limit 🫠
|
||||||
const pdfContent = await pdfProcessor.fetchAndProcessPdf('https://arxiv.org/pdf/2307.06435.pdf');
|
// it('should download and read a simple PDF file by URL', async () => {
|
||||||
|
// const pdfContent = await pdfProcessor.fetchAndProcessPdf('https://s3.us-east-1.amazonaws.com/storage.mendable.ai/rafa-testing/test%20%281%29.pdf');
|
||||||
|
// expect(pdfContent).toEqual("Dummy PDF file");
|
||||||
|
// });
|
||||||
|
|
||||||
const expectedContent = 'A Comprehensive Overview of Large Language Models\n' +
|
// it('should download and read a complex PDF file by URL', async () => {
|
||||||
' a a,∗ b,∗ c,d,∗ e,f e,f g,i\n' +
|
// const pdfContent = await pdfProcessor.fetchAndProcessPdf('https://arxiv.org/pdf/2307.06435.pdf');
|
||||||
' Humza Naveed , Asad Ullah Khan , Shi Qiu , Muhammad Saqib , Saeed Anwar , Muhammad Usman , Naveed Akhtar ,\n' +
|
|
||||||
' Nick Barnes h, Ajmal Mian i\n' +
|
// const expectedContent = 'A Comprehensive Overview of Large Language Models\n' +
|
||||||
' aUniversity of Engineering and Technology (UET), Lahore, Pakistan\n' +
|
// ' a a,∗ b,∗ c,d,∗ e,f e,f g,i\n' +
|
||||||
' bThe Chinese University of Hong Kong (CUHK), HKSAR, China\n' +
|
// ' Humza Naveed , Asad Ullah Khan , Shi Qiu , Muhammad Saqib , Saeed Anwar , Muhammad Usman , Naveed Akhtar ,\n' +
|
||||||
' cUniversity of Technology Sydney (UTS), Sydney, Australia\n' +
|
// ' Nick Barnes h, Ajmal Mian i\n' +
|
||||||
' dCommonwealth Scientific and Industrial Research Organisation (CSIRO), Sydney, Australia\n' +
|
// ' aUniversity of Engineering and Technology (UET), Lahore, Pakistan\n' +
|
||||||
' eKing Fahd University of Petroleum and Minerals (KFUPM), Dhahran, Saudi Arabia\n' +
|
// ' bThe Chinese University of Hong Kong (CUHK), HKSAR, China\n' +
|
||||||
' fSDAIA-KFUPM Joint Research Center for Artificial Intelligence (JRCAI), Dhahran, Saudi Arabia\n' +
|
// ' cUniversity of Technology Sydney (UTS), Sydney, Australia\n' +
|
||||||
' gThe University of Melbourne (UoM), Melbourne, Australia\n' +
|
// ' dCommonwealth Scientific and Industrial Research Organisation (CSIRO), Sydney, Australia\n' +
|
||||||
' hAustralian National University (ANU), Canberra, Australia\n' +
|
// ' eKing Fahd University of Petroleum and Minerals (KFUPM), Dhahran, Saudi Arabia\n' +
|
||||||
' iThe University of Western Australia (UWA), Perth, Australia\n' +
|
// ' fSDAIA-KFUPM Joint Research Center for Artificial Intelligence (JRCAI), Dhahran, Saudi Arabia\n' +
|
||||||
' Abstract\n' +
|
// ' gThe University of Melbourne (UoM), Melbourne, Australia\n' +
|
||||||
' Large Language Models (LLMs) have recently demonstrated remarkable capabilities in natural language processing tasks and\n' +
|
// ' hAustralian National University (ANU), Canberra, Australia\n' +
|
||||||
' beyond. This success of LLMs has led to a large influx of research contributions in this direction. These works encompass diverse\n' +
|
// ' iThe University of Western Australia (UWA), Perth, Australia\n' +
|
||||||
' topics such as architectural innovations, better training strategies, context length improvements, fine-tuning, multi-modal LLMs,\n' +
|
// ' Abstract\n' +
|
||||||
' robotics, datasets, benchmarking, efficiency, and more. With the rapid development of techniques and regular breakthroughs in\n' +
|
// ' Large Language Models (LLMs) have recently demonstrated remarkable capabilities in natural language processing tasks and\n' +
|
||||||
' LLM research, it has become considerably challenging to perceive the bigger picture of the advances in this direction. Considering\n' +
|
// ' beyond. This success of LLMs has led to a large influx of research contributions in this direction. These works encompass diverse\n' +
|
||||||
' the rapidly emerging plethora of literature on LLMs, it is imperative that the research community is able to benefit from a concise\n' +
|
// ' topics such as architectural innovations, better training strategies, context length improvements, fine-tuning, multi-modal LLMs,\n' +
|
||||||
' yet comprehensive overview of the recent developments in this field. This article provides an overview of the existing literature\n' +
|
// ' robotics, datasets, benchmarking, efficiency, and more. With the rapid development of techniques and regular breakthroughs in\n' +
|
||||||
' on a broad range of LLM-related concepts. Our self-contained comprehensive overview of LLMs discusses relevant background\n' +
|
// ' LLM research, it has become considerably challenging to perceive the bigger picture of the advances in this direction. Considering\n' +
|
||||||
' concepts along with covering the advanced topics at the frontier of research in LLMs. This review article is intended to not only\n' +
|
// ' the rapidly emerging plethora of literature on LLMs, it is imperative that the research community is able to benefit from a concise\n' +
|
||||||
' provide a systematic survey but also a quick comprehensive reference for the researchers and practitioners to draw insights from\n' +
|
// ' yet comprehensive overview of the recent developments in this field. This article provides an overview of the existing literature\n' +
|
||||||
' extensive informative summaries of the existing works to advance the LLM research.\n'
|
// ' on a broad range of LLM-related concepts. Our self-contained comprehensive overview of LLMs discusses relevant background\n' +
|
||||||
expect(pdfContent).toContain(expectedContent);
|
// ' concepts along with covering the advanced topics at the frontier of research in LLMs. This review article is intended to not only\n' +
|
||||||
}, 60000);
|
// ' provide a systematic survey but also a quick comprehensive reference for the researchers and practitioners to draw insights from\n' +
|
||||||
|
// ' extensive informative summaries of the existing works to advance the LLM research.\n'
|
||||||
|
// expect(pdfContent).toContain(expectedContent);
|
||||||
|
// }, 60000);
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
@ -0,0 +1,114 @@
|
||||||
|
import { Document } from "../../../../lib/entities";
|
||||||
|
import { replacePathsWithAbsolutePaths, replaceImgPathsWithAbsolutePaths } from "../replacePaths";
|
||||||
|
|
||||||
|
describe('replacePaths', () => {
|
||||||
|
describe('replacePathsWithAbsolutePaths', () => {
|
||||||
|
it('should replace relative paths with absolute paths', () => {
|
||||||
|
const documents: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'This is a [link](/path/to/resource) and an image .'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const expectedDocuments: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'This is a [link](https://example.com/path/to/resource) and an image .'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const result = replacePathsWithAbsolutePaths(documents);
|
||||||
|
expect(result).toEqual(expectedDocuments);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should not alter absolute URLs', () => {
|
||||||
|
const documents: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'This is an [external link](https://external.com/path) and an image .'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const result = replacePathsWithAbsolutePaths(documents);
|
||||||
|
expect(result).toEqual(documents); // Expect no change
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should not alter data URLs for images', () => {
|
||||||
|
const documents: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'This is an image: .'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const result = replacePathsWithAbsolutePaths(documents);
|
||||||
|
expect(result).toEqual(documents); // Expect no change
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle multiple links and images correctly', () => {
|
||||||
|
const documents: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'Here are two links: [link1](/path1) and [link2](/path2), and two images:  .'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const expectedDocuments: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'Here are two links: [link1](https://example.com/path1) and [link2](https://example.com/path2), and two images:  .'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const result = replacePathsWithAbsolutePaths(documents);
|
||||||
|
expect(result).toEqual(expectedDocuments);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should correctly handle a mix of absolute and relative paths', () => {
|
||||||
|
const documents: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'Mixed paths: [relative](/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const expectedDocuments: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'Mixed paths: [relative](https://example.com/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const result = replacePathsWithAbsolutePaths(documents);
|
||||||
|
expect(result).toEqual(expectedDocuments);
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('replaceImgPathsWithAbsolutePaths', () => {
|
||||||
|
it('should replace relative image paths with absolute paths', () => {
|
||||||
|
const documents: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'Here is an image: .'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const expectedDocuments: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'Here is an image: .'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const result = replaceImgPathsWithAbsolutePaths(documents);
|
||||||
|
expect(result).toEqual(expectedDocuments);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should not alter data:image URLs', () => {
|
||||||
|
const documents: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'An image with a data URL: .'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const result = replaceImgPathsWithAbsolutePaths(documents);
|
||||||
|
expect(result).toEqual(documents); // Expect no change
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle multiple images with a mix of data and relative URLs', () => {
|
||||||
|
const documents: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'Multiple images:   .'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const expectedDocuments: Document[] = [{
|
||||||
|
metadata: { sourceURL: 'https://example.com' },
|
||||||
|
content: 'Multiple images:   .'
|
||||||
|
}];
|
||||||
|
|
||||||
|
const result = replaceImgPathsWithAbsolutePaths(documents);
|
||||||
|
expect(result).toEqual(expectedDocuments);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
80
apps/api/src/scraper/WebScraper/utils/replacePaths.ts
Normal file
80
apps/api/src/scraper/WebScraper/utils/replacePaths.ts
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
import { Document } from "../../../lib/entities";
|
||||||
|
|
||||||
|
export const replacePathsWithAbsolutePaths = (documents: Document[]): Document[] => {
|
||||||
|
try {
|
||||||
|
documents.forEach((document) => {
|
||||||
|
const baseUrl = new URL(document.metadata.sourceURL).origin;
|
||||||
|
const paths =
|
||||||
|
document.content.match(
|
||||||
|
/(!?\[.*?\])\(((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\)|href="([^"]+)"/g
|
||||||
|
) || [];
|
||||||
|
|
||||||
|
paths.forEach((path: string) => {
|
||||||
|
const isImage = path.startsWith("!");
|
||||||
|
let matchedUrl = path.match(/\(([^)]+)\)/) || path.match(/href="([^"]+)"/);
|
||||||
|
let url = matchedUrl[1];
|
||||||
|
|
||||||
|
if (!url.startsWith("data:") && !url.startsWith("http")) {
|
||||||
|
if (url.startsWith("/")) {
|
||||||
|
url = url.substring(1);
|
||||||
|
}
|
||||||
|
url = new URL(url, baseUrl).toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
const markdownLinkOrImageText = path.match(/(!?\[.*?\])/)[0];
|
||||||
|
if (isImage) {
|
||||||
|
document.content = document.content.replace(
|
||||||
|
path,
|
||||||
|
`${markdownLinkOrImageText}(${url})`
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
document.content = document.content.replace(
|
||||||
|
path,
|
||||||
|
`${markdownLinkOrImageText}(${url})`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return documents;
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Error replacing paths with absolute paths", error);
|
||||||
|
return documents;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const replaceImgPathsWithAbsolutePaths = (documents: Document[]): Document[] => {
|
||||||
|
try {
|
||||||
|
documents.forEach((document) => {
|
||||||
|
const baseUrl = new URL(document.metadata.sourceURL).origin;
|
||||||
|
const images =
|
||||||
|
document.content.match(
|
||||||
|
/!\[.*?\]\(((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\)/g
|
||||||
|
) || [];
|
||||||
|
|
||||||
|
images.forEach((image: string) => {
|
||||||
|
let imageUrl = image.match(/\(([^)]+)\)/)[1];
|
||||||
|
let altText = image.match(/\[(.*?)\]/)[1];
|
||||||
|
|
||||||
|
if (!imageUrl.startsWith("data:image")) {
|
||||||
|
if (!imageUrl.startsWith("http")) {
|
||||||
|
if (imageUrl.startsWith("/")) {
|
||||||
|
imageUrl = imageUrl.substring(1);
|
||||||
|
}
|
||||||
|
imageUrl = new URL(imageUrl, baseUrl).toString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
document.content = document.content.replace(
|
||||||
|
image,
|
||||||
|
``
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return documents;
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Error replacing img paths with absolute paths", error);
|
||||||
|
return documents;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
@ -3,7 +3,6 @@ import { getWebScraperQueue } from "./queue-service";
|
||||||
import "dotenv/config";
|
import "dotenv/config";
|
||||||
import { logtail } from "./logtail";
|
import { logtail } from "./logtail";
|
||||||
import { startWebScraperPipeline } from "../main/runWebScraper";
|
import { startWebScraperPipeline } from "../main/runWebScraper";
|
||||||
import { WebScraperDataProvider } from "../scraper/WebScraper";
|
|
||||||
import { callWebhook } from "./webhook";
|
import { callWebhook } from "./webhook";
|
||||||
|
|
||||||
getWebScraperQueue().process(
|
getWebScraperQueue().process(
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user