mirror of
https://github.com/mendableai/firecrawl.git
synced 2024-11-16 11:42:24 +08:00
Merge pull request #25 from mendableai/feat/replace-all-paths-to-absolute-paths
Added option to replace all relative paths with absolute paths
This commit is contained in:
Rafael Miller
GitHub
commit
3c14b02f8b
|
|
@ -22,6 +22,7 @@ export type WebScraperOptions = {
|
|||
maxCrawledLinks?: number;
|
||||
limit?: number;
|
||||
generateImgAltText?: boolean;
|
||||
replaceAllPathsWithAbsolutePaths?: boolean;
|
||||
};
|
||||
pageOptions?: PageOptions;
|
||||
concurrentRequests?: number;
|
||||
|
|
|
|||
|
|
@ -1,179 +0,0 @@
|
|||
import { WebScraperDataProvider } from "../index";
|
||||
|
||||
describe("WebScraperDataProvider", () => {
|
||||
describe("replaceImgPathsWithAbsolutePaths", () => {
|
||||
it("should replace image paths with absolute paths", () => {
|
||||
const webScraperDataProvider = new WebScraperDataProvider();
|
||||
const documents = [
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/page" },
|
||||
content: "",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page" },
|
||||
content: "",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page" },
|
||||
content: "",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/data-image" },
|
||||
content: "",
|
||||
},
|
||||
];
|
||||
|
||||
const expectedDocuments = [
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/page" },
|
||||
content: "",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page" },
|
||||
content: "",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page" },
|
||||
content: "",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/data-image" },
|
||||
content: "",
|
||||
},
|
||||
];
|
||||
|
||||
const result =
|
||||
webScraperDataProvider.replaceImgPathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(expectedDocuments);
|
||||
});
|
||||
|
||||
it("should handle absolute URLs without modification", () => {
|
||||
const webScraperDataProvider = new WebScraperDataProvider();
|
||||
const documents = [
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/page" },
|
||||
content: "",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page" },
|
||||
content:
|
||||
"",
|
||||
},
|
||||
];
|
||||
|
||||
const expectedDocuments = [
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/page" },
|
||||
content: "",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page" },
|
||||
content:
|
||||
"",
|
||||
},
|
||||
];
|
||||
|
||||
const result =
|
||||
webScraperDataProvider.replaceImgPathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(expectedDocuments);
|
||||
});
|
||||
|
||||
it("should not replace non-image content within the documents", () => {
|
||||
const webScraperDataProvider = new WebScraperDataProvider();
|
||||
const documents = [
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/page" },
|
||||
content:
|
||||
"This is a test.  Here is a link: [Example](https://example.com).",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page" },
|
||||
content:
|
||||
"Another test.  Here is some **bold text**.",
|
||||
},
|
||||
];
|
||||
|
||||
const expectedDocuments = [
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/page" },
|
||||
content:
|
||||
"This is a test.  Here is a link: [Example](https://example.com).",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page" },
|
||||
content:
|
||||
"Another test.  Here is some **bold text**.",
|
||||
},
|
||||
];
|
||||
|
||||
const result =
|
||||
webScraperDataProvider.replaceImgPathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(expectedDocuments);
|
||||
});
|
||||
it("should replace multiple image paths within the documents", () => {
|
||||
const webScraperDataProvider = new WebScraperDataProvider();
|
||||
const documents = [
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/page" },
|
||||
content:
|
||||
"This is a test.  Here is a link: [Example](https://example.com). ",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page" },
|
||||
content:
|
||||
"Another test.  Here is some **bold text**. ",
|
||||
},
|
||||
];
|
||||
|
||||
const expectedDocuments = [
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/page" },
|
||||
content:
|
||||
"This is a test.  Here is a link: [Example](https://example.com). ",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page" },
|
||||
content:
|
||||
"Another test.  Here is some **bold text**. ",
|
||||
},
|
||||
];
|
||||
|
||||
const result =
|
||||
webScraperDataProvider.replaceImgPathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(expectedDocuments);
|
||||
});
|
||||
|
||||
it("should replace image paths within the documents with complex URLs", () => {
|
||||
const webScraperDataProvider = new WebScraperDataProvider();
|
||||
const documents = [
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/page/subpage" },
|
||||
content:
|
||||
"This is a test.  Here is a link: [Example](https://example.com). ",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page/subpage" },
|
||||
content:
|
||||
"Another test.  Here is some **bold text**. ",
|
||||
},
|
||||
];
|
||||
|
||||
const expectedDocuments = [
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/page/subpage" },
|
||||
content:
|
||||
"This is a test.  Here is a link: [Example](https://example.com). ",
|
||||
},
|
||||
{
|
||||
metadata: { sourceURL: "https://example.com/another-page/subpage" },
|
||||
content:
|
||||
"Another test.  Here is some **bold text**. ",
|
||||
},
|
||||
];
|
||||
|
||||
const result =
|
||||
webScraperDataProvider.replaceImgPathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(expectedDocuments);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -6,6 +6,7 @@ import { WebCrawler } from "./crawler";
|
|||
import { getValue, setValue } from "../../services/redis";
|
||||
import { getImageDescription } from "./utils/gptVision";
|
||||
import { fetchAndProcessPdf } from "./utils/pdfProcessor";
|
||||
import { replaceImgPathsWithAbsolutePaths, replacePathsWithAbsolutePaths } from "./utils/replacePaths";
|
||||
|
||||
|
||||
export class WebScraperDataProvider {
|
||||
|
|
@ -19,6 +20,7 @@ export class WebScraperDataProvider {
|
|||
private concurrentRequests: number = 20;
|
||||
private generateImgAltText: boolean = false;
|
||||
private pageOptions?: PageOptions;
|
||||
private replaceAllPathsWithAbsolutePaths?: boolean = false;
|
||||
|
||||
authorize(): void {
|
||||
throw new Error("Method not implemented.");
|
||||
|
|
@ -100,7 +102,13 @@ export class WebScraperDataProvider {
|
|||
|
||||
let documents = await this.convertUrlsToDocuments(links, inProgress);
|
||||
documents = await this.getSitemapData(this.urls[0], documents);
|
||||
documents = this.replaceImgPathsWithAbsolutePaths(documents);
|
||||
|
||||
if (this.replaceAllPathsWithAbsolutePaths) {
|
||||
documents = replacePathsWithAbsolutePaths(documents);
|
||||
} else {
|
||||
documents = replaceImgPathsWithAbsolutePaths(documents);
|
||||
}
|
||||
|
||||
if (this.generateImgAltText) {
|
||||
documents = await this.generatesImgAltText(documents);
|
||||
}
|
||||
|
|
@ -164,7 +172,13 @@ export class WebScraperDataProvider {
|
|||
this.urls.filter((link) => !link.endsWith(".pdf")),
|
||||
inProgress
|
||||
);
|
||||
documents = this.replaceImgPathsWithAbsolutePaths(documents);
|
||||
|
||||
if (this.replaceAllPathsWithAbsolutePaths) {
|
||||
documents = replacePathsWithAbsolutePaths(documents);
|
||||
} else {
|
||||
documents = replaceImgPathsWithAbsolutePaths(documents);
|
||||
}
|
||||
|
||||
if (this.generateImgAltText) {
|
||||
documents = await this.generatesImgAltText(documents);
|
||||
}
|
||||
|
|
@ -197,7 +211,13 @@ export class WebScraperDataProvider {
|
|||
);
|
||||
|
||||
documents = await this.getSitemapData(this.urls[0], documents);
|
||||
documents = this.replaceImgPathsWithAbsolutePaths(documents);
|
||||
|
||||
if (this.replaceAllPathsWithAbsolutePaths) {
|
||||
documents = replacePathsWithAbsolutePaths(documents);
|
||||
} else {
|
||||
documents = replaceImgPathsWithAbsolutePaths(documents);
|
||||
}
|
||||
|
||||
if (this.generateImgAltText) {
|
||||
documents = await this.generatesImgAltText(documents);
|
||||
}
|
||||
|
|
@ -351,6 +371,7 @@ export class WebScraperDataProvider {
|
|||
this.generateImgAltText =
|
||||
options.crawlerOptions?.generateImgAltText ?? false;
|
||||
this.pageOptions = options.pageOptions ?? {onlyMainContent: false};
|
||||
this.replaceAllPathsWithAbsolutePaths = options.crawlerOptions?.replaceAllPathsWithAbsolutePaths ?? false;
|
||||
|
||||
//! @nicolas, for some reason this was being injected and breakign everything. Don't have time to find source of the issue so adding this check
|
||||
this.excludes = this.excludes.filter((item) => item !== "");
|
||||
|
|
@ -436,40 +457,4 @@ export class WebScraperDataProvider {
|
|||
|
||||
return documents;
|
||||
};
|
||||
|
||||
replaceImgPathsWithAbsolutePaths = (documents: Document[]): Document[] => {
|
||||
try {
|
||||
documents.forEach((document) => {
|
||||
const baseUrl = new URL(document.metadata.sourceURL).origin;
|
||||
const images =
|
||||
document.content.match(
|
||||
/!\[.*?\]\(((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\)/g
|
||||
) || [];
|
||||
|
||||
images.forEach((image: string) => {
|
||||
let imageUrl = image.match(/\(([^)]+)\)/)[1];
|
||||
let altText = image.match(/\[(.*?)\]/)[1];
|
||||
|
||||
if (!imageUrl.startsWith("data:image")) {
|
||||
if (!imageUrl.startsWith("http")) {
|
||||
if (imageUrl.startsWith("/")) {
|
||||
imageUrl = imageUrl.substring(1);
|
||||
}
|
||||
imageUrl = new URL(imageUrl, baseUrl).toString();
|
||||
}
|
||||
}
|
||||
|
||||
document.content = document.content.replace(
|
||||
image,
|
||||
``
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
return documents;
|
||||
} catch (error) {
|
||||
console.error("Error replacing img paths with absolute paths", error);
|
||||
return documents;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,40 +1,47 @@
|
|||
import * as pdfProcessor from '../pdfProcessor';
|
||||
|
||||
describe('PDF Processing Module - Integration Test', () => {
|
||||
it('should download and read a simple PDF file by URL', async () => {
|
||||
it('should correctly process a simple PDF file without the LLAMAPARSE_API_KEY', async () => {
|
||||
delete process.env.LLAMAPARSE_API_KEY;
|
||||
const pdfContent = await pdfProcessor.fetchAndProcessPdf('https://s3.us-east-1.amazonaws.com/storage.mendable.ai/rafa-testing/test%20%281%29.pdf');
|
||||
expect(pdfContent).toEqual("Dummy PDF file");
|
||||
expect(pdfContent.trim()).toEqual("Dummy PDF file");
|
||||
});
|
||||
|
||||
it('should download and read a complex PDF file by URL', async () => {
|
||||
const pdfContent = await pdfProcessor.fetchAndProcessPdf('https://arxiv.org/pdf/2307.06435.pdf');
|
||||
// We're hitting the LLAMAPARSE rate limit 🫠
|
||||
// it('should download and read a simple PDF file by URL', async () => {
|
||||
// const pdfContent = await pdfProcessor.fetchAndProcessPdf('https://s3.us-east-1.amazonaws.com/storage.mendable.ai/rafa-testing/test%20%281%29.pdf');
|
||||
// expect(pdfContent).toEqual("Dummy PDF file");
|
||||
// });
|
||||
|
||||
const expectedContent = 'A Comprehensive Overview of Large Language Models\n' +
|
||||
' a a,∗ b,∗ c,d,∗ e,f e,f g,i\n' +
|
||||
' Humza Naveed , Asad Ullah Khan , Shi Qiu , Muhammad Saqib , Saeed Anwar , Muhammad Usman , Naveed Akhtar ,\n' +
|
||||
' Nick Barnes h, Ajmal Mian i\n' +
|
||||
' aUniversity of Engineering and Technology (UET), Lahore, Pakistan\n' +
|
||||
' bThe Chinese University of Hong Kong (CUHK), HKSAR, China\n' +
|
||||
' cUniversity of Technology Sydney (UTS), Sydney, Australia\n' +
|
||||
' dCommonwealth Scientific and Industrial Research Organisation (CSIRO), Sydney, Australia\n' +
|
||||
' eKing Fahd University of Petroleum and Minerals (KFUPM), Dhahran, Saudi Arabia\n' +
|
||||
' fSDAIA-KFUPM Joint Research Center for Artificial Intelligence (JRCAI), Dhahran, Saudi Arabia\n' +
|
||||
' gThe University of Melbourne (UoM), Melbourne, Australia\n' +
|
||||
' hAustralian National University (ANU), Canberra, Australia\n' +
|
||||
' iThe University of Western Australia (UWA), Perth, Australia\n' +
|
||||
' Abstract\n' +
|
||||
' Large Language Models (LLMs) have recently demonstrated remarkable capabilities in natural language processing tasks and\n' +
|
||||
' beyond. This success of LLMs has led to a large influx of research contributions in this direction. These works encompass diverse\n' +
|
||||
' topics such as architectural innovations, better training strategies, context length improvements, fine-tuning, multi-modal LLMs,\n' +
|
||||
' robotics, datasets, benchmarking, efficiency, and more. With the rapid development of techniques and regular breakthroughs in\n' +
|
||||
' LLM research, it has become considerably challenging to perceive the bigger picture of the advances in this direction. Considering\n' +
|
||||
' the rapidly emerging plethora of literature on LLMs, it is imperative that the research community is able to benefit from a concise\n' +
|
||||
' yet comprehensive overview of the recent developments in this field. This article provides an overview of the existing literature\n' +
|
||||
' on a broad range of LLM-related concepts. Our self-contained comprehensive overview of LLMs discusses relevant background\n' +
|
||||
' concepts along with covering the advanced topics at the frontier of research in LLMs. This review article is intended to not only\n' +
|
||||
' provide a systematic survey but also a quick comprehensive reference for the researchers and practitioners to draw insights from\n' +
|
||||
' extensive informative summaries of the existing works to advance the LLM research.\n'
|
||||
expect(pdfContent).toContain(expectedContent);
|
||||
}, 60000);
|
||||
// it('should download and read a complex PDF file by URL', async () => {
|
||||
// const pdfContent = await pdfProcessor.fetchAndProcessPdf('https://arxiv.org/pdf/2307.06435.pdf');
|
||||
|
||||
// const expectedContent = 'A Comprehensive Overview of Large Language Models\n' +
|
||||
// ' a a,∗ b,∗ c,d,∗ e,f e,f g,i\n' +
|
||||
// ' Humza Naveed , Asad Ullah Khan , Shi Qiu , Muhammad Saqib , Saeed Anwar , Muhammad Usman , Naveed Akhtar ,\n' +
|
||||
// ' Nick Barnes h, Ajmal Mian i\n' +
|
||||
// ' aUniversity of Engineering and Technology (UET), Lahore, Pakistan\n' +
|
||||
// ' bThe Chinese University of Hong Kong (CUHK), HKSAR, China\n' +
|
||||
// ' cUniversity of Technology Sydney (UTS), Sydney, Australia\n' +
|
||||
// ' dCommonwealth Scientific and Industrial Research Organisation (CSIRO), Sydney, Australia\n' +
|
||||
// ' eKing Fahd University of Petroleum and Minerals (KFUPM), Dhahran, Saudi Arabia\n' +
|
||||
// ' fSDAIA-KFUPM Joint Research Center for Artificial Intelligence (JRCAI), Dhahran, Saudi Arabia\n' +
|
||||
// ' gThe University of Melbourne (UoM), Melbourne, Australia\n' +
|
||||
// ' hAustralian National University (ANU), Canberra, Australia\n' +
|
||||
// ' iThe University of Western Australia (UWA), Perth, Australia\n' +
|
||||
// ' Abstract\n' +
|
||||
// ' Large Language Models (LLMs) have recently demonstrated remarkable capabilities in natural language processing tasks and\n' +
|
||||
// ' beyond. This success of LLMs has led to a large influx of research contributions in this direction. These works encompass diverse\n' +
|
||||
// ' topics such as architectural innovations, better training strategies, context length improvements, fine-tuning, multi-modal LLMs,\n' +
|
||||
// ' robotics, datasets, benchmarking, efficiency, and more. With the rapid development of techniques and regular breakthroughs in\n' +
|
||||
// ' LLM research, it has become considerably challenging to perceive the bigger picture of the advances in this direction. Considering\n' +
|
||||
// ' the rapidly emerging plethora of literature on LLMs, it is imperative that the research community is able to benefit from a concise\n' +
|
||||
// ' yet comprehensive overview of the recent developments in this field. This article provides an overview of the existing literature\n' +
|
||||
// ' on a broad range of LLM-related concepts. Our self-contained comprehensive overview of LLMs discusses relevant background\n' +
|
||||
// ' concepts along with covering the advanced topics at the frontier of research in LLMs. This review article is intended to not only\n' +
|
||||
// ' provide a systematic survey but also a quick comprehensive reference for the researchers and practitioners to draw insights from\n' +
|
||||
// ' extensive informative summaries of the existing works to advance the LLM research.\n'
|
||||
// expect(pdfContent).toContain(expectedContent);
|
||||
// }, 60000);
|
||||
|
||||
});
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
import { Document } from "../../../../lib/entities";
|
||||
import { replacePathsWithAbsolutePaths, replaceImgPathsWithAbsolutePaths } from "../replacePaths";
|
||||
|
||||
describe('replacePaths', () => {
|
||||
describe('replacePathsWithAbsolutePaths', () => {
|
||||
it('should replace relative paths with absolute paths', () => {
|
||||
const documents: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'This is a [link](/path/to/resource) and an image .'
|
||||
}];
|
||||
|
||||
const expectedDocuments: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'This is a [link](https://example.com/path/to/resource) and an image .'
|
||||
}];
|
||||
|
||||
const result = replacePathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(expectedDocuments);
|
||||
});
|
||||
|
||||
it('should not alter absolute URLs', () => {
|
||||
const documents: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'This is an [external link](https://external.com/path) and an image .'
|
||||
}];
|
||||
|
||||
const result = replacePathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(documents); // Expect no change
|
||||
});
|
||||
|
||||
it('should not alter data URLs for images', () => {
|
||||
const documents: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'This is an image: .'
|
||||
}];
|
||||
|
||||
const result = replacePathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(documents); // Expect no change
|
||||
});
|
||||
|
||||
it('should handle multiple links and images correctly', () => {
|
||||
const documents: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'Here are two links: [link1](/path1) and [link2](/path2), and two images:  .'
|
||||
}];
|
||||
|
||||
const expectedDocuments: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'Here are two links: [link1](https://example.com/path1) and [link2](https://example.com/path2), and two images:  .'
|
||||
}];
|
||||
|
||||
const result = replacePathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(expectedDocuments);
|
||||
});
|
||||
|
||||
it('should correctly handle a mix of absolute and relative paths', () => {
|
||||
const documents: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'Mixed paths: [relative](/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).'
|
||||
}];
|
||||
|
||||
const expectedDocuments: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'Mixed paths: [relative](https://example.com/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).'
|
||||
}];
|
||||
|
||||
const result = replacePathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(expectedDocuments);
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
describe('replaceImgPathsWithAbsolutePaths', () => {
|
||||
it('should replace relative image paths with absolute paths', () => {
|
||||
const documents: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'Here is an image: .'
|
||||
}];
|
||||
|
||||
const expectedDocuments: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'Here is an image: .'
|
||||
}];
|
||||
|
||||
const result = replaceImgPathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(expectedDocuments);
|
||||
});
|
||||
|
||||
it('should not alter data:image URLs', () => {
|
||||
const documents: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'An image with a data URL: .'
|
||||
}];
|
||||
|
||||
const result = replaceImgPathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(documents); // Expect no change
|
||||
});
|
||||
|
||||
it('should handle multiple images with a mix of data and relative URLs', () => {
|
||||
const documents: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'Multiple images:   .'
|
||||
}];
|
||||
|
||||
const expectedDocuments: Document[] = [{
|
||||
metadata: { sourceURL: 'https://example.com' },
|
||||
content: 'Multiple images:   .'
|
||||
}];
|
||||
|
||||
const result = replaceImgPathsWithAbsolutePaths(documents);
|
||||
expect(result).toEqual(expectedDocuments);
|
||||
});
|
||||
});
|
||||
});
|
||||
80
apps/api/src/scraper/WebScraper/utils/replacePaths.ts
Normal file
80
apps/api/src/scraper/WebScraper/utils/replacePaths.ts
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
import { Document } from "../../../lib/entities";
|
||||
|
||||
export const replacePathsWithAbsolutePaths = (documents: Document[]): Document[] => {
|
||||
try {
|
||||
documents.forEach((document) => {
|
||||
const baseUrl = new URL(document.metadata.sourceURL).origin;
|
||||
const paths =
|
||||
document.content.match(
|
||||
/(!?\[.*?\])\(((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\)|href="([^"]+)"/g
|
||||
) || [];
|
||||
|
||||
paths.forEach((path: string) => {
|
||||
const isImage = path.startsWith("!");
|
||||
let matchedUrl = path.match(/\(([^)]+)\)/) || path.match(/href="([^"]+)"/);
|
||||
let url = matchedUrl[1];
|
||||
|
||||
if (!url.startsWith("data:") && !url.startsWith("http")) {
|
||||
if (url.startsWith("/")) {
|
||||
url = url.substring(1);
|
||||
}
|
||||
url = new URL(url, baseUrl).toString();
|
||||
}
|
||||
|
||||
const markdownLinkOrImageText = path.match(/(!?\[.*?\])/)[0];
|
||||
if (isImage) {
|
||||
document.content = document.content.replace(
|
||||
path,
|
||||
`${markdownLinkOrImageText}(${url})`
|
||||
);
|
||||
} else {
|
||||
document.content = document.content.replace(
|
||||
path,
|
||||
`${markdownLinkOrImageText}(${url})`
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return documents;
|
||||
} catch (error) {
|
||||
console.error("Error replacing paths with absolute paths", error);
|
||||
return documents;
|
||||
}
|
||||
};
|
||||
|
||||
export const replaceImgPathsWithAbsolutePaths = (documents: Document[]): Document[] => {
|
||||
try {
|
||||
documents.forEach((document) => {
|
||||
const baseUrl = new URL(document.metadata.sourceURL).origin;
|
||||
const images =
|
||||
document.content.match(
|
||||
/!\[.*?\]\(((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\)/g
|
||||
) || [];
|
||||
|
||||
images.forEach((image: string) => {
|
||||
let imageUrl = image.match(/\(([^)]+)\)/)[1];
|
||||
let altText = image.match(/\[(.*?)\]/)[1];
|
||||
|
||||
if (!imageUrl.startsWith("data:image")) {
|
||||
if (!imageUrl.startsWith("http")) {
|
||||
if (imageUrl.startsWith("/")) {
|
||||
imageUrl = imageUrl.substring(1);
|
||||
}
|
||||
imageUrl = new URL(imageUrl, baseUrl).toString();
|
||||
}
|
||||
}
|
||||
|
||||
document.content = document.content.replace(
|
||||
image,
|
||||
``
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
return documents;
|
||||
} catch (error) {
|
||||
console.error("Error replacing img paths with absolute paths", error);
|
||||
return documents;
|
||||
}
|
||||
};
|
||||
|
|
@ -3,7 +3,6 @@ import { getWebScraperQueue } from "./queue-service";
|
|||
import "dotenv/config";
|
||||
import { logtail } from "./logtail";
|
||||
import { startWebScraperPipeline } from "../main/runWebScraper";
|
||||
import { WebScraperDataProvider } from "../scraper/WebScraper";
|
||||
import { callWebhook } from "./webhook";
|
||||
|
||||
getWebScraperQueue().process(
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user