Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: detect when readability does not work

Browse Source
This commit is contained in:
yanlong.wang 2024-06-20 18:20:13 +08:00
parent eaa06781e3
commit 579f259cb9
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
backend/functions/src/cloud-functions/crawler.ts
View File

@ -327,8 +327,19 @@ export class CrawlerHost extends RPCHost {
break;
}
const toBeTurnedToMd = mode === 'markdown' ? snapshot.html : snapshot.parsed?.content;
let turnDownService = mode === 'markdown' ? this.getTurndown({ url: snapshot.href }) : this.getTurndown({ noRules: true, url: snapshot.href });
let toBeTurnedToMd = snapshot.html;
let turnDownService = this.getTurndown({ url: nominalUrl });
if (mode !== 'markdown' && snapshot.parsed?.content) {
const par1 = turnDownService.turndown(toBeTurnedToMd);
const par2 = turnDownService.turndown(snapshot.parsed.content)
// If Readability did its job
if (par2.length >= 0.3 * par1.length) {
turnDownService = this.getTurndown({ noRules: true, url: snapshot.href });
toBeTurnedToMd = snapshot.parsed.content;
}
}
for (const plugin of this.turnDownPlugins) {
turnDownService = turnDownService.use(plugin);
}
@ -585,7 +596,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
let urlToCrawl;
const normalizeUrl = (await pNormalizeUrl).default;
try {
urlToCrawl = new URL(normalizeUrl(noSlashURL.trim(), { stripWWW: false, removeTrailingSlash: false, removeSingleSlash: false, sortQueryParameters:false }));
urlToCrawl = new URL(normalizeUrl(noSlashURL.trim(), { stripWWW: false, removeTrailingSlash: false, removeSingleSlash: false, sortQueryParameters: false }));
} catch (err) {
throw new ParamValidationError({
message: `${err}`,