Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: detect when readability does not work

Browse Source
This commit is contained in:
yanlong.wang 2024-06-20 18:20:13 +08:00
parent eaa06781e3
commit 579f259cb9
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
backend/functions/src/cloud-functions/crawler.ts
View File

@ -327,8 +327,19 @@ export class CrawlerHost extends RPCHost {
break;
}
const toBeTurnedToMd = mode === 'markdown' ? snapshot.html : snapshot.parsed?.content;
let turnDownService = mode === 'markdown' ? this.getTurndown({ url: snapshot.href }) : this.getTurndown({ noRules: true, url: snapshot.href });
let toBeTurnedToMd = snapshot.html;
let turnDownService = this.getTurndown({ url: nominalUrl });
if (mode !== 'markdown' && snapshot.parsed?.content) {
const par1 = turnDownService.turndown(toBeTurnedToMd);
const par2 = turnDownService.turndown(snapshot.parsed.content)
// If Readability did its job
if (par2.length >= 0.3 * par1.length) {
turnDownService = this.getTurndown({ noRules: true, url: snapshot.href });
toBeTurnedToMd = snapshot.parsed.content;
}
}
for (const plugin of this.turnDownPlugins) {
turnDownService = turnDownService.use(plugin);
}