Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: search with failed pages

Browse Source
This commit is contained in:
Yanlong Wang 2024-08-08 15:49:23 +08:00
parent 0dd05b5dab
commit e529369ba6
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
backend/functions/src/cloud-functions/crawler.ts
View File

@ -1036,14 +1036,18 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
let concluded = false;
const handler = async (it: AsyncGenerator<PageSnapshot | undefined>, idx: number) => {
for await (const x of it) {
results[idx] = x;
try {
for await (const x of it) {
results[idx] = x;
if (x) {
nextDeferred.resolve();
nextDeferred = Defer();
}
if (x) {
nextDeferred.resolve();
nextDeferred = Defer();
}
} catch (err: any) {
this.logger.warn(`Failed to scrap ${urls[idx]}`, { err: marshalErrorLike(err) });
}
};