Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fixes

Browse Source
This commit is contained in:
Harsh Gupta 2024-08-14 14:41:02 +05:30
parent e9ac98a628
commit db6cd7d76c
3 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/functions/src/cloud-functions/crawler.ts
View File

@ -567,7 +567,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
} }
async crawl(req: Request, res: Response) { async crawl(req: Request, res: Response) {
const rpcReflect: RPCReflection = {}; // const rpcReflect: RPCReflection = {};
const ctx = { req, res }; const ctx = { req, res };
const crawlerOptionsHeaderOnly = CrawlerOptionsHeaderOnly.from(req.headers); const crawlerOptionsHeaderOnly = CrawlerOptionsHeaderOnly.from(req.headers);
const crawlerOptionsParamsAllowed = CrawlerOptions.from(req.method === 'POST' ? req.body : req.query); const crawlerOptionsParamsAllowed = CrawlerOptions.from(req.method === 'POST' ? req.body : req.query);
@ -619,7 +619,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
if (!ctx.req.accepts('text/plain') && ctx.req.accepts('text/event-stream')) { if (!ctx.req.accepts('text/plain') && ctx.req.accepts('text/event-stream')) {
const sseStream = new OutputServerEventStream(); const sseStream = new OutputServerEventStream();
rpcReflect.return(sseStream); // rpcReflect.return(sseStream);
try { try {
for await (const scrapped of this.scrap(urlToCrawl, crawlOpts, crawlerOptions)) { for await (const scrapped of this.scrap(urlToCrawl, crawlOpts, crawlerOptions)) {

4
backend/functions/src/services/puppeteer.ts
View File

@ -2,14 +2,14 @@ import os from 'os';
import fs from 'fs'; import fs from 'fs';
import { container, singleton } from 'tsyringe'; import { container, singleton } from 'tsyringe';
import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency } from 'civkit'; import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency } from 'civkit';
import { Logger } from '../shared/services/logger'; import { Logger } from '../shared/logger';
import type { Browser, CookieParam, Page } from 'puppeteer'; import type { Browser, CookieParam, Page } from 'puppeteer';
import puppeteer from 'puppeteer-extra'; import puppeteer from 'puppeteer-extra';
import puppeteerBlockResources from 'puppeteer-extra-plugin-block-resources'; import puppeteerBlockResources from 'puppeteer-extra-plugin-block-resources';
import puppeteerPageProxy from 'puppeteer-extra-plugin-page-proxy'; import puppeteerPageProxy from 'puppeteer-extra-plugin-page-proxy';
import { SecurityCompromiseError, ServiceCrashedError } from '../shared/lib/errors'; import { SecurityCompromiseError, ServiceCrashedError } from '../shared/errors';
import { TimeoutError } from 'puppeteer'; import { TimeoutError } from 'puppeteer';
const tldExtract = require('tld-extract'); const tldExtract = require('tld-extract');

1
backend/functions/tsconfig.json
View File

@ -15,6 +15,7 @@
"experimentalDecorators": true, "experimentalDecorators": true,
"emitDecoratorMetadata": true, "emitDecoratorMetadata": true,
"esModuleInterop": true, "esModuleInterop": true,
"noImplicitAny": false,
"noImplicitOverride": true, "noImplicitOverride": true,
}, },
"compileOnSave": true, "compileOnSave": true,