Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 03:32:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

strip more stuff

Browse Source
This commit is contained in:
Harsh Gupta 2024-08-14 13:47:25 +05:30
parent bf27d39f1b
commit d380599986
2 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,3 +1,4 @@
node_modules/ node_modules/
.DS_Store .DS_Store
/package-lock.json /package-lock.json
.aider*

4
backend/functions/src/cloud-functions/crawler.ts
View File

@ -6,7 +6,7 @@ import {
} from 'civkit'; } from 'civkit';
import { singleton } from 'tsyringe'; import { singleton } from 'tsyringe';
import { AsyncContext, CloudHTTPv2, Ctx, FirebaseStorageBucketControl, Logger, OutputServerEventStream, RPCReflect } from '../shared'; import { AsyncContext, CloudHTTPv2, FirebaseStorageBucketControl, Logger, OutputServerEventStream, RPCReflect } from '../shared';
import _ from 'lodash'; import _ from 'lodash';
import { PageSnapshot, PuppeteerControl, ScrappingOptions } from '../services/puppeteer'; import { PageSnapshot, PuppeteerControl, ScrappingOptions } from '../services/puppeteer';
import { Request, Response } from 'express'; import { Request, Response } from 'express';
@ -23,7 +23,6 @@ const estimateToken = (text: string) => Math.ceil(text.length / 4);
import { CrawlerOptions, CrawlerOptionsHeaderOnly } from '../dto/scrapping-options'; import { CrawlerOptions, CrawlerOptionsHeaderOnly } from '../dto/scrapping-options';
import { PDFExtractor } from '../services/pdf-extract'; import { PDFExtractor } from '../services/pdf-extract';
import { DomainBlockade } from '../db/domain-blockade'; import { DomainBlockade } from '../db/domain-blockade';
import { FirebaseRoundTripChecker } from '../shared/services/firebase-roundtrip-checker';
import { JSDomControl } from '../services/jsdom'; import { JSDomControl } from '../services/jsdom';
const md5Hasher = new HashManager('md5', 'hex'); const md5Hasher = new HashManager('md5', 'hex');
@ -82,7 +81,6 @@ export class CrawlerHost extends RPCHost {
protected pdfExtractor: PDFExtractor, protected pdfExtractor: PDFExtractor,
protected firebaseObjectStorage: FirebaseStorageBucketControl, protected firebaseObjectStorage: FirebaseStorageBucketControl,
protected threadLocal: AsyncContext, protected threadLocal: AsyncContext,
protected fbHealthCheck: FirebaseRoundTripChecker,
) { ) {
super(...arguments); super(...arguments);