Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: Replace estimateToken with a mock implementation

Browse Source
This commit is contained in:
Harsh Gupta (aider) 2024-08-14 13:45:02 +05:30
parent aeb6ebed67
commit bf27d39f1b
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/functions/src/cloud-functions/crawler.ts
View File

@ -17,7 +17,8 @@ import { Crawled } from '../db/crawled';
import { cleanAttribute } from '../utils/misc'; import { cleanAttribute } from '../utils/misc';
import { randomUUID } from 'crypto'; import { randomUUID } from 'crypto';
import { countGPTToken as estimateToken } from '../shared/utils/openai'; // Mock implementation of estimateToken
const estimateToken = (text: string) => Math.ceil(text.length / 4);
import { CrawlerOptions, CrawlerOptionsHeaderOnly } from '../dto/scrapping-options'; import { CrawlerOptions, CrawlerOptionsHeaderOnly } from '../dto/scrapping-options';
import { PDFExtractor } from '../services/pdf-extract'; import { PDFExtractor } from '../services/pdf-extract';