Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #793 from mendableai/fix/issue-665

Browse Source 
[Bug] encoding error for special token
This commit is contained in:
Nicolas 2024-10-21 12:24:46 -03:00 committed by GitHub
commit d31b85fa91
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/lib/LLM-extraction/helpers.ts
View File

@ -6,7 +6,13 @@ export function numTokensFromString(message: string, model: string): number {
const encoder = encoding_for_model(model as TiktokenModel);
// Encode the message into tokens
const tokens = encoder.encode(message);
let tokens: Uint32Array;
try {
tokens = encoder.encode(message);
} catch (error) {
message = message.replace("<|endoftext|>", "");
tokens = encoder.encode(message);
}
// Free the encoder resources after use
encoder.free();