Update application files and fix LLM models, create new retriever class

This commit is contained in:
Alex committed 2024-04-09 14:02:33 +01:00
1 parent e07df29ab9
commit 391f686173
13 files changed
+202 -185

No files matched your search

+6
View File
@@ -0,0 +1,6 @@
from transformers import GPT2TokenizerFast
def count_tokens(string):
tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
return len(tokenizer(string)['input_ids'])