mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-15 16:14:16 +00:00
Update application files and fix LLM models, create new retriever class
This commit is contained in:
1 parent
e07df29ab9
commit
391f686173
13 files changed
+202
-185
No files matched your search
@@ -0,0 +1,6 @@
|
||||
from transformers import GPT2TokenizerFast
|
||||
|
||||
|
||||
def count_tokens(string):
|
||||
tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
|
||||
return len(tokenizer(string)['input_ids'])
|
||||
Reference in new issue
Block a user