Merge pull request #295 from larinam/min_tokens_logic_fix

Fix min_tokens logic for grouping documents

application/parser/token_func.py

@@ -25,7 +25,7 @@ def group_documents(documents: List[Document], min_tokens: int, max_tokens: int)
             current_group = Document(text=doc.text, doc_id=doc.doc_id, embedding=doc.embedding,
                                      extra_info=doc.extra_info)
         elif len(tiktoken.get_encoding("cl100k_base").encode(
-                current_group.text)) + doc_len < max_tokens and doc_len >= min_tokens:
+                current_group.text)) + doc_len < max_tokens and doc_len < min_tokens:
             current_group.text += " " + doc.text
         else:
             docs.append(current_group)

scripts/parser/token_func.py

@@ -24,7 +24,7 @@ def group_documents(documents: List[Document], min_tokens: int, max_tokens: int)
             current_group = Document(text=doc.text, doc_id=doc.doc_id, embedding=doc.embedding,
                                      extra_info=doc.extra_info)
         elif len(tiktoken.get_encoding("cl100k_base").encode(
-                current_group.text)) + doc_len < max_tokens and doc_len >= min_tokens:
+                current_group.text)) + doc_len < max_tokens and doc_len < min_tokens:
             current_group.text += " " + doc.text
         else:
             docs.append(current_group)