feat: template-based prompt rendering with dynamic namespace injection (#2091)

* feat: template-based prompt rendering with dynamic namespace injection

* refactor: improve template engine initialization with clearer formatting

* refactor: streamline ReActAgent methods and improve content extraction logic

feat: enhance error handling in NamespaceManager and TemplateEngine

fix: update NewAgent component to ensure consistent form data submission

test: modify tests for ReActAgent and prompt renderer to reflect method changes and improve coverage

* feat: tools namespace + three-tier token budget

* refactor: remove unused variable assignment in message building tests

* Enhance prompt customization and tool pre-fetching functionality

* ruff lint fix

* refactor: cleaner error handling and reduce code clutter

---------

Co-authored-by: Alex <a@tushynski.me>

application/utils.py

@@ -74,6 +74,17 @@ def count_tokens_docs(docs):
     return tokens
 
 
+def calculate_doc_token_budget(
+    gpt_model: str = "gpt-4o", history_token_limit: int = 2000
+) -> int:
+    total_context = settings.LLM_TOKEN_LIMITS.get(
+        gpt_model, settings.DEFAULT_LLM_TOKEN_LIMIT
+    )
+    reserved = sum(settings.RESERVED_TOKENS.values())
+    doc_budget = total_context - history_token_limit - reserved
+    return max(doc_budget, 1000)
+
+
 def get_missing_fields(data, required_fields):
     """Check for missing required fields. Returns list of missing field names."""
     return [field for field in required_fields if field not in data]
@@ -141,8 +152,8 @@ def limit_chat_history(history, max_token_limit=None, gpt_model="docsgpt"):
         max_token_limit
         if max_token_limit
         and max_token_limit
-        < settings.LLM_TOKEN_LIMITS.get(gpt_model, settings.DEFAULT_MAX_HISTORY)
-        else settings.LLM_TOKEN_LIMITS.get(gpt_model, settings.DEFAULT_MAX_HISTORY)
+        < settings.LLM_TOKEN_LIMITS.get(gpt_model, settings.DEFAULT_LLM_TOKEN_LIMIT)
+        else settings.LLM_TOKEN_LIMITS.get(gpt_model, settings.DEFAULT_LLM_TOKEN_LIMIT)
     )
 
     if not history: