Merge branch 'main' into feat/standardize-css

.env-template

@@ -3,6 +3,14 @@ LLM_NAME=docsgpt
 VITE_API_STREAMING=true
 INTERNAL_KEY=<internal key for worker-to-backend authentication>
 
+# Provider-specific API keys (optional - use these to enable multiple providers)
+# OPENAI_API_KEY=<your-openai-api-key>
+# ANTHROPIC_API_KEY=<your-anthropic-api-key>
+# GOOGLE_API_KEY=<your-google-api-key>
+# GROQ_API_KEY=<your-groq-api-key>
+# NOVITA_API_KEY=<your-novita-api-key>
+# OPEN_ROUTER_API_KEY=<your-openrouter-api-key>
+
 # Remote Embeddings (Optional - for using a remote embeddings API instead of local SentenceTransformer)
 # When set, the app will use the remote API and won't load SentenceTransformer (saves RAM)
 EMBEDDINGS_BASE_URL=

.github/workflows/npm-publish.yml

@@ -0,0 +1,114 @@
+name: Publish npm libraries
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: >
+          Version bump type (patch | minor | major) or explicit semver (e.g. 1.2.3).
+          Applies to both docsgpt and docsgpt-react.
+        required: true
+        default: patch
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: npm-release
+    defaults:
+      run:
+        working-directory: extensions/react-widget
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          registry-url: https://registry.npmjs.org
+
+      - name: Install dependencies
+        run: npm ci
+
+      # ── docsgpt (HTML embedding bundle) ──────────────────────────────────
+      # Uses the `build` script (parcel build src/browser.tsx) and keeps
+      # the `targets` field so Parcel produces browser-optimised bundles.
+
+      - name: Set package name → docsgpt
+        run: jq --arg n "docsgpt" '.name=$n' package.json > _tmp.json && mv _tmp.json package.json
+
+      - name: Bump version (docsgpt)
+        id: version_docsgpt
+        run: |
+          VERSION="${{ github.event.inputs.version }}"
+          NEW_VER=$(npm version "${VERSION:-patch}" --no-git-tag-version)
+          echo "version=${NEW_VER#v}" >> "$GITHUB_OUTPUT"
+
+      - name: Build docsgpt
+        run: npm run build
+
+      - name: Publish docsgpt
+        run: npm publish --verbose
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      # ── docsgpt-react (React library bundle) ─────────────────────────────
+      # Uses `build:react` script (parcel build src/index.ts) and strips
+      # the `targets` field so Parcel treats the output as a plain library
+      # without browser-specific target resolution, producing a smaller bundle.
+
+      - name: Reset package.json from source control
+        run: git checkout -- package.json
+
+      - name: Set package name → docsgpt-react
+        run: jq --arg n "docsgpt-react" '.name=$n' package.json > _tmp.json && mv _tmp.json package.json
+
+      - name: Remove targets field (react library build)
+        run: jq 'del(.targets)' package.json > _tmp.json && mv _tmp.json package.json
+
+      - name: Bump version (docsgpt-react) to match docsgpt
+        run: npm version "${{ steps.version_docsgpt.outputs.version }}" --no-git-tag-version
+
+      - name: Clean dist before react build
+        run: rm -rf dist
+
+      - name: Build docsgpt-react
+        run: npm run build:react
+
+      - name: Publish docsgpt-react
+        run: npm publish --verbose
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      # ── Commit the bumped version back to the repository ─────────────────
+
+      - name: Reset package.json and write final version
+        run: |
+          git checkout -- package.json
+          jq --arg v "${{ steps.version_docsgpt.outputs.version }}" '.version=$v' \
+            package.json > _tmp.json && mv _tmp.json package.json
+          npm install --package-lock-only
+
+      - name: Commit version bump and create PR
+        run: |
+          git config user.name  "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          BRANCH="chore/bump-npm-v${{ steps.version_docsgpt.outputs.version }}"
+          git checkout -b "$BRANCH"
+          git add package.json package-lock.json
+          git commit -m "chore: bump npm libraries to v${{ steps.version_docsgpt.outputs.version }}"
+          git push origin "$BRANCH"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create PR
+        run: |
+          gh pr create \
+            --title "chore: bump npm libraries to v${{ steps.version_docsgpt.outputs.version }}" \
+            --body "Automated version bump after npm publish." \
+            --base main
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/react-widget-build.yml

@@ -0,0 +1,34 @@
+name: React Widget Build
+
+on:
+  push:
+    paths:
+      - 'extensions/react-widget/**'
+  pull_request:
+    paths:
+      - 'extensions/react-widget/**'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: extensions/react-widget
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: extensions/react-widget/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build
+        run: npm run build

.gitignore

@@ -2,6 +2,7 @@
 __pycache__/
 *.py[cod]
 *$py.class
+results.txt
 experiments/
 
 experiments

README.md

@@ -29,7 +29,7 @@
 
 <div align="center">
   <br>
-<img src="https://d3dg1063dc54p9.cloudfront.net/videos/demov7.gif" alt="video-example-of-docs-gpt" width="800" height="450">
+<img src="https://d3dg1063dc54p9.cloudfront.net/videos/demo-26.gif" alt="video-example-of-docs-gpt" width="800" height="480">
 </div>
 <h3 align="left">
   <strong>Key Features:</strong>

application/agents/agent_creator.py

@@ -1,7 +1,8 @@
 import logging
 
+from application.agents.agentic_agent import AgenticAgent
 from application.agents.classic_agent import ClassicAgent
-from application.agents.react_agent import ReActAgent
+from application.agents.research_agent import ResearchAgent
 from application.agents.workflow_agent import WorkflowAgent
 
 logger = logging.getLogger(__name__)
@@ -10,7 +11,9 @@ logger = logging.getLogger(__name__)
 class AgentCreator:
     agents = {
         "classic": ClassicAgent,
-        "react": ReActAgent,
+        "react": ClassicAgent,  # backwards compat: react falls back to classic
+        "agentic": AgenticAgent,
+        "research": ResearchAgent,
         "workflow": WorkflowAgent,
     }
 

application/agents/agentic_agent.py

@@ -0,0 +1,63 @@
+import logging
+from typing import Dict, Generator, Optional
+
+from application.agents.base import BaseAgent
+from application.agents.tools.internal_search import (
+    INTERNAL_TOOL_ID,
+    add_internal_search_tool,
+)
+from application.logging import LogContext
+
+logger = logging.getLogger(__name__)
+
+
+class AgenticAgent(BaseAgent):
+    """Agent where the LLM controls retrieval via tools.
+
+    Unlike ClassicAgent which pre-fetches docs into the prompt,
+    AgenticAgent gives the LLM an internal_search tool so it can
+    decide when, what, and whether to search.
+    """
+
+    def __init__(
+        self,
+        retriever_config: Optional[Dict] = None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.retriever_config = retriever_config or {}
+
+    def _gen_inner(
+        self, query: str, log_context: LogContext
+    ) -> Generator[Dict, None, None]:
+        tools_dict = self.tool_executor.get_tools()
+        add_internal_search_tool(tools_dict, self.retriever_config)
+        self._prepare_tools(tools_dict)
+
+        # 4. Build messages (prompt has NO pre-fetched docs)
+        messages = self._build_messages(self.prompt, query)
+
+        # 5. Call LLM — the handler manages the tool loop
+        llm_response = self._llm_gen(messages, log_context)
+
+        yield from self._handle_response(
+            llm_response, tools_dict, messages, log_context
+        )
+
+        # 6. Collect sources from internal search tool results
+        self._collect_internal_sources()
+
+        yield {"sources": self.retrieved_docs}
+        yield {"tool_calls": self._get_truncated_tool_calls()}
+
+        log_context.stacks.append(
+            {"component": "agent", "data": {"tool_calls": self.tool_calls.copy()}}
+        )
+
+    def _collect_internal_sources(self):
+        """Collect retrieved docs from the cached InternalSearchTool instance."""
+        cache_key = f"internal_search:{INTERNAL_TOOL_ID}:{self.user or ''}"
+        tool = self.tool_executor._loaded_tools.get(cache_key)
+        if tool and hasattr(tool, "retrieved_docs") and tool.retrieved_docs:
+            self.retrieved_docs = tool.retrieved_docs

application/agents/base.py

@@ -3,20 +3,15 @@ import uuid
 from abc import ABC, abstractmethod
 from typing import Dict, Generator, List, Optional
 
-from bson.objectid import ObjectId
-
-from application.agents.tools.tool_action_parser import ToolActionParser
-from application.agents.tools.tool_manager import ToolManager
+from application.agents.tool_executor import ToolExecutor
 from application.core.json_schema_utils import (
     JsonSchemaValidationError,
     normalize_json_schema_payload,
 )
-from application.core.mongo_db import MongoDB
 from application.core.settings import settings
 from application.llm.handlers.handler_creator import LLMHandlerCreator
 from application.llm.llm_creator import LLMCreator
 from application.logging import build_stack_data, log_activity, LogContext
-from application.security.encryption import decrypt_credentials
 
 logger = logging.getLogger(__name__)
 
@@ -41,6 +36,10 @@ class BaseAgent(ABC):
         limited_request_mode: Optional[bool] = False,
         request_limit: Optional[int] = settings.DEFAULT_AGENT_LIMITS["request_limit"],
         compressed_summary: Optional[str] = None,
+        llm=None,
+        llm_handler=None,
+        tool_executor: Optional[ToolExecutor] = None,
+        backup_models: Optional[List[str]] = None,
     ):
         self.endpoint = endpoint
         self.llm_name = llm_name
@@ -51,22 +50,42 @@ class BaseAgent(ABC):
         self.prompt = prompt
         self.decoded_token = decoded_token or {}
         self.user: str = self.decoded_token.get("sub")
-        self.tool_config: Dict = {}
         self.tools: List[Dict] = []
-        self.tool_calls: List[Dict] = []
         self.chat_history: List[Dict] = chat_history if chat_history is not None else []
-        self.llm = LLMCreator.create_llm(
-            llm_name,
-            api_key=api_key,
-            user_api_key=user_api_key,
-            decoded_token=decoded_token,
-            model_id=model_id,
-            agent_id=agent_id,
-        )
+
+        # Dependency injection for LLM — fall back to creating if not provided
+        if llm is not None:
+            self.llm = llm
+        else:
+            self.llm = LLMCreator.create_llm(
+                llm_name,
+                api_key=api_key,
+                user_api_key=user_api_key,
+                decoded_token=decoded_token,
+                model_id=model_id,
+                agent_id=agent_id,
+                backup_models=backup_models,
+            )
+
         self.retrieved_docs = retrieved_docs or []
-        self.llm_handler = LLMHandlerCreator.create_handler(
-            llm_name if llm_name else "default"
-        )
+
+        if llm_handler is not None:
+            self.llm_handler = llm_handler
+        else:
+            self.llm_handler = LLMHandlerCreator.create_handler(
+                llm_name if llm_name else "default"
+            )
+
+        # Tool executor — injected or created
+        if tool_executor is not None:
+            self.tool_executor = tool_executor
+        else:
+            self.tool_executor = ToolExecutor(
+                user_api_key=user_api_key,
+                user=self.user,
+                decoded_token=decoded_token,
+            )
+
         self.attachments = attachments or []
         self.json_schema = None
         if json_schema is not None:
@@ -94,333 +113,72 @@ class BaseAgent(ABC):
     ) -> Generator[Dict, None, None]:
         pass
 
+    # ---- Tool delegation (thin wrappers around ToolExecutor) ----
+
+    @property
+    def tool_calls(self) -> List[Dict]:
+        return self.tool_executor.tool_calls
+
+    @tool_calls.setter
+    def tool_calls(self, value: List[Dict]):
+        self.tool_executor.tool_calls = value
+
     def _get_tools(self, api_key: str = None) -> Dict[str, Dict]:
-        mongo = MongoDB.get_client()
-        db = mongo[settings.MONGO_DB_NAME]
-        agents_collection = db["agents"]
-        tools_collection = db["user_tools"]
-
-        agent_data = agents_collection.find_one({"key": api_key or self.user_api_key})
-        tool_ids = agent_data.get("tools", []) if agent_data else []
-
-        tools = (
-            tools_collection.find(
-                {"_id": {"$in": [ObjectId(tool_id) for tool_id in tool_ids]}}
-            )
-            if tool_ids
-            else []
-        )
-        tools = list(tools)
-        tools_by_id = {str(tool["_id"]): tool for tool in tools} if tools else {}
-
-        return tools_by_id
+        return self.tool_executor._get_tools_by_api_key(api_key or self.user_api_key)
 
     def _get_user_tools(self, user="local"):
-        mongo = MongoDB.get_client()
-        db = mongo[settings.MONGO_DB_NAME]
-        user_tools_collection = db["user_tools"]
-        user_tools = user_tools_collection.find({"user": user, "status": True})
-        user_tools = list(user_tools)
-
-        return {str(i): tool for i, tool in enumerate(user_tools)}
+        return self.tool_executor._get_user_tools(user)
 
     def _build_tool_parameters(self, action):
-        params = {"type": "object", "properties": {}, "required": []}
-        for param_type in ["query_params", "headers", "body", "parameters"]:
-            if param_type in action and action[param_type].get("properties"):
-                for k, v in action[param_type]["properties"].items():
-                    if v.get("filled_by_llm", True):
-                        params["properties"][k] = {
-                            key: value
-                            for key, value in v.items()
-                            if key not in ("filled_by_llm", "value", "required")
-                        }
-                        if v.get("required", False):
-                            params["required"].append(k)
-        return params
+        return self.tool_executor._build_tool_parameters(action)
 
     def _prepare_tools(self, tools_dict):
-        self.tools = [
-            {
-                "type": "function",
-                "function": {
-                    "name": f"{action['name']}_{tool_id}",
-                    "description": action["description"],
-                    "parameters": self._build_tool_parameters(action),
-                },
-            }
-            for tool_id, tool in tools_dict.items()
-            if (
-                (tool["name"] == "api_tool" and "actions" in tool.get("config", {}))
-                or (tool["name"] != "api_tool" and "actions" in tool)
-            )
-            for action in (
-                tool["config"]["actions"].values()
-                if tool["name"] == "api_tool"
-                else tool["actions"]
-            )
-            if action.get("active", True)
-        ]
+        self.tools = self.tool_executor.prepare_tools_for_llm(tools_dict)
 
     def _execute_tool_action(self, tools_dict, call):
-        parser = ToolActionParser(self.llm.__class__.__name__)
-        tool_id, action_name, call_args = parser.parse_args(call)
-
-        call_id = getattr(call, "id", None) or str(uuid.uuid4())
-
-        # Check if parsing failed
-
-        if tool_id is None or action_name is None:
-            error_message = f"Error: Failed to parse LLM tool call. Tool name: {getattr(call, 'name', 'unknown')}"
-            logger.error(error_message)
-
-            tool_call_data = {
-                "tool_name": "unknown",
-                "call_id": call_id,
-                "action_name": getattr(call, "name", "unknown"),
-                "arguments": call_args or {},
-                "result": f"Failed to parse tool call. Invalid tool name format: {getattr(call, 'name', 'unknown')}",
-            }
-            yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
-            self.tool_calls.append(tool_call_data)
-            return "Failed to parse tool call.", call_id
-        # Check if tool_id exists in available tools
-
-        if tool_id not in tools_dict:
-            error_message = f"Error: Tool ID '{tool_id}' extracted from LLM call not found in available tools_dict. Available IDs: {list(tools_dict.keys())}"
-            logger.error(error_message)
-
-            # Return error result
-
-            tool_call_data = {
-                "tool_name": "unknown",
-                "call_id": call_id,
-                "action_name": f"{action_name}_{tool_id}",
-                "arguments": call_args,
-                "result": f"Tool with ID {tool_id} not found. Available tools: {list(tools_dict.keys())}",
-            }
-            yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
-            self.tool_calls.append(tool_call_data)
-            return f"Tool with ID {tool_id} not found.", call_id
-        tool_call_data = {
-            "tool_name": tools_dict[tool_id]["name"],
-            "call_id": call_id,
-            "action_name": f"{action_name}_{tool_id}",
-            "arguments": call_args,
-        }
-        yield {"type": "tool_call", "data": {**tool_call_data, "status": "pending"}}
-
-        tool_data = tools_dict[tool_id]
-        action_data = (
-            tool_data["config"]["actions"][action_name]
-            if tool_data["name"] == "api_tool"
-            else next(
-                action
-                for action in tool_data["actions"]
-                if action["name"] == action_name
-            )
+        return self.tool_executor.execute(
+            tools_dict, call, self.llm.__class__.__name__
         )
 
-        query_params, headers, body, parameters = {}, {}, {}, {}
-        param_types = {
-            "query_params": query_params,
-            "headers": headers,
-            "body": body,
-            "parameters": parameters,
-        }
-
-        for param_type, target_dict in param_types.items():
-            if param_type in action_data and action_data[param_type].get("properties"):
-                for param, details in action_data[param_type]["properties"].items():
-                    if (
-                        param not in call_args
-                        and "value" in details
-                        and details["value"]
-                    ):
-                        target_dict[param] = details["value"]
-        for param, value in call_args.items():
-            for param_type, target_dict in param_types.items():
-                if param_type in action_data and param in action_data[param_type].get(
-                    "properties", {}
-                ):
-                    target_dict[param] = value
-        tm = ToolManager(config={})
-
-        # Prepare tool_config and add tool_id for memory tools
-
-        if tool_data["name"] == "api_tool":
-            action_config = tool_data["config"]["actions"][action_name]
-            tool_config = {
-                "url": action_config["url"],
-                "method": action_config["method"],
-                "headers": headers,
-                "query_params": query_params,
-            }
-            if "body_content_type" in action_config:
-                tool_config["body_content_type"] = action_config.get(
-                    "body_content_type", "application/json"
-                )
-                tool_config["body_encoding_rules"] = action_config.get(
-                    "body_encoding_rules", {}
-                )
-        else:
-            tool_config = tool_data["config"].copy() if tool_data["config"] else {}
-            if tool_config.get("encrypted_credentials") and self.user:
-                decrypted = decrypt_credentials(
-                    tool_config["encrypted_credentials"], self.user
-                )
-                tool_config.update(decrypted)
-                tool_config["auth_credentials"] = decrypted
-                tool_config.pop("encrypted_credentials", None)
-            tool_config["tool_id"] = str(tool_data.get("_id", tool_id))
-            if hasattr(self, "conversation_id") and self.conversation_id:
-                tool_config["conversation_id"] = self.conversation_id
-            if tool_data["name"] == "mcp_tool":
-                tool_config["query_mode"] = True
-        tool = tm.load_tool(
-            tool_data["name"],
-            tool_config=tool_config,
-            user_id=self.user,
-        )
-        resolved_arguments = (
-            {"query_params": query_params, "headers": headers, "body": body}
-            if tool_data["name"] == "api_tool"
-            else parameters
-        )
-        if tool_data["name"] == "api_tool":
-            logger.debug(
-                f"Executing api: {action_name} with query_params: {query_params}, headers: {headers}, body: {body}"
-            )
-            result = tool.execute_action(action_name, **body)
-        else:
-            logger.debug(f"Executing tool: {action_name} with args: {call_args}")
-            result = tool.execute_action(action_name, **parameters)
-
-        get_artifact_id = (
-            getattr(tool, "get_artifact_id", None)
-            if tool_data["name"] != "api_tool"
-            else None
-        )
-
-        artifact_id = None
-        if callable(get_artifact_id):
-            try:
-                artifact_id = get_artifact_id(action_name, **parameters)
-            except Exception:
-                logger.exception(
-                    "Failed to extract artifact_id from tool %s for action %s",
-                    tool_data["name"],
-                    action_name,
-                )
-
-        artifact_id = str(artifact_id).strip() if artifact_id is not None else ""
-        if artifact_id:
-            tool_call_data["artifact_id"] = artifact_id
-        result_full = str(result)
-        tool_call_data["resolved_arguments"] = resolved_arguments
-        tool_call_data["result_full"] = result_full
-        tool_call_data["result"] = (
-            f"{result_full[:50]}..." if len(result_full) > 50 else result_full
-        )
-
-        stream_tool_call_data = {
-            key: value
-            for key, value in tool_call_data.items()
-            if key not in {"result_full", "resolved_arguments"}
-        }
-        yield {"type": "tool_call", "data": {**stream_tool_call_data, "status": "completed"}}
-        self.tool_calls.append(tool_call_data)
-
-        return result, call_id
-
     def _get_truncated_tool_calls(self):
-        return [
-            {
-                "tool_name": tool_call.get("tool_name"),
-                "call_id": tool_call.get("call_id"),
-                "action_name": tool_call.get("action_name"),
-                "arguments": tool_call.get("arguments"),
-                "artifact_id": tool_call.get("artifact_id"),
-                "result": (
-                    f"{str(tool_call['result'])[:50]}..."
-                    if len(str(tool_call["result"])) > 50
-                    else tool_call["result"]
-                ),
-                "status": "completed",
-            }
-            for tool_call in self.tool_calls
-        ]
+        return self.tool_executor.get_truncated_tool_calls()
+
+    # ---- Context / token management ----
 
     def _calculate_current_context_tokens(self, messages: List[Dict]) -> int:
-        """
-        Calculate total tokens in current context (messages).
-
-        Args:
-            messages: List of message dicts
-
-        Returns:
-            Total token count
-        """
         from application.api.answer.services.compression.token_counter import (
             TokenCounter,
         )
-
         return TokenCounter.count_message_tokens(messages)
 
     def _check_context_limit(self, messages: List[Dict]) -> bool:
-        """
-        Check if we're approaching context limit (80%).
-
-        Args:
-            messages: Current message list
-
-        Returns:
-            True if at or above 80% of context limit
-        """
         from application.core.model_utils import get_token_limit
-        from application.core.settings import settings
 
         try:
-            # Calculate current tokens
             current_tokens = self._calculate_current_context_tokens(messages)
             self.current_token_count = current_tokens
-
-            # Get context limit for model
             context_limit = get_token_limit(self.model_id)
-
-            # Calculate threshold (80%)
             threshold = int(context_limit * settings.COMPRESSION_THRESHOLD_PERCENTAGE)
 
-            # Check if we've reached the limit
             if current_tokens >= threshold:
                 logger.warning(
                     f"Context limit approaching: {current_tokens}/{context_limit} tokens "
                     f"({(current_tokens/context_limit)*100:.1f}%)"
                 )
                 return True
-
             return False
-
         except Exception as e:
             logger.error(f"Error checking context limit: {str(e)}", exc_info=True)
             return False
 
     def _validate_context_size(self, messages: List[Dict]) -> None:
-        """
-        Pre-flight validation before calling LLM. Logs warnings but never raises errors.
-
-        Args:
-            messages: Messages to be sent to LLM
-        """
         from application.core.model_utils import get_token_limit
 
         current_tokens = self._calculate_current_context_tokens(messages)
         self.current_token_count = current_tokens
         context_limit = get_token_limit(self.model_id)
-
         percentage = (current_tokens / context_limit) * 100
 
-        # Log based on usage level
         if current_tokens >= context_limit:
             logger.warning(
                 f"Context at limit: {current_tokens:,}/{context_limit:,} tokens "
@@ -435,44 +193,32 @@ class BaseAgent(ABC):
             )
 
     def _truncate_text_middle(self, text: str, max_tokens: int) -> str:
-        """
-        Truncate text by removing content from the middle, preserving start and end.
-
-        Args:
-            text: Text to truncate
-            max_tokens: Maximum tokens allowed
-
-        Returns:
-            Truncated text with middle removed if needed
-        """
         from application.utils import num_tokens_from_string
 
         current_tokens = num_tokens_from_string(text)
         if current_tokens <= max_tokens:
             return text
 
-        # Estimate chars per token (roughly 4 chars per token for English)
         chars_per_token = len(text) / current_tokens if current_tokens > 0 else 4
-        target_chars = int(max_tokens * chars_per_token * 0.95)  # 5% safety margin
+        target_chars = int(max_tokens * chars_per_token * 0.95)
 
         if target_chars <= 0:
             return ""
 
-        # Split: keep 40% from start, 40% from end, remove middle
         start_chars = int(target_chars * 0.4)
         end_chars = int(target_chars * 0.4)
 
         truncation_marker = "\n\n[... content truncated to fit context limit ...]\n\n"
-
         truncated = text[:start_chars] + truncation_marker + text[-end_chars:]
 
         logger.info(
             f"Truncated text from {current_tokens:,} to ~{max_tokens:,} tokens "
             f"(removed middle section)"
         )
-
         return truncated
 
+    # ---- Message building ----
+
     def _build_messages(
         self,
         system_prompt: str,
@@ -482,7 +228,6 @@ class BaseAgent(ABC):
         from application.core.model_utils import get_token_limit
         from application.utils import num_tokens_from_string
 
-        # Append compression summary to system prompt if present
         if self.compressed_summary:
             compression_context = (
                 "\n\n---\n\n"
@@ -496,23 +241,18 @@ class BaseAgent(ABC):
         context_limit = get_token_limit(self.model_id)
         system_tokens = num_tokens_from_string(system_prompt)
 
-        # Reserve 10% for response/tools
         safety_buffer = int(context_limit * 0.1)
         available_after_system = context_limit - system_tokens - safety_buffer
 
-        # Max tokens for query: 80% of available space (leave room for history)
         max_query_tokens = int(available_after_system * 0.8)
         query_tokens = num_tokens_from_string(query)
 
-        # Truncate query from middle if it exceeds 80% of available context
         if query_tokens > max_query_tokens:
             query = self._truncate_text_middle(query, max_query_tokens)
             query_tokens = num_tokens_from_string(query)
 
-        # Calculate remaining budget for chat history
         available_for_history = max(available_after_system - query_tokens, 0)
 
-        # Truncate chat history to fit within available budget
         working_history = self._truncate_history_to_fit(
             self.chat_history,
             available_for_history,
@@ -557,16 +297,6 @@ class BaseAgent(ABC):
         history: List[Dict],
         max_tokens: int,
     ) -> List[Dict]:
-        """
-        Truncate chat history to fit within token budget, keeping most recent messages.
-
-        Args:
-            history: Full chat history
-            max_tokens: Maximum tokens allowed for history
-
-        Returns:
-            Truncated history (most recent messages that fit)
-        """
         from application.utils import num_tokens_from_string
 
         if not history or max_tokens <= 0:
@@ -575,7 +305,6 @@ class BaseAgent(ABC):
         truncated = []
         current_tokens = 0
 
-        # Iterate from newest to oldest
         for message in reversed(history):
             message_tokens = 0
 
@@ -595,7 +324,7 @@ class BaseAgent(ABC):
 
             if current_tokens + message_tokens <= max_tokens:
                 current_tokens += message_tokens
-                truncated.insert(0, message)  # Maintain chronological order
+                truncated.insert(0, message)
             else:
                 break
 
@@ -607,13 +336,13 @@ class BaseAgent(ABC):
 
         return truncated
 
+    # ---- LLM generation ----
+
     def _llm_gen(self, messages: List[Dict], log_context: Optional[LogContext] = None):
-        # Pre-flight context validation - fail fast if over limit
         self._validate_context_size(messages)
 
         gen_kwargs = {"model": self.model_id, "messages": messages}
         if self.attachments:
-            # Usage accounting only; stripped before provider invocation.
             gen_kwargs["_usage_attachments"] = self.attachments
 
         if (

application/agents/classic_agent.py

@@ -15,11 +15,7 @@ class ClassicAgent(BaseAgent):
     ) -> Generator[Dict, None, None]:
         """Core generator function for ClassicAgent execution flow"""
 
-        tools_dict = (
-            self._get_user_tools(self.user)
-            if not self.user_api_key
-            else self._get_tools(self.user_api_key)
-        )
+        tools_dict = self.tool_executor.get_tools()
         self._prepare_tools(tools_dict)
 
         messages = self._build_messages(self.prompt, query)

application/agents/react_agent.py

@@ -1,238 +0,0 @@
-import logging
-import os
-from typing import Any, Dict, Generator, List
-
-from application.agents.base import BaseAgent
-from application.logging import build_stack_data, LogContext
-
-logger = logging.getLogger(__name__)
-
-MAX_ITERATIONS_REASONING = 10
-
-current_dir = os.path.dirname(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-)
-with open(
-    os.path.join(current_dir, "application/prompts", "react_planning_prompt.txt"), "r"
-) as f:
-    PLANNING_PROMPT_TEMPLATE = f.read()
-with open(
-    os.path.join(current_dir, "application/prompts", "react_final_prompt.txt"), "r"
-) as f:
-    FINAL_PROMPT_TEMPLATE = f.read()
-
-
-class ReActAgent(BaseAgent):
-    """
-    Research and Action (ReAct) Agent - Advanced reasoning agent with iterative planning.
-
-    Implements a think-act-observe loop for complex problem-solving:
-    1. Creates a strategic plan based on the query
-    2. Executes tools and gathers observations
-    3. Iteratively refines approach until satisfied
-    4. Synthesizes final answer from all observations
-    """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.plan: str = ""
-        self.observations: List[str] = []
-
-    def _gen_inner(
-        self, query: str, log_context: LogContext
-    ) -> Generator[Dict, None, None]:
-        """Execute ReAct reasoning loop with planning, action, and observation cycles"""
-
-        self._reset_state()
-
-        tools_dict = (
-            self._get_tools(self.user_api_key)
-            if self.user_api_key
-            else self._get_user_tools(self.user)
-        )
-        self._prepare_tools(tools_dict)
-
-        for iteration in range(1, MAX_ITERATIONS_REASONING + 1):
-            yield {"thought": f"Reasoning... (iteration {iteration})\n\n"}
-
-            yield from self._planning_phase(query, log_context)
-
-            if not self.plan:
-                logger.warning(
-                    f"ReActAgent: No plan generated in iteration {iteration}"
-                )
-                break
-            self.observations.append(f"Plan (iteration {iteration}): {self.plan}")
-
-            satisfied = yield from self._execution_phase(query, tools_dict, log_context)
-
-            if satisfied:
-                logger.info("ReActAgent: Goal satisfied, stopping reasoning loop")
-                break
-        yield from self._synthesis_phase(query, log_context)
-
-    def _reset_state(self):
-        """Reset agent state for new query"""
-        self.plan = ""
-        self.observations = []
-
-    def _planning_phase(
-        self, query: str, log_context: LogContext
-    ) -> Generator[Dict, None, None]:
-        """Generate strategic plan for query"""
-        logger.info("ReActAgent: Creating plan...")
-
-        plan_prompt = self._build_planning_prompt(query)
-        messages = [{"role": "user", "content": plan_prompt}]
-
-        plan_stream = self.llm.gen_stream(
-            model=self.model_id,
-            messages=messages,
-            tools=self.tools if self.tools else None,
-        )
-
-        if log_context:
-            log_context.stacks.append(
-                {"component": "planning_llm", "data": build_stack_data(self.llm)}
-            )
-        plan_parts = []
-        for chunk in plan_stream:
-            content = self._extract_content(chunk)
-            if content:
-                plan_parts.append(content)
-                yield {"thought": content}
-        self.plan = "".join(plan_parts)
-
-    def _execution_phase(
-        self, query: str, tools_dict: Dict, log_context: LogContext
-    ) -> Generator[bool, None, None]:
-        """Execute plan with tool calls and observations"""
-        execution_prompt = self._build_execution_prompt(query)
-        messages = self._build_messages(execution_prompt, query)
-
-        llm_response = self._llm_gen(messages, log_context)
-        initial_content = self._extract_content(llm_response)
-
-        if initial_content:
-            self.observations.append(f"Initial response: {initial_content}")
-        processed_response = self._llm_handler(
-            llm_response, tools_dict, messages, log_context
-        )
-
-        for tool_call in self.tool_calls:
-            observation = (
-                f"Executed: {tool_call.get('tool_name', 'Unknown')} "
-                f"with args {tool_call.get('arguments', {})}. "
-                f"Result: {str(tool_call.get('result', ''))[:200]}"
-            )
-            self.observations.append(observation)
-        final_content = self._extract_content(processed_response)
-        if final_content:
-            self.observations.append(f"Response after tools: {final_content}")
-        if log_context:
-            log_context.stacks.append(
-                {
-                    "component": "agent_tool_calls",
-                    "data": {"tool_calls": self.tool_calls.copy()},
-                }
-            )
-        yield {"sources": self.retrieved_docs}
-        yield {"tool_calls": self._get_truncated_tool_calls()}
-
-        return "SATISFIED" in (final_content or "")
-
-    def _synthesis_phase(
-        self, query: str, log_context: LogContext
-    ) -> Generator[Dict, None, None]:
-        """Synthesize final answer from all observations"""
-        logger.info("ReActAgent: Generating final answer...")
-
-        final_prompt = self._build_final_answer_prompt(query)
-        messages = [{"role": "user", "content": final_prompt}]
-
-        final_stream = self.llm.gen_stream(
-            model=self.model_id, messages=messages, tools=None
-        )
-
-        if log_context:
-            log_context.stacks.append(
-                {"component": "final_answer_llm", "data": build_stack_data(self.llm)}
-            )
-        for chunk in final_stream:
-            content = self._extract_content(chunk)
-            if content:
-                yield {"answer": content}
-
-    def _build_planning_prompt(self, query: str) -> str:
-        """Build planning phase prompt"""
-        prompt = PLANNING_PROMPT_TEMPLATE.replace("{query}", query)
-        prompt = prompt.replace("{prompt}", self.prompt or "")
-        prompt = prompt.replace("{summaries}", "")
-        prompt = prompt.replace("{observations}", "\n".join(self.observations))
-        return prompt
-
-    def _build_execution_prompt(self, query: str) -> str:
-        """Build execution phase prompt with plan and observations"""
-        observations_str = "\n".join(self.observations)
-
-        if len(observations_str) > 20000:
-            observations_str = observations_str[:20000] + "\n...[truncated]"
-        return (
-            f"{self.prompt or ''}\n\n"
-            f"Follow this plan:\n{self.plan}\n\n"
-            f"Observations:\n{observations_str}\n\n"
-            f"If sufficient data exists to answer '{query}', respond with 'SATISFIED'. "
-            f"Otherwise, continue executing the plan."
-        )
-
-    def _build_final_answer_prompt(self, query: str) -> str:
-        """Build final synthesis prompt"""
-        observations_str = "\n".join(self.observations)
-
-        if len(observations_str) > 10000:
-            observations_str = observations_str[:10000] + "\n...[truncated]"
-            logger.warning("ReActAgent: Observations truncated for final answer")
-        return FINAL_PROMPT_TEMPLATE.format(query=query, observations=observations_str)
-
-    def _extract_content(self, response: Any) -> str:
-        """Extract text content from various LLM response formats"""
-        if not response:
-            return ""
-        collected = []
-
-        if isinstance(response, str):
-            return response
-        if hasattr(response, "message") and hasattr(response.message, "content"):
-            if response.message.content:
-                return response.message.content
-        if hasattr(response, "choices") and response.choices:
-            if hasattr(response.choices[0], "message"):
-                content = response.choices[0].message.content
-                if content:
-                    return content
-        if hasattr(response, "content") and isinstance(response.content, list):
-            if response.content and hasattr(response.content[0], "text"):
-                return response.content[0].text
-        try:
-            for chunk in response:
-                content_piece = ""
-
-                if hasattr(chunk, "choices") and chunk.choices:
-                    if hasattr(chunk.choices[0], "delta"):
-                        delta_content = chunk.choices[0].delta.content
-                        if delta_content:
-                            content_piece = delta_content
-                elif hasattr(chunk, "type") and chunk.type == "content_block_delta":
-                    if hasattr(chunk, "delta") and hasattr(chunk.delta, "text"):
-                        content_piece = chunk.delta.text
-                elif isinstance(chunk, str):
-                    content_piece = chunk
-                if content_piece:
-                    collected.append(content_piece)
-        except (TypeError, AttributeError):
-            logger.debug(
-                f"Response not iterable or unexpected format: {type(response)}"
-            )
-        except Exception as e:
-            logger.error(f"Error extracting content: {e}")
-        return "".join(collected)

application/agents/research_agent.py

@@ -0,0 +1,692 @@
+import json
+import logging
+import os
+import time
+from typing import Dict, Generator, List, Optional
+
+from application.agents.base import BaseAgent
+from application.agents.tool_executor import ToolExecutor
+from application.agents.tools.internal_search import (
+    INTERNAL_TOOL_ID,
+    add_internal_search_tool,
+)
+from application.agents.tools.think import THINK_TOOL_ENTRY, THINK_TOOL_ID
+from application.logging import LogContext
+
+logger = logging.getLogger(__name__)
+
+# Defaults (can be overridden via constructor)
+DEFAULT_MAX_STEPS = 6
+DEFAULT_MAX_SUB_ITERATIONS = 5
+DEFAULT_TIMEOUT_SECONDS = 300  # 5 minutes
+DEFAULT_TOKEN_BUDGET = 100_000
+DEFAULT_PARALLEL_WORKERS = 3
+
+# Adaptive depth caps per complexity level
+COMPLEXITY_CAPS = {
+    "simple": 2,
+    "moderate": 4,
+    "complex": 6,
+}
+
+_PROMPTS_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+    "prompts",
+    "research",
+)
+
+
+def _load_prompt(name: str) -> str:
+    with open(os.path.join(_PROMPTS_DIR, name), "r") as f:
+        return f.read()
+
+
+CLARIFICATION_PROMPT = _load_prompt("clarification.txt")
+PLANNING_PROMPT = _load_prompt("planning.txt")
+STEP_PROMPT = _load_prompt("step.txt")
+SYNTHESIS_PROMPT = _load_prompt("synthesis.txt")
+
+
+# ---------------------------------------------------------------------------
+# CitationManager
+# ---------------------------------------------------------------------------
+
+
+class CitationManager:
+    """Tracks and deduplicates citations across research steps."""
+
+    def __init__(self):
+        self.citations: Dict[int, Dict] = {}
+        self._counter = 0
+
+    def add(self, doc: Dict) -> int:
+        """Register a source, return its citation number. Deduplicates by source."""
+        source = doc.get("source", "")
+        title = doc.get("title", "")
+        for num, existing in self.citations.items():
+            if existing.get("source") == source and existing.get("title") == title:
+                return num
+        self._counter += 1
+        self.citations[self._counter] = doc
+        return self._counter
+
+    def add_docs(self, docs: List[Dict]) -> str:
+        """Register multiple docs, return formatted citation mapping text."""
+        mapping_lines = []
+        for doc in docs:
+            num = self.add(doc)
+            title = doc.get("title", "Untitled")
+            mapping_lines.append(f"[{num}] {title}")
+        return "\n".join(mapping_lines)
+
+    def format_references(self) -> str:
+        """Generate [N] -> source mapping for report footer."""
+        if not self.citations:
+            return "No sources found."
+        lines = []
+        for num, doc in sorted(self.citations.items()):
+            title = doc.get("title", "Untitled")
+            source = doc.get("source", "Unknown")
+            filename = doc.get("filename", "")
+            display = filename or title
+            lines.append(f"[{num}] {display} — {source}")
+        return "\n".join(lines)
+
+    def get_all_docs(self) -> List[Dict]:
+        return list(self.citations.values())
+
+
+# ---------------------------------------------------------------------------
+# ResearchAgent
+# ---------------------------------------------------------------------------
+
+
+class ResearchAgent(BaseAgent):
+    """Multi-step research agent with parallel execution and budget controls.
+
+    Orchestrates: Plan -> Research (per step, optionally parallel) -> Synthesize.
+    """
+
+    def __init__(
+        self,
+        retriever_config: Optional[Dict] = None,
+        max_steps: int = DEFAULT_MAX_STEPS,
+        max_sub_iterations: int = DEFAULT_MAX_SUB_ITERATIONS,
+        timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
+        token_budget: int = DEFAULT_TOKEN_BUDGET,
+        parallel_workers: int = DEFAULT_PARALLEL_WORKERS,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.retriever_config = retriever_config or {}
+        self.max_steps = max_steps
+        self.max_sub_iterations = max_sub_iterations
+        self.timeout_seconds = timeout_seconds
+        self.token_budget = token_budget
+        self.parallel_workers = parallel_workers
+        self.citations = CitationManager()
+        self._start_time: float = 0
+        self._tokens_used: int = 0
+        self._last_token_snapshot: int = 0
+
+    # ------------------------------------------------------------------
+    # Budget & timeout helpers
+    # ------------------------------------------------------------------
+
+    def _is_timed_out(self) -> bool:
+        return (time.monotonic() - self._start_time) >= self.timeout_seconds
+
+    def _elapsed(self) -> float:
+        return round(time.monotonic() - self._start_time, 1)
+
+    def _track_tokens(self, count: int):
+        self._tokens_used += count
+
+    def _budget_remaining(self) -> int:
+        return max(self.token_budget - self._tokens_used, 0)
+
+    def _is_over_budget(self) -> bool:
+        return self._tokens_used >= self.token_budget
+
+    def _snapshot_llm_tokens(self) -> int:
+        """Read current token usage from LLM and return delta since last snapshot."""
+        current = self.llm.token_usage.get("prompt_tokens", 0) + self.llm.token_usage.get("generated_tokens", 0)
+        delta = current - self._last_token_snapshot
+        self._last_token_snapshot = current
+        return delta
+
+    # ------------------------------------------------------------------
+    # Main orchestration
+    # ------------------------------------------------------------------
+
+    def _gen_inner(
+        self, query: str, log_context: LogContext
+    ) -> Generator[Dict, None, None]:
+        self._start_time = time.monotonic()
+        tools_dict = self._setup_tools()
+
+        # Phase 0: Clarification (skip if user is responding to a prior clarification)
+        if not self._is_follow_up():
+            clarification = self._clarification_phase(query)
+            if clarification:
+                yield {"metadata": {"is_clarification": True}}
+                yield {"answer": clarification}
+                yield {"sources": []}
+                yield {"tool_calls": []}
+                log_context.stacks.append(
+                    {"component": "agent", "data": {"clarification": True}}
+                )
+                return
+
+        # Phase 1: Planning (with adaptive depth)
+        yield {"type": "research_progress", "data": {"status": "planning"}}
+        plan, complexity = self._planning_phase(query)
+
+        if not plan:
+            logger.warning("ResearchAgent: Planning produced no steps, falling back")
+            plan = [{"query": query, "rationale": "Direct investigation"}]
+            complexity = "simple"
+
+        yield {
+            "type": "research_plan",
+            "data": {"steps": plan, "complexity": complexity},
+        }
+
+        # Phase 2: Research each step (yields progress events in real-time)
+        intermediate_reports = []
+        for i, step in enumerate(plan):
+            step_num = i + 1
+            step_query = step.get("query", query)
+
+            if self._is_timed_out():
+                logger.warning(
+                    f"ResearchAgent: Timeout at step {step_num}/{len(plan)} "
+                    f"({self._elapsed()}s)"
+                )
+                break
+            if self._is_over_budget():
+                logger.warning(
+                    f"ResearchAgent: Token budget exhausted at step {step_num}/{len(plan)}"
+                )
+                break
+
+            yield {
+                "type": "research_progress",
+                "data": {
+                    "step": step_num,
+                    "total": len(plan),
+                    "query": step_query,
+                    "status": "researching",
+                },
+            }
+
+            report = self._research_step(step_query, tools_dict)
+            intermediate_reports.append({"step": step, "content": report})
+
+            yield {
+                "type": "research_progress",
+                "data": {
+                    "step": step_num,
+                    "total": len(plan),
+                    "query": step_query,
+                    "status": "complete",
+                },
+            }
+
+        # Phase 3: Synthesis (streaming)
+        if self._is_timed_out():
+            logger.warning(
+                f"ResearchAgent: Timeout ({self._elapsed()}s) before synthesis, "
+                f"synthesizing with {len(intermediate_reports)} reports"
+            )
+        yield {
+            "type": "research_progress",
+            "data": {
+                "status": "synthesizing",
+                "elapsed_seconds": self._elapsed(),
+                "tokens_used": self._tokens_used,
+            },
+        }
+        yield from self._synthesis_phase(
+            query, plan, intermediate_reports, tools_dict, log_context
+        )
+
+        # Sources and tool calls
+        self.retrieved_docs = self.citations.get_all_docs()
+        yield {"sources": self.retrieved_docs}
+        yield {"tool_calls": self._get_truncated_tool_calls()}
+
+        logger.info(
+            f"ResearchAgent completed: {len(intermediate_reports)}/{len(plan)} steps, "
+            f"{self._elapsed()}s, ~{self._tokens_used} tokens"
+        )
+        log_context.stacks.append(
+            {"component": "agent", "data": {"tool_calls": self.tool_calls.copy()}}
+        )
+
+    # ------------------------------------------------------------------
+    # Tool setup
+    # ------------------------------------------------------------------
+
+    def _setup_tools(self) -> Dict:
+        """Build tools_dict with user tools + internal search + think."""
+        tools_dict = self.tool_executor.get_tools()
+
+        add_internal_search_tool(tools_dict, self.retriever_config)
+
+        think_entry = dict(THINK_TOOL_ENTRY)
+        think_entry["config"] = {}
+        tools_dict[THINK_TOOL_ID] = think_entry
+
+        self._prepare_tools(tools_dict)
+        return tools_dict
+
+    # ------------------------------------------------------------------
+    # Phase 0: Clarification
+    # ------------------------------------------------------------------
+
+    def _is_follow_up(self) -> bool:
+        """Check if the user is responding to a prior clarification.
+
+        Uses the metadata flag stored in the conversation DB — no string matching.
+        Only skip clarification when the last query was explicitly flagged
+        as a clarification by this agent.
+        """
+        if not self.chat_history:
+            return False
+        last = self.chat_history[-1]
+        meta = last.get("metadata", {})
+        return bool(meta.get("is_clarification"))
+
+    def _clarification_phase(self, question: str) -> Optional[str]:
+        """Ask the LLM whether the question needs clarification.
+
+        Returns formatted clarification text if needed, or None to proceed.
+        Uses response_format to force valid JSON output.
+        """
+        messages = [
+            {"role": "system", "content": CLARIFICATION_PROMPT},
+            {"role": "user", "content": question},
+        ]
+
+        try:
+            response = self.llm.gen(
+                model=self.model_id,
+                messages=messages,
+                tools=None,
+                response_format={"type": "json_object"},
+            )
+            text = self._extract_text(response)
+            self._track_tokens(self._snapshot_llm_tokens())
+            logger.info(f"ResearchAgent clarification response: {text[:300]}")
+
+            data = self._parse_clarification_json(text)
+            if not data or not data.get("needs_clarification"):
+                return None
+
+            questions = data.get("questions", [])
+            if not questions:
+                return None
+
+            # Format as a friendly response
+            lines = [
+                "Before I begin researching, I'd like to clarify a few things:\n"
+            ]
+            for i, q in enumerate(questions[:3], 1):
+                lines.append(f"{i}. {q}")
+            lines.append(
+                "\nPlease provide these details and I'll start the research."
+            )
+            return "\n".join(lines)
+
+        except Exception as e:
+            logger.error(f"Clarification phase failed: {e}", exc_info=True)
+            return None  # proceed with research on failure
+
+    def _parse_clarification_json(self, text: str) -> Optional[Dict]:
+        """Parse clarification JSON from LLM response."""
+        try:
+            return json.loads(text)
+        except json.JSONDecodeError:
+            pass
+
+        # Try extracting from code fences
+        for marker in ["```json", "```"]:
+            if marker in text:
+                start = text.index(marker) + len(marker)
+                end = text.index("```", start) if "```" in text[start:] else len(text)
+                try:
+                    return json.loads(text[start:end].strip())
+                except (json.JSONDecodeError, ValueError):
+                    pass
+
+        # Try finding JSON object
+        for i, ch in enumerate(text):
+            if ch == "{":
+                for j in range(len(text) - 1, i, -1):
+                    if text[j] == "}":
+                        try:
+                            return json.loads(text[i : j + 1])
+                        except json.JSONDecodeError:
+                            continue
+                break
+
+        return None
+
+    # ------------------------------------------------------------------
+    # Phase 1: Planning (with adaptive depth)
+    # ------------------------------------------------------------------
+
+    def _planning_phase(self, question: str) -> tuple[List[Dict], str]:
+        """Decompose the question into research steps via LLM.
+
+        Returns (steps, complexity) where complexity is simple/moderate/complex.
+        """
+        messages = [
+            {"role": "system", "content": PLANNING_PROMPT},
+            {"role": "user", "content": question},
+        ]
+
+        try:
+            response = self.llm.gen(
+                model=self.model_id,
+                messages=messages,
+                tools=None,
+                response_format={"type": "json_object"},
+            )
+            text = self._extract_text(response)
+            self._track_tokens(self._snapshot_llm_tokens())
+            logger.info(f"ResearchAgent planning LLM response: {text[:500]}")
+
+            plan_data = self._parse_plan_json(text)
+            if isinstance(plan_data, dict):
+                complexity = plan_data.get("complexity", "moderate")
+                steps = plan_data.get("steps", [])
+            else:
+                complexity = "moderate"
+                steps = plan_data
+
+            # Adaptive depth: cap steps based on assessed complexity
+            cap = COMPLEXITY_CAPS.get(complexity, self.max_steps)
+            cap = min(cap, self.max_steps)
+            steps = steps[:cap]
+
+            logger.info(
+                f"ResearchAgent plan: complexity={complexity}, "
+                f"steps={len(steps)} (cap={cap})"
+            )
+            return steps, complexity
+
+        except Exception as e:
+            logger.error(f"Planning phase failed: {e}", exc_info=True)
+            return (
+                [{"query": question, "rationale": "Direct investigation (planning failed)"}],
+                "simple",
+            )
+
+    def _parse_plan_json(self, text: str):
+        """Extract JSON plan from LLM response. Returns dict or list."""
+        # Try direct parse
+        try:
+            data = json.loads(text)
+            if isinstance(data, dict) and "steps" in data:
+                return data
+            if isinstance(data, list):
+                return data
+        except json.JSONDecodeError:
+            pass
+
+        # Try extracting from markdown code fences
+        for marker in ["```json", "```"]:
+            if marker in text:
+                start = text.index(marker) + len(marker)
+                end = text.index("```", start) if "```" in text[start:] else len(text)
+                try:
+                    data = json.loads(text[start:end].strip())
+                    if isinstance(data, dict) and "steps" in data:
+                        return data
+                    if isinstance(data, list):
+                        return data
+                except (json.JSONDecodeError, ValueError):
+                    pass
+
+        # Try finding JSON object in text
+        for i, ch in enumerate(text):
+            if ch == "{":
+                for j in range(len(text) - 1, i, -1):
+                    if text[j] == "}":
+                        try:
+                            data = json.loads(text[i : j + 1])
+                            if isinstance(data, dict) and "steps" in data:
+                                return data
+                        except json.JSONDecodeError:
+                            continue
+                break
+
+        logger.warning(f"Could not parse plan JSON from: {text[:200]}")
+        return []
+
+    # ------------------------------------------------------------------
+    # Phase 2: Research step (core loop)
+    # ------------------------------------------------------------------
+
+    def _research_step(self, step_query: str, tools_dict: Dict) -> str:
+        """Run a focused research loop for one sub-question (sequential path)."""
+        report = self._research_step_with_executor(
+            step_query, tools_dict, self.tool_executor
+        )
+        self._collect_step_sources()
+        return report
+
+    def _research_step_with_executor(
+        self, step_query: str, tools_dict: Dict, executor: ToolExecutor
+    ) -> str:
+        """Core research loop. Works with any ToolExecutor instance."""
+        system_prompt = STEP_PROMPT.replace("{step_query}", step_query)
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": step_query},
+        ]
+
+        last_search_empty = False
+
+        for iteration in range(self.max_sub_iterations):
+            # Check timeout and budget
+            if self._is_timed_out():
+                logger.info(
+                    f"Research step '{step_query[:50]}' timed out at iteration {iteration}"
+                )
+                break
+            if self._is_over_budget():
+                logger.info(
+                    f"Research step '{step_query[:50]}' hit token budget at iteration {iteration}"
+                )
+                break
+
+            try:
+                response = self.llm.gen(
+                    model=self.model_id,
+                    messages=messages,
+                    tools=self.tools if self.tools else None,
+                )
+                self._track_tokens(self._snapshot_llm_tokens())
+            except Exception as e:
+                logger.error(
+                    f"Research step LLM call failed (iteration {iteration}): {e}",
+                    exc_info=True,
+                )
+                break
+
+            parsed = self.llm_handler.parse_response(response)
+
+            if not parsed.requires_tool_call:
+                return parsed.content or "No findings for this step."
+
+            # Execute tool calls
+            messages, last_search_empty = self._execute_step_tools_with_refinement(
+                parsed.tool_calls, tools_dict, messages, executor, last_search_empty
+            )
+
+        # Max iterations / timeout / budget — ask for summary
+        messages.append(
+            {
+                "role": "user",
+                "content": "Please summarize your findings so far based on the information gathered.",
+            }
+        )
+        try:
+            response = self.llm.gen(
+                model=self.model_id, messages=messages, tools=None
+            )
+            self._track_tokens(self._snapshot_llm_tokens())
+            text = self._extract_text(response)
+            return text or "Research step completed."
+        except Exception:
+            return "Research step completed."
+
+    def _execute_step_tools_with_refinement(
+        self,
+        tool_calls,
+        tools_dict: Dict,
+        messages: List[Dict],
+        executor: ToolExecutor,
+        last_search_empty: bool,
+    ) -> tuple[List[Dict], bool]:
+        """Execute tool calls with query refinement on empty results.
+
+        Returns (updated_messages, was_last_search_empty).
+        """
+        search_returned_empty = False
+
+        for call in tool_calls:
+            gen = executor.execute(
+                tools_dict, call, self.llm.__class__.__name__
+            )
+            result = None
+            call_id = None
+            while True:
+                try:
+                    event = next(gen)
+                    # Log tool_call status events instead of discarding them
+                    if isinstance(event, dict) and event.get("type") == "tool_call":
+                        logger.debug(
+                            "Tool %s status: %s",
+                            event.get("data", {}).get("action_name", ""),
+                            event.get("data", {}).get("status", ""),
+                        )
+                except StopIteration as e:
+                    result, call_id = e.value
+                    break
+
+            # Detect empty search results for refinement
+            is_search = "search" in (call.name or "").lower()
+            result_str = str(result) if result else ""
+            if is_search and "No documents found" in result_str:
+                search_returned_empty = True
+                if last_search_empty:
+                    # Two consecutive empty searches — inject refinement hint
+                    result_str += (
+                        "\n\nHint: Previous search also returned no results. "
+                        "Try a very different query with different keywords, "
+                        "or broaden your search terms."
+                    )
+                    result = result_str
+
+            function_call_content = {
+                "function_call": {
+                    "name": call.name,
+                    "args": call.arguments,
+                    "call_id": call_id,
+                }
+            }
+            messages.append(
+                {"role": "assistant", "content": [function_call_content]}
+            )
+            tool_message = self.llm_handler.create_tool_message(call, result)
+            messages.append(tool_message)
+
+        return messages, search_returned_empty
+
+    def _collect_step_sources(self):
+        """Collect sources from InternalSearchTool and register with CitationManager."""
+        cache_key = f"internal_search:{INTERNAL_TOOL_ID}:{self.user or ''}"
+        tool = self.tool_executor._loaded_tools.get(cache_key)
+        if tool and hasattr(tool, "retrieved_docs"):
+            for doc in tool.retrieved_docs:
+                self.citations.add(doc)
+
+    # ------------------------------------------------------------------
+    # Phase 3: Synthesis
+    # ------------------------------------------------------------------
+
+    def _synthesis_phase(
+        self,
+        question: str,
+        plan: List[Dict],
+        intermediate_reports: List[Dict],
+        tools_dict: Dict,
+        log_context: LogContext,
+    ) -> Generator[Dict, None, None]:
+        """Compile all findings into a final cited report (streaming)."""
+        plan_lines = []
+        for i, step in enumerate(plan, 1):
+            plan_lines.append(
+                f"{i}. {step.get('query', 'Unknown')} — {step.get('rationale', '')}"
+            )
+        plan_summary = "\n".join(plan_lines)
+
+        findings_parts = []
+        for i, report in enumerate(intermediate_reports, 1):
+            step_query = report["step"].get("query", "Unknown")
+            content = report["content"]
+            findings_parts.append(
+                f"--- Step {i}: {step_query} ---\n{content}"
+            )
+        findings = "\n\n".join(findings_parts)
+
+        references = self.citations.format_references()
+
+        synthesis_prompt = SYNTHESIS_PROMPT.replace("{question}", question)
+        synthesis_prompt = synthesis_prompt.replace("{plan_summary}", plan_summary)
+        synthesis_prompt = synthesis_prompt.replace("{findings}", findings)
+        synthesis_prompt = synthesis_prompt.replace("{references}", references)
+
+        messages = [
+            {"role": "system", "content": synthesis_prompt},
+            {"role": "user", "content": f"Please write the research report for: {question}"},
+        ]
+
+        llm_response = self.llm.gen_stream(
+            model=self.model_id, messages=messages, tools=None
+        )
+
+        if log_context:
+            from application.logging import build_stack_data
+
+            log_context.stacks.append(
+                {"component": "synthesis_llm", "data": build_stack_data(self.llm)}
+            )
+
+        yield from self._handle_response(
+            llm_response, tools_dict, messages, log_context
+        )
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def _extract_text(self, response) -> str:
+        """Extract text content from a non-streaming LLM response."""
+        if isinstance(response, str):
+            return response
+        if hasattr(response, "message") and hasattr(response.message, "content"):
+            return response.message.content or ""
+        if hasattr(response, "choices") and response.choices:
+            choice = response.choices[0]
+            if hasattr(choice, "message") and hasattr(choice.message, "content"):
+                return choice.message.content or ""
+        if hasattr(response, "content") and isinstance(response.content, list):
+            if response.content and hasattr(response.content[0], "text"):
+                return response.content[0].text or ""
+        return str(response) if response else ""

application/agents/tool_executor.py

@@ -0,0 +1,313 @@
+import logging
+import uuid
+from typing import Dict, List, Optional
+
+from bson.objectid import ObjectId
+
+from application.agents.tools.tool_action_parser import ToolActionParser
+from application.agents.tools.tool_manager import ToolManager
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+from application.security.encryption import decrypt_credentials
+
+logger = logging.getLogger(__name__)
+
+
+class ToolExecutor:
+    """Handles tool discovery, preparation, and execution.
+
+    Extracted from BaseAgent to separate concerns and enable tool caching.
+    """
+
+    def __init__(
+        self,
+        user_api_key: Optional[str] = None,
+        user: Optional[str] = None,
+        decoded_token: Optional[Dict] = None,
+    ):
+        self.user_api_key = user_api_key
+        self.user = user
+        self.decoded_token = decoded_token
+        self.tool_calls: List[Dict] = []
+        self._loaded_tools: Dict[str, object] = {}
+        self.conversation_id: Optional[str] = None
+
+    def get_tools(self) -> Dict[str, Dict]:
+        """Load tool configs from DB based on user context."""
+        if self.user_api_key:
+            return self._get_tools_by_api_key(self.user_api_key)
+        return self._get_user_tools(self.user or "local")
+
+    def _get_tools_by_api_key(self, api_key: str) -> Dict[str, Dict]:
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        agents_collection = db["agents"]
+        tools_collection = db["user_tools"]
+
+        agent_data = agents_collection.find_one({"key": api_key})
+        tool_ids = agent_data.get("tools", []) if agent_data else []
+
+        tools = (
+            tools_collection.find(
+                {"_id": {"$in": [ObjectId(tool_id) for tool_id in tool_ids]}}
+            )
+            if tool_ids
+            else []
+        )
+        tools = list(tools)
+        return {str(tool["_id"]): tool for tool in tools} if tools else {}
+
+    def _get_user_tools(self, user: str = "local") -> Dict[str, Dict]:
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        user_tools_collection = db["user_tools"]
+        user_tools = user_tools_collection.find({"user": user, "status": True})
+        user_tools = list(user_tools)
+        return {str(i): tool for i, tool in enumerate(user_tools)}
+
+    def prepare_tools_for_llm(self, tools_dict: Dict) -> List[Dict]:
+        """Convert tool configs to LLM function schemas."""
+        return [
+            {
+                "type": "function",
+                "function": {
+                    "name": f"{action['name']}_{tool_id}",
+                    "description": action["description"],
+                    "parameters": self._build_tool_parameters(action),
+                },
+            }
+            for tool_id, tool in tools_dict.items()
+            if (
+                (tool["name"] == "api_tool" and "actions" in tool.get("config", {}))
+                or (tool["name"] != "api_tool" and "actions" in tool)
+            )
+            for action in (
+                tool["config"]["actions"].values()
+                if tool["name"] == "api_tool"
+                else tool["actions"]
+            )
+            if action.get("active", True)
+        ]
+
+    def _build_tool_parameters(self, action: Dict) -> Dict:
+        params = {"type": "object", "properties": {}, "required": []}
+        for param_type in ["query_params", "headers", "body", "parameters"]:
+            if param_type in action and action[param_type].get("properties"):
+                for k, v in action[param_type]["properties"].items():
+                    if v.get("filled_by_llm", True):
+                        params["properties"][k] = {
+                            key: value
+                            for key, value in v.items()
+                            if key not in ("filled_by_llm", "value", "required")
+                        }
+                        if v.get("required", False):
+                            params["required"].append(k)
+        return params
+
+    def execute(self, tools_dict: Dict, call, llm_class_name: str):
+        """Execute a tool call. Yields status events, returns (result, call_id)."""
+        parser = ToolActionParser(llm_class_name)
+        tool_id, action_name, call_args = parser.parse_args(call)
+
+        call_id = getattr(call, "id", None) or str(uuid.uuid4())
+
+        if tool_id is None or action_name is None:
+            error_message = f"Error: Failed to parse LLM tool call. Tool name: {getattr(call, 'name', 'unknown')}"
+            logger.error(error_message)
+
+            tool_call_data = {
+                "tool_name": "unknown",
+                "call_id": call_id,
+                "action_name": getattr(call, "name", "unknown"),
+                "arguments": call_args or {},
+                "result": f"Failed to parse tool call. Invalid tool name format: {getattr(call, 'name', 'unknown')}",
+            }
+            yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
+            self.tool_calls.append(tool_call_data)
+            return "Failed to parse tool call.", call_id
+
+        if tool_id not in tools_dict:
+            error_message = f"Error: Tool ID '{tool_id}' extracted from LLM call not found in available tools_dict. Available IDs: {list(tools_dict.keys())}"
+            logger.error(error_message)
+
+            tool_call_data = {
+                "tool_name": "unknown",
+                "call_id": call_id,
+                "action_name": f"{action_name}_{tool_id}",
+                "arguments": call_args,
+                "result": f"Tool with ID {tool_id} not found. Available tools: {list(tools_dict.keys())}",
+            }
+            yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
+            self.tool_calls.append(tool_call_data)
+            return f"Tool with ID {tool_id} not found.", call_id
+
+        tool_call_data = {
+            "tool_name": tools_dict[tool_id]["name"],
+            "call_id": call_id,
+            "action_name": f"{action_name}_{tool_id}",
+            "arguments": call_args,
+        }
+        yield {"type": "tool_call", "data": {**tool_call_data, "status": "pending"}}
+
+        tool_data = tools_dict[tool_id]
+        action_data = (
+            tool_data["config"]["actions"][action_name]
+            if tool_data["name"] == "api_tool"
+            else next(
+                action
+                for action in tool_data["actions"]
+                if action["name"] == action_name
+            )
+        )
+
+        query_params, headers, body, parameters = {}, {}, {}, {}
+        param_types = {
+            "query_params": query_params,
+            "headers": headers,
+            "body": body,
+            "parameters": parameters,
+        }
+
+        for param_type, target_dict in param_types.items():
+            if param_type in action_data and action_data[param_type].get("properties"):
+                for param, details in action_data[param_type]["properties"].items():
+                    if (
+                        param not in call_args
+                        and "value" in details
+                        and details["value"]
+                    ):
+                        target_dict[param] = details["value"]
+        for param, value in call_args.items():
+            for param_type, target_dict in param_types.items():
+                if param_type in action_data and param in action_data[param_type].get(
+                    "properties", {}
+                ):
+                    target_dict[param] = value
+
+        # Load tool (with caching)
+        tool = self._get_or_load_tool(
+            tool_data, tool_id, action_name,
+            headers=headers, query_params=query_params,
+        )
+
+        resolved_arguments = (
+            {"query_params": query_params, "headers": headers, "body": body}
+            if tool_data["name"] == "api_tool"
+            else parameters
+        )
+        if tool_data["name"] == "api_tool":
+            logger.debug(
+                f"Executing api: {action_name} with query_params: {query_params}, headers: {headers}, body: {body}"
+            )
+            result = tool.execute_action(action_name, **body)
+        else:
+            logger.debug(f"Executing tool: {action_name} with args: {call_args}")
+            result = tool.execute_action(action_name, **parameters)
+
+        get_artifact_id = (
+            getattr(tool, "get_artifact_id", None)
+            if tool_data["name"] != "api_tool"
+            else None
+        )
+
+        artifact_id = None
+        if callable(get_artifact_id):
+            try:
+                artifact_id = get_artifact_id(action_name, **parameters)
+            except Exception:
+                logger.exception(
+                    "Failed to extract artifact_id from tool %s for action %s",
+                    tool_data["name"],
+                    action_name,
+                )
+
+        artifact_id = str(artifact_id).strip() if artifact_id is not None else ""
+        if artifact_id:
+            tool_call_data["artifact_id"] = artifact_id
+        result_full = str(result)
+        tool_call_data["resolved_arguments"] = resolved_arguments
+        tool_call_data["result_full"] = result_full
+        tool_call_data["result"] = (
+            f"{result_full[:50]}..." if len(result_full) > 50 else result_full
+        )
+
+        stream_tool_call_data = {
+            key: value
+            for key, value in tool_call_data.items()
+            if key not in {"result_full", "resolved_arguments"}
+        }
+        yield {"type": "tool_call", "data": {**stream_tool_call_data, "status": "completed"}}
+        self.tool_calls.append(tool_call_data)
+
+        return result, call_id
+
+    def _get_or_load_tool(
+        self, tool_data: Dict, tool_id: str, action_name: str,
+        headers: Optional[Dict] = None, query_params: Optional[Dict] = None,
+    ):
+        """Load a tool, using cache when possible."""
+        cache_key = f"{tool_data['name']}:{tool_id}:{self.user or ''}"
+        if cache_key in self._loaded_tools:
+            return self._loaded_tools[cache_key]
+
+        tm = ToolManager(config={})
+
+        if tool_data["name"] == "api_tool":
+            action_config = tool_data["config"]["actions"][action_name]
+            tool_config = {
+                "url": action_config["url"],
+                "method": action_config["method"],
+                "headers": headers or {},
+                "query_params": query_params or {},
+            }
+            if "body_content_type" in action_config:
+                tool_config["body_content_type"] = action_config.get(
+                    "body_content_type", "application/json"
+                )
+                tool_config["body_encoding_rules"] = action_config.get(
+                    "body_encoding_rules", {}
+                )
+        else:
+            tool_config = tool_data["config"].copy() if tool_data["config"] else {}
+            if tool_config.get("encrypted_credentials") and self.user:
+                decrypted = decrypt_credentials(
+                    tool_config["encrypted_credentials"], self.user
+                )
+                tool_config.update(decrypted)
+                tool_config["auth_credentials"] = decrypted
+                tool_config.pop("encrypted_credentials", None)
+            tool_config["tool_id"] = str(tool_data.get("_id", tool_id))
+            if self.conversation_id:
+                tool_config["conversation_id"] = self.conversation_id
+            if tool_data["name"] == "mcp_tool":
+                tool_config["query_mode"] = True
+
+        tool = tm.load_tool(
+            tool_data["name"],
+            tool_config=tool_config,
+            user_id=self.user,
+        )
+
+        # Don't cache api_tool since config varies by action
+        if tool_data["name"] != "api_tool":
+            self._loaded_tools[cache_key] = tool
+
+        return tool
+
+    def get_truncated_tool_calls(self) -> List[Dict]:
+        return [
+            {
+                "tool_name": tool_call.get("tool_name"),
+                "call_id": tool_call.get("call_id"),
+                "action_name": tool_call.get("action_name"),
+                "arguments": tool_call.get("arguments"),
+                "artifact_id": tool_call.get("artifact_id"),
+                "result": (
+                    f"{str(tool_call['result'])[:50]}..."
+                    if len(str(tool_call["result"])) > 50
+                    else tool_call["result"]
+                ),
+                "status": "completed",
+            }
+            for tool_call in self.tool_calls
+        ]

application/agents/tools/internal_search.py

@@ -0,0 +1,436 @@
+import json
+import logging
+from typing import Dict, List, Optional
+
+from application.agents.tools.base import Tool
+from application.core.settings import settings
+from application.retriever.retriever_creator import RetrieverCreator
+
+logger = logging.getLogger(__name__)
+
+
+class InternalSearchTool(Tool):
+    """Wraps the ClassicRAG retriever as an LLM-callable tool.
+
+    Instead of pre-fetching docs into the prompt, the LLM decides
+    when and what to search. Supports multiple searches per session.
+
+    Optional capabilities (enabled when sources have directory_structure):
+    - path_filter on search: restrict results to a specific file/folder
+    - list_files action: browse the file/folder structure
+    """
+
+    def __init__(self, config: Dict):
+        self.config = config
+        self.retrieved_docs: List[Dict] = []
+        self._retriever = None
+        self._directory_structure: Optional[Dict] = None
+        self._dir_structure_loaded = False
+
+    def _get_retriever(self):
+        if self._retriever is None:
+            self._retriever = RetrieverCreator.create_retriever(
+                self.config.get("retriever_name", "classic"),
+                source=self.config.get("source", {}),
+                chat_history=[],
+                prompt="",
+                chunks=int(self.config.get("chunks", 2)),
+                doc_token_limit=int(self.config.get("doc_token_limit", 50000)),
+                model_id=self.config.get("model_id", "docsgpt-local"),
+                user_api_key=self.config.get("user_api_key"),
+                agent_id=self.config.get("agent_id"),
+                llm_name=self.config.get("llm_name", settings.LLM_PROVIDER),
+                api_key=self.config.get("api_key", settings.API_KEY),
+                decoded_token=self.config.get("decoded_token"),
+            )
+        return self._retriever
+
+    def _get_directory_structure(self) -> Optional[Dict]:
+        """Load directory structure from MongoDB for the configured sources."""
+        if self._dir_structure_loaded:
+            return self._directory_structure
+
+        self._dir_structure_loaded = True
+        source = self.config.get("source", {})
+        active_docs = source.get("active_docs", [])
+        if not active_docs:
+            return None
+
+        try:
+            from bson.objectid import ObjectId
+            from application.core.mongo_db import MongoDB
+
+            mongo = MongoDB.get_client()
+            db = mongo[settings.MONGO_DB_NAME]
+            sources_collection = db["sources"]
+
+            if isinstance(active_docs, str):
+                active_docs = [active_docs]
+
+            merged_structure = {}
+            for doc_id in active_docs:
+                try:
+                    source_doc = sources_collection.find_one(
+                        {"_id": ObjectId(doc_id)}
+                    )
+                    if not source_doc:
+                        continue
+                    dir_str = source_doc.get("directory_structure")
+                    if dir_str:
+                        if isinstance(dir_str, str):
+                            dir_str = json.loads(dir_str)
+                        source_name = source_doc.get("name", doc_id)
+                        if len(active_docs) > 1:
+                            merged_structure[source_name] = dir_str
+                        else:
+                            merged_structure = dir_str
+                except Exception as e:
+                    logger.debug(f"Could not load dir structure for {doc_id}: {e}")
+
+            self._directory_structure = merged_structure if merged_structure else None
+        except Exception as e:
+            logger.debug(f"Failed to load directory structures: {e}")
+
+        return self._directory_structure
+
+    def execute_action(self, action_name: str, **kwargs):
+        if action_name == "search":
+            return self._execute_search(**kwargs)
+        elif action_name == "list_files":
+            return self._execute_list_files(**kwargs)
+        return f"Unknown action: {action_name}"
+
+    def _execute_search(self, **kwargs) -> str:
+        query = kwargs.get("query", "")
+        path_filter = kwargs.get("path_filter", "")
+
+        if not query:
+            return "Error: 'query' parameter is required."
+
+        try:
+            retriever = self._get_retriever()
+            docs = retriever.search(query)
+        except Exception as e:
+            logger.error(f"Internal search failed: {e}", exc_info=True)
+            return "Search failed: an internal error occurred."
+
+        if not docs:
+            return "No documents found matching your query."
+
+        # Apply path filter if specified
+        if path_filter:
+            path_lower = path_filter.lower()
+            docs = [
+                d
+                for d in docs
+                if path_lower in d.get("source", "").lower()
+                or path_lower in d.get("filename", "").lower()
+                or path_lower in d.get("title", "").lower()
+            ]
+            if not docs:
+                return f"No documents found matching query '{query}' in path '{path_filter}'."
+
+        # Accumulate for source tracking
+        for doc in docs:
+            if doc not in self.retrieved_docs:
+                self.retrieved_docs.append(doc)
+
+        # Format results for the LLM
+        formatted = []
+        for i, doc in enumerate(docs, 1):
+            title = doc.get("title", "Untitled")
+            text = doc.get("text", "")
+            source = doc.get("source", "Unknown")
+            filename = doc.get("filename", "")
+            header = filename or title
+            formatted.append(f"[{i}] {header} (source: {source})\n{text}")
+
+        return "\n\n---\n\n".join(formatted)
+
+    def _execute_list_files(self, **kwargs) -> str:
+        path = kwargs.get("path", "")
+        dir_structure = self._get_directory_structure()
+
+        if not dir_structure:
+            return "No file structure available for the current sources."
+
+        # Navigate to the requested path
+        current = dir_structure
+        if path:
+            for part in path.strip("/").split("/"):
+                if not part:
+                    continue
+                if isinstance(current, dict) and part in current:
+                    current = current[part]
+                else:
+                    return f"Path '{path}' not found in the file structure."
+
+        # Format the structure for the LLM
+        return self._format_structure(current, path or "/")
+
+    def _format_structure(self, node: Dict, current_path: str) -> str:
+        if not isinstance(node, dict):
+            return f"'{current_path}' is a file, not a directory."
+
+        lines = [f"File structure at '{current_path}':\n"]
+        folders = []
+        files = []
+
+        for name, value in sorted(node.items()):
+            if isinstance(value, dict):
+                # Check if it's a file metadata dict or a folder
+                if "type" in value or "size_bytes" in value or "token_count" in value:
+                    # It's a file with metadata
+                    size = value.get("token_count", "")
+                    ftype = value.get("type", "")
+                    info_parts = []
+                    if ftype:
+                        info_parts.append(ftype)
+                    if size:
+                        info_parts.append(f"{size} tokens")
+                    info = f" ({', '.join(info_parts)})" if info_parts else ""
+                    files.append(f"  {name}{info}")
+                else:
+                    # It's a folder
+                    count = self._count_files(value)
+                    folders.append(f"  {name}/ ({count} items)")
+            else:
+                files.append(f"  {name}")
+
+        if folders:
+            lines.append("Folders:")
+            lines.extend(folders)
+        if files:
+            lines.append("Files:")
+            lines.extend(files)
+        if not folders and not files:
+            lines.append("  (empty)")
+
+        return "\n".join(lines)
+
+    def _count_files(self, node: Dict) -> int:
+        count = 0
+        for value in node.values():
+            if isinstance(value, dict):
+                if "type" in value or "size_bytes" in value or "token_count" in value:
+                    count += 1
+                else:
+                    count += self._count_files(value)
+            else:
+                count += 1
+        return count
+
+    def get_actions_metadata(self):
+        actions = [
+            {
+                "name": "search",
+                "description": (
+                    "Search the user's uploaded documents and knowledge base. "
+                    "Use this to find relevant information before answering questions. "
+                    "You can call this multiple times with different queries."
+                ),
+                "parameters": {
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "The search query. Be specific and focused.",
+                            "filled_by_llm": True,
+                            "required": True,
+                        },
+                    }
+                },
+            }
+        ]
+
+        # Add path_filter and list_files only if directory structure exists
+        has_structure = self.config.get("has_directory_structure", False)
+        if has_structure:
+            actions[0]["parameters"]["properties"]["path_filter"] = {
+                "type": "string",
+                "description": (
+                    "Optional: filter results to a specific file or folder path. "
+                    "Use list_files first to see available paths."
+                ),
+                "filled_by_llm": True,
+                "required": False,
+            }
+            actions.append(
+                {
+                    "name": "list_files",
+                    "description": (
+                        "Browse the file and folder structure of the knowledge base. "
+                        "Use this to see what files are available before searching. "
+                        "Optionally provide a path to browse a specific folder."
+                    ),
+                    "parameters": {
+                        "properties": {
+                            "path": {
+                                "type": "string",
+                                "description": "Optional: folder path to browse. Leave empty for root.",
+                                "filled_by_llm": True,
+                                "required": False,
+                            }
+                        }
+                    },
+                }
+            )
+
+        return actions
+
+    def get_config_requirements(self):
+        return {}
+
+
+# Constants for building synthetic tools_dict entries
+INTERNAL_TOOL_ID = "internal"
+
+
+def build_internal_tool_entry(has_directory_structure: bool = False) -> Dict:
+    """Build the tools_dict entry for InternalSearchTool.
+
+    Dynamically includes list_files and path_filter based on
+    whether the sources have directory structure.
+    """
+    search_params = {
+        "properties": {
+            "query": {
+                "type": "string",
+                "description": "The search query. Be specific and focused.",
+                "filled_by_llm": True,
+                "required": True,
+            }
+        }
+    }
+
+    actions = [
+        {
+            "name": "search",
+            "description": (
+                "Search the user's uploaded documents and knowledge base. "
+                "Use this to find relevant information before answering questions. "
+                "You can call this multiple times with different queries."
+            ),
+            "active": True,
+            "parameters": search_params,
+        }
+    ]
+
+    if has_directory_structure:
+        search_params["properties"]["path_filter"] = {
+            "type": "string",
+            "description": (
+                "Optional: filter results to a specific file or folder path. "
+                "Use list_files first to see available paths."
+            ),
+            "filled_by_llm": True,
+            "required": False,
+        }
+        actions.append(
+            {
+                "name": "list_files",
+                "description": (
+                    "Browse the file and folder structure of the knowledge base. "
+                    "Use this to see what files are available before searching. "
+                    "Optionally provide a path to browse a specific folder."
+                ),
+                "active": True,
+                "parameters": {
+                    "properties": {
+                        "path": {
+                            "type": "string",
+                            "description": "Optional: folder path to browse. Leave empty for root.",
+                            "filled_by_llm": True,
+                            "required": False,
+                        }
+                    }
+                },
+            }
+        )
+
+    return {"name": "internal_search", "actions": actions}
+
+
+# Keep backward compat
+INTERNAL_TOOL_ENTRY = build_internal_tool_entry(has_directory_structure=False)
+
+
+def sources_have_directory_structure(source: Dict) -> bool:
+    """Check if any of the active sources have directory_structure in MongoDB."""
+    active_docs = source.get("active_docs", [])
+    if not active_docs:
+        return False
+
+    try:
+        from bson.objectid import ObjectId
+        from application.core.mongo_db import MongoDB
+
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        sources_collection = db["sources"]
+
+        if isinstance(active_docs, str):
+            active_docs = [active_docs]
+
+        for doc_id in active_docs:
+            try:
+                source_doc = sources_collection.find_one(
+                    {"_id": ObjectId(doc_id)},
+                    {"directory_structure": 1},
+                )
+                if source_doc and source_doc.get("directory_structure"):
+                    return True
+            except Exception:
+                continue
+    except Exception as e:
+        logger.debug(f"Could not check directory structure: {e}")
+
+    return False
+
+
+def add_internal_search_tool(tools_dict: Dict, retriever_config: Dict) -> None:
+    """Add the internal search tool to tools_dict if sources are configured.
+
+    Shared by AgenticAgent and ResearchAgent to avoid duplicate setup logic.
+    Mutates tools_dict in place.
+    """
+    source = retriever_config.get("source", {})
+    has_sources = bool(source.get("active_docs"))
+    if not retriever_config or not has_sources:
+        return
+
+    has_dir = sources_have_directory_structure(source)
+    internal_entry = build_internal_tool_entry(has_directory_structure=has_dir)
+    internal_entry["config"] = build_internal_tool_config(
+        **retriever_config,
+        has_directory_structure=has_dir,
+    )
+    tools_dict[INTERNAL_TOOL_ID] = internal_entry
+
+
+def build_internal_tool_config(
+    source: Dict,
+    retriever_name: str = "classic",
+    chunks: int = 2,
+    doc_token_limit: int = 50000,
+    model_id: str = "docsgpt-local",
+    user_api_key: Optional[str] = None,
+    agent_id: Optional[str] = None,
+    llm_name: str = None,
+    api_key: str = None,
+    decoded_token: Optional[Dict] = None,
+    has_directory_structure: bool = False,
+) -> Dict:
+    """Build the config dict for InternalSearchTool."""
+    return {
+        "source": source,
+        "retriever_name": retriever_name,
+        "chunks": chunks,
+        "doc_token_limit": doc_token_limit,
+        "model_id": model_id,
+        "user_api_key": user_api_key,
+        "agent_id": agent_id,
+        "llm_name": llm_name or settings.LLM_PROVIDER,
+        "api_key": api_key or settings.API_KEY,
+        "decoded_token": decoded_token,
+        "has_directory_structure": has_directory_structure,
+    }

application/agents/tools/think.py

@@ -0,0 +1,68 @@
+from application.agents.tools.base import Tool
+
+
+THINK_TOOL_ID = "think"
+
+THINK_TOOL_ENTRY = {
+    "name": "think",
+    "actions": [
+        {
+            "name": "reason",
+            "description": (
+                "Use this tool to think through your reasoning step by step "
+                "before deciding on your next action. Always reason before "
+                "searching or answering."
+            ),
+            "active": True,
+            "parameters": {
+                "properties": {
+                    "reasoning": {
+                        "type": "string",
+                        "description": "Your step-by-step reasoning and analysis",
+                        "filled_by_llm": True,
+                        "required": True,
+                    }
+                }
+            },
+        }
+    ],
+}
+
+
+class ThinkTool(Tool):
+    """Pseudo-tool that captures chain-of-thought reasoning.
+
+    Returns a short acknowledgment so the LLM can continue.
+    The reasoning content is captured in tool_call data for transparency.
+    """
+
+    def __init__(self, config=None):
+        pass
+
+    def execute_action(self, action_name: str, **kwargs):
+        return "Continue."
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "reason",
+                "description": (
+                    "Use this tool to think through your reasoning step by step "
+                    "before deciding on your next action. Always reason before "
+                    "searching or answering."
+                ),
+                "parameters": {
+                    "properties": {
+                        "reasoning": {
+                            "type": "string",
+                            "description": "Your step-by-step reasoning and analysis",
+                            "filled_by_llm": True,
+                            "required": True,
+                        }
+                    }
+                },
+            }
+        ]
+
+    def get_config_requirements(self):
+        return {}

application/agents/workflows/node_agent.py

@@ -2,9 +2,10 @@
 
 from typing import Any, Dict, List, Optional, Type
 
+from application.agents.agentic_agent import AgenticAgent
 from application.agents.base import BaseAgent
 from application.agents.classic_agent import ClassicAgent
-from application.agents.react_agent import ReActAgent
+from application.agents.research_agent import ResearchAgent
 from application.agents.workflows.schemas import AgentType
 
 
@@ -36,7 +37,8 @@ class ToolFilterMixin:
         return filtered_tools
 
 
-class WorkflowNodeClassicAgent(ToolFilterMixin, ClassicAgent):
+class _WorkflowNodeMixin:
+    """Common __init__ for all workflow node agents."""
 
     def __init__(
         self,
@@ -57,32 +59,25 @@ class WorkflowNodeClassicAgent(ToolFilterMixin, ClassicAgent):
         self._allowed_tool_ids = tool_ids or []
 
 
-class WorkflowNodeReActAgent(ToolFilterMixin, ReActAgent):
+class WorkflowNodeClassicAgent(ToolFilterMixin, _WorkflowNodeMixin, ClassicAgent):
+    pass
 
-    def __init__(
-        self,
-        endpoint: str,
-        llm_name: str,
-        model_id: str,
-        api_key: str,
-        tool_ids: Optional[List[str]] = None,
-        **kwargs,
-    ):
-        super().__init__(
-            endpoint=endpoint,
-            llm_name=llm_name,
-            model_id=model_id,
-            api_key=api_key,
-            **kwargs,
-        )
-        self._allowed_tool_ids = tool_ids or []
+
+class WorkflowNodeAgenticAgent(ToolFilterMixin, _WorkflowNodeMixin, AgenticAgent):
+    pass
+
+
+class WorkflowNodeResearchAgent(ToolFilterMixin, _WorkflowNodeMixin, ResearchAgent):
+    pass
 
 
 class WorkflowNodeAgentFactory:
 
     _agents: Dict[AgentType, Type[BaseAgent]] = {
         AgentType.CLASSIC: WorkflowNodeClassicAgent,
-        AgentType.REACT: WorkflowNodeReActAgent,
+        AgentType.REACT: WorkflowNodeClassicAgent,  # backwards compat
+        AgentType.AGENTIC: WorkflowNodeAgenticAgent,
+        AgentType.RESEARCH: WorkflowNodeResearchAgent,
     }
 
     @classmethod

application/agents/workflows/schemas.py

@@ -18,6 +18,8 @@ class NodeType(str, Enum):
 class AgentType(str, Enum):
     CLASSIC = "classic"
     REACT = "react"
+    AGENTIC = "agentic"
+    RESEARCH = "research"
 
 
 class ExecutionStatus(str, Enum):

application/agents/workflows/workflow_engine.py

@@ -7,6 +7,7 @@ from application.agents.workflows.cel_evaluator import CelEvaluationError, evalu
 from application.agents.workflows.node_agent import WorkflowNodeAgentFactory
 from application.agents.workflows.schemas import (
     AgentNodeConfig,
+    AgentType,
     ConditionNodeConfig,
     ExecutionStatus,
     NodeExecutionLog,
@@ -223,18 +224,32 @@ class WorkflowEngine:
                     f'Model "{node_model_id}" does not support structured output for node "{node.title}"'
                 )
 
-        node_agent = WorkflowNodeAgentFactory.create(
-            agent_type=node_config.agent_type,
-            endpoint=self.agent.endpoint,
-            llm_name=node_llm_name,
-            model_id=node_model_id,
-            api_key=node_api_key,
-            tool_ids=node_config.tools,
-            prompt=node_config.system_prompt,
-            chat_history=self.agent.chat_history,
-            decoded_token=self.agent.decoded_token,
-            json_schema=node_json_schema,
-        )
+        factory_kwargs = {
+            "agent_type": node_config.agent_type,
+            "endpoint": self.agent.endpoint,
+            "llm_name": node_llm_name,
+            "model_id": node_model_id,
+            "api_key": node_api_key,
+            "tool_ids": node_config.tools,
+            "prompt": node_config.system_prompt,
+            "chat_history": self.agent.chat_history,
+            "decoded_token": self.agent.decoded_token,
+            "json_schema": node_json_schema,
+        }
+
+        # Agentic/research agents need retriever_config for on-demand search
+        if node_config.agent_type in (AgentType.AGENTIC, AgentType.RESEARCH):
+            factory_kwargs["retriever_config"] = {
+                "source": {"active_docs": node_config.sources} if node_config.sources else {},
+                "retriever_name": node_config.retriever or "classic",
+                "chunks": int(node_config.chunks) if node_config.chunks else 2,
+                "model_id": node_model_id,
+                "llm_name": node_llm_name,
+                "api_key": node_api_key,
+                "decoded_token": self.agent.decoded_token,
+            }
+
+        node_agent = WorkflowNodeAgentFactory.create(**factory_kwargs)
 
         full_response_parts: List[str] = []
         structured_response_parts: List[str] = []

application/api/answer/routes/answer.py

@@ -74,21 +74,10 @@ class AnswerResource(Resource, BaseAnswerResource):
         decoded_token = getattr(request, "decoded_token", None)
         processor = StreamProcessor(data, decoded_token)
         try:
-            processor.initialize()
+            agent = processor.build_agent(data.get("question", ""))
             if not processor.decoded_token:
                 return make_response({"error": "Unauthorized"}, 401)
 
-            docs_together, docs_list = processor.pre_fetch_docs(
-                data.get("question", "")
-            )
-            tools_data = processor.pre_fetch_tools()
-
-            agent = processor.create_agent(
-                docs_together=docs_together,
-                docs=docs_list,
-                tools_data=tools_data,
-            )
-
             if error := self.check_usage(processor.agent_config):
                 return error
 

application/api/answer/routes/base.py

@@ -206,9 +206,12 @@ class BaseAnswerResource:
             is_structured = False
             schema_info = None
             structured_chunks = []
+            query_metadata = {}
 
             for line in agent.gen(query=question):
-                if "answer" in line:
+                if "metadata" in line:
+                    query_metadata.update(line["metadata"])
+                elif "answer" in line:
                     response_full += str(line["answer"])
                     if line.get("structured"):
                         is_structured = True
@@ -295,6 +298,7 @@ class BaseAnswerResource:
                     is_shared_usage=is_shared_usage,
                     shared_token=shared_token,
                     attachment_ids=attachment_ids,
+                    metadata=query_metadata if query_metadata else None,
                 )
                 # Persist compression metadata/summary if it exists and wasn't saved mid-execution
                 compression_meta = getattr(agent, "compression_metadata", None)
@@ -384,6 +388,7 @@ class BaseAnswerResource:
                         is_shared_usage=is_shared_usage,
                         shared_token=shared_token,
                         attachment_ids=attachment_ids,
+                        metadata=query_metadata if query_metadata else None,
                     )
                     compression_meta = getattr(agent, "compression_metadata", None)
                     compression_saved = getattr(agent, "compression_saved", False)

application/api/answer/routes/stream.py

@@ -80,7 +80,7 @@ class StreamResource(Resource, BaseAnswerResource):
         decoded_token = getattr(request, "decoded_token", None)
         processor = StreamProcessor(data, decoded_token)
         try:
-            processor.initialize()
+            agent = processor.build_agent(data["question"])
             if not processor.decoded_token:
                 return Response(
                     self.error_stream_generate("Unauthorized"),
@@ -88,13 +88,6 @@ class StreamResource(Resource, BaseAnswerResource):
                     mimetype="text/event-stream",
                 )
 
-            docs_together, docs_list = processor.pre_fetch_docs(data["question"])
-            tools_data = processor.pre_fetch_tools()
-
-            agent = processor.create_agent(
-                docs_together=docs_together, docs=docs_list, tools_data=tools_data
-            )
-
             if error := self.check_usage(processor.agent_config):
                 return error
             return Response(

application/api/answer/services/conversation_service.py

@@ -60,6 +60,7 @@ class ConversationService:
         is_shared_usage: bool = False,
         shared_token: Optional[str] = None,
         attachment_ids: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
     ) -> str:
         """Save or update a conversation in the database"""
         if decoded_token is None:
@@ -93,6 +94,11 @@ class ConversationService:
                         f"queries.{index}.timestamp": current_time,
                         f"queries.{index}.attachments": attachment_ids,
                         f"queries.{index}.model_id": model_id,
+                        **(
+                            {f"queries.{index}.metadata": metadata}
+                            if metadata
+                            else {}
+                        ),
                     }
                 },
             )
@@ -124,6 +130,7 @@ class ConversationService:
                             "timestamp": current_time,
                             "attachments": attachment_ids,
                             "model_id": model_id,
+                            **({"metadata": metadata} if metadata else {}),
                         }
                     }
                 },
@@ -156,22 +163,24 @@ class ConversationService:
             if not completion or not completion.strip():
                 completion = question[:50] if question else "New Conversation"
 
+            query_doc = {
+                "prompt": question,
+                "response": response,
+                "thought": thought,
+                "sources": sources,
+                "tool_calls": tool_calls,
+                "timestamp": current_time,
+                "attachments": attachment_ids,
+                "model_id": model_id,
+            }
+            if metadata:
+                query_doc["metadata"] = metadata
+
             conversation_data = {
                 "user": user_id,
                 "date": current_time,
                 "name": completion,
-                "queries": [
-                    {
-                        "prompt": question,
-                        "response": response,
-                        "thought": thought,
-                        "sources": sources,
-                        "tool_calls": tool_calls,
-                        "timestamp": current_time,
-                        "attachments": attachment_ids,
-                        "model_id": model_id,
-                    }
-                ],
+                "queries": [query_doc],
             }
 
             if api_key:

application/api/answer/services/stream_processor.py

@@ -38,13 +38,23 @@ def get_prompt(prompt_id: str, prompts_collection=None) -> str:
     current_dir = Path(__file__).resolve().parents[3]
     prompts_dir = current_dir / "prompts"
 
-    preset_mapping = {
+    # Maps for classic agent types
+    CLASSIC_PRESETS = {
         "default": "chat_combine_default.txt",
         "creative": "chat_combine_creative.txt",
         "strict": "chat_combine_strict.txt",
         "reduce": "chat_reduce_prompt.txt",
     }
 
+    # Agentic counterparts — same styles, but with search tool instructions
+    AGENTIC_PRESETS = {
+        "default": "agentic/default.txt",
+        "creative": "agentic/creative.txt",
+        "strict": "agentic/strict.txt",
+    }
+
+    preset_mapping = {**CLASSIC_PRESETS, **{f"agentic_{k}": v for k, v in AGENTIC_PRESETS.items()}}
+
     if prompt_id in preset_mapping:
         file_path = os.path.join(prompts_dir, preset_mapping[prompt_id])
         try:
@@ -91,6 +101,7 @@ class StreamProcessor:
         self.is_shared_usage = False
         self.shared_token = None
         self.agent_id = self.data.get("agent_id")
+        self.agent_key = None
         self.model_id: Optional[str] = None
         self.conversation_service = ConversationService()
         self.compression_orchestrator = CompressionOrchestrator(
@@ -111,6 +122,29 @@ class StreamProcessor:
         self._load_conversation_history()
         self._process_attachments()
 
+    def build_agent(self, question: str):
+        """One call to go from request data to a ready-to-run agent.
+
+        Combines initialize(), pre_fetch_docs(), pre_fetch_tools(), and
+        create_agent() into a single convenience method.
+        """
+        self.initialize()
+
+        agent_type = self.agent_config.get("agent_type", "classic")
+
+        # Agentic/research agents skip pre-fetch — the LLM searches on-demand via tools
+        if agent_type in ("agentic", "research"):
+            tools_data = self.pre_fetch_tools()
+            return self.create_agent(tools_data=tools_data)
+
+        docs_together, docs_list = self.pre_fetch_docs(question)
+        tools_data = self.pre_fetch_tools()
+        return self.create_agent(
+            docs_together=docs_together,
+            docs=docs_list,
+            tools_data=tools_data,
+        )
+
     def _load_conversation_history(self):
         """Load conversation history either from DB or request"""
         if self.conversation_id and self.initial_user_id:
@@ -124,9 +158,17 @@ class StreamProcessor:
             if settings.ENABLE_CONVERSATION_COMPRESSION:
                 self._handle_compression(conversation)
             else:
-                # Original behavior - load all history
+                # Original behavior - load all history (include metadata if present)
                 self.history = [
-                    {"prompt": query["prompt"], "response": query["response"]}
+                    {
+                        "prompt": query["prompt"],
+                        "response": query["response"],
+                        **(
+                            {"metadata": query["metadata"]}
+                            if "metadata" in query
+                            else {}
+                        ),
+                    }
                     for query in conversation.get("queries", [])
                 ]
         else:
@@ -135,14 +177,8 @@ class StreamProcessor:
             )
 
     def _handle_compression(self, conversation: Dict[str, Any]):
-        """
-        Handle conversation compression logic using orchestrator.
-
-        Args:
-            conversation: Full conversation document
-        """
+        """Handle conversation compression logic using orchestrator."""
         try:
-            # Use orchestrator to handle all compression logic
             result = self.compression_orchestrator.compress_if_needed(
                 conversation_id=self.conversation_id,
                 user_id=self.initial_user_id,
@@ -153,12 +189,15 @@ class StreamProcessor:
             if not result.success:
                 logger.error(f"Compression failed: {result.error}, using full history")
                 self.history = [
-                    {"prompt": query["prompt"], "response": query["response"]}
+                    {
+                        "prompt": query["prompt"],
+                        "response": query["response"],
+                        **({"metadata": query["metadata"]} if "metadata" in query else {}),
+                    }
                     for query in conversation.get("queries", [])
                 ]
                 return
 
-            # Set compressed summary if compression was performed
             if result.compression_performed and result.compressed_summary:
                 self.compressed_summary = result.compressed_summary
                 self.compressed_summary_tokens = TokenCounter.count_message_tokens(
@@ -169,17 +208,27 @@ class StreamProcessor:
                     f"+ {len(result.recent_queries)} recent messages"
                 )
 
-            # Build history from recent queries
             self.history = result.as_history()
+            # Preserve metadata from recent queries (as_history only has prompt/response)
+            recent = result.recent_queries if result.recent_queries else conversation.get("queries", [])
+            for i, entry in enumerate(self.history):
+                # Match by index from the end of recent queries
+                offset = len(recent) - len(self.history)
+                qi = offset + i
+                if 0 <= qi < len(recent) and "metadata" in recent[qi]:
+                    entry["metadata"] = recent[qi]["metadata"]
 
         except Exception as e:
             logger.error(
                 f"Error handling compression, falling back to standard history: {str(e)}",
                 exc_info=True,
             )
-            # Fallback to original behavior
             self.history = [
-                {"prompt": query["prompt"], "response": query["response"]}
+                {
+                    "prompt": query["prompt"],
+                    "response": query["response"],
+                    **({"metadata": query["metadata"]} if "metadata" in query else {}),
+                }
                 for query in conversation.get("queries", [])
             ]
 
@@ -191,9 +240,6 @@ class StreamProcessor:
         )
 
     def _get_attachments_content(self, attachment_ids, user_id):
-        """
-        Retrieve content from attachment documents based on their IDs.
-        """
         if not attachment_ids:
             return []
         attachments = []
@@ -202,7 +248,6 @@ class StreamProcessor:
                 attachment_doc = self.attachments_collection.find_one(
                     {"_id": ObjectId(attachment_id), "user": user_id}
                 )
-
                 if attachment_doc:
                     attachments.append(attachment_doc)
             except Exception as e:
@@ -232,7 +277,6 @@ class StreamProcessor:
                 )
             self.model_id = requested_model
         else:
-            # Check if agent has a default model configured
             agent_default_model = self.agent_config.get("default_model_id", "")
             if agent_default_model and validate_model_id(agent_default_model):
                 self.model_id = agent_default_model
@@ -285,7 +329,6 @@ class StreamProcessor:
             data["source"] = "default"
         else:
             data["source"] = None
-        # Handle multiple sources
 
         sources = data.get("sources", [])
         if sources and isinstance(sources, list):
@@ -311,7 +354,6 @@ class StreamProcessor:
         else:
             data["sources"] = []
 
-        # Preserve model configuration from agent
         data["default_model_id"] = data.get("default_model_id", "")
 
         return data
@@ -376,7 +418,10 @@ class StreamProcessor:
         return None
 
     def _configure_agent(self):
-        """Configure the agent based on request data"""
+        """Configure the agent based on request data.
+
+        Unified flow: resolve the effective API key, then extract config once.
+        """
         agent_id = self._resolve_agent_id()
 
         self.agent_key, self.is_shared_usage, self.shared_token = self._get_agent_key(
@@ -384,55 +429,37 @@ class StreamProcessor:
         )
         self.agent_id = str(agent_id) if agent_id else None
 
-        api_key = self.data.get("api_key")
-        if api_key:
-            data_key = self._get_data_from_api_key(api_key)
+        # Determine the effective API key (explicit > agent-derived)
+        effective_key = self.data.get("api_key") or self.agent_key
+
+        if effective_key:
+            data_key = self._get_data_from_api_key(effective_key)
             if data_key.get("_id"):
                 self.agent_id = str(data_key.get("_id"))
+
             self.agent_config.update(
                 {
                     "prompt_id": data_key.get("prompt_id", "default"),
                     "agent_type": data_key.get("agent_type", settings.AGENT_NAME),
-                    "user_api_key": api_key,
+                    "user_api_key": effective_key,
                     "json_schema": data_key.get("json_schema"),
                     "default_model_id": data_key.get("default_model_id", ""),
+                    "models": data_key.get("models", []),
                 }
             )
-            self.initial_user_id = data_key.get("user")
-            self.decoded_token = {"sub": data_key.get("user")}
-            if data_key.get("source"):
-                self.source = {"active_docs": data_key["source"]}
-            if data_key.get("workflow"):
-                self.agent_config["workflow"] = data_key["workflow"]
-                self.agent_config["workflow_owner"] = data_key.get("user")
-            if data_key.get("retriever"):
-                self.retriever_config["retriever_name"] = data_key["retriever"]
-            if data_key.get("chunks") is not None:
-                try:
-                    self.retriever_config["chunks"] = int(data_key["chunks"])
-                except (ValueError, TypeError):
-                    logger.warning(
-                        f"Invalid chunks value: {data_key['chunks']}, using default value 2"
-                    )
-                    self.retriever_config["chunks"] = 2
-        elif self.agent_key:
-            data_key = self._get_data_from_api_key(self.agent_key)
-            if data_key.get("_id"):
-                self.agent_id = str(data_key.get("_id"))
-            self.agent_config.update(
-                {
-                    "prompt_id": data_key.get("prompt_id", "default"),
-                    "agent_type": data_key.get("agent_type", settings.AGENT_NAME),
-                    "user_api_key": self.agent_key,
-                    "json_schema": data_key.get("json_schema"),
-                    "default_model_id": data_key.get("default_model_id", ""),
-                }
-            )
-            self.decoded_token = (
-                self.decoded_token
-                if self.is_shared_usage
-                else {"sub": data_key.get("user")}
-            )
+
+            # Set identity context
+            if self.data.get("api_key"):
+                # External API key: use the key owner's identity
+                self.initial_user_id = data_key.get("user")
+                self.decoded_token = {"sub": data_key.get("user")}
+            elif self.is_shared_usage:
+                # Shared agent: keep the caller's identity
+                pass
+            else:
+                # Owner using their own agent
+                self.decoded_token = {"sub": data_key.get("user")}
+
             if data_key.get("source"):
                 self.source = {"active_docs": data_key["source"]}
             if data_key.get("workflow"):
@@ -449,6 +476,7 @@ class StreamProcessor:
                     )
                     self.retriever_config["chunks"] = 2
         else:
+            # No API key — default/workflow configuration
             agent_type = settings.AGENT_NAME
             if self.data.get("workflow") and isinstance(
                 self.data.get("workflow"), dict
@@ -531,12 +559,7 @@ class StreamProcessor:
             return None, None
 
     def pre_fetch_tools(self) -> Optional[Dict[str, Any]]:
-        """Pre-fetch tool data for template rendering before agent creation
-
-        Can be controlled via:
-        1. Global setting: ENABLE_TOOL_PREFETCH in .env
-        2. Per-request: disable_tool_prefetch in request data
-        """
+        """Pre-fetch tool data for template rendering before agent creation"""
         if not settings.ENABLE_TOOL_PREFETCH:
             logger.info(
                 "Tool pre-fetching disabled globally via ENABLE_TOOL_PREFETCH setting"
@@ -755,11 +778,21 @@ class StreamProcessor:
         tools_data: Optional[Dict[str, Any]] = None,
     ):
         """Create and return the configured agent with rendered prompt"""
+        agent_type = self.agent_config["agent_type"]
+
+        # For agentic agents, swap standard presets for their agentic
+        # counterparts (which include search tool instructions instead of
+        # {summaries}). Custom / user-provided prompts pass through as-is.
         raw_prompt = self._get_prompt_content()
         if raw_prompt is None:
-            raw_prompt = get_prompt(
-                self.agent_config["prompt_id"], self.prompts_collection
-            )
+            prompt_id = self.agent_config.get("prompt_id", "default")
+            agentic_presets = {"default", "creative", "strict"}
+            if agent_type in ("agentic", "research") and prompt_id in agentic_presets:
+                raw_prompt = get_prompt(
+                    f"agentic_{prompt_id}", self.prompts_collection
+                )
+            else:
+                raw_prompt = get_prompt(prompt_id, self.prompts_collection)
             self._prompt_content = raw_prompt
 
         rendered_prompt = self.prompt_renderer.render_prompt(
@@ -779,7 +812,35 @@ class StreamProcessor:
         )
         system_api_key = get_api_key_for_provider(provider or settings.LLM_PROVIDER)
 
-        agent_type = self.agent_config["agent_type"]
+        # Create LLM and handler (dependency injection)
+        from application.llm.llm_creator import LLMCreator
+        from application.llm.handlers.handler_creator import LLMHandlerCreator
+        from application.agents.tool_executor import ToolExecutor
+
+        # Compute backup models: agent's configured models minus the active one
+        agent_models = self.agent_config.get("models", [])
+        backup_models = [m for m in agent_models if m != self.model_id]
+
+        llm = LLMCreator.create_llm(
+            provider or settings.LLM_PROVIDER,
+            api_key=system_api_key,
+            user_api_key=self.agent_config["user_api_key"],
+            decoded_token=self.decoded_token,
+            model_id=self.model_id,
+            agent_id=self.agent_id,
+            backup_models=backup_models,
+        )
+        llm_handler = LLMHandlerCreator.create_handler(
+            provider if provider else "default"
+        )
+
+        user = self.decoded_token.get("sub") if self.decoded_token else None
+        tool_executor = ToolExecutor(
+            user_api_key=self.agent_config["user_api_key"],
+            user=user,
+            decoded_token=self.decoded_token,
+        )
+        tool_executor.conversation_id = self.conversation_id
 
         # Base agent kwargs
         agent_kwargs = {
@@ -796,10 +857,31 @@ class StreamProcessor:
             "attachments": self.attachments,
             "json_schema": self.agent_config.get("json_schema"),
             "compressed_summary": self.compressed_summary,
+            "llm": llm,
+            "llm_handler": llm_handler,
+            "tool_executor": tool_executor,
         }
 
-        # Workflow-specific kwargs for workflow agents
-        if agent_type == "workflow":
+        # Type-specific kwargs
+        if agent_type in ("agentic", "research"):
+            agent_kwargs["retriever_config"] = {
+                "source": self.source,
+                "retriever_name": self.retriever_config.get(
+                    "retriever_name", "classic"
+                ),
+                "chunks": self.retriever_config.get("chunks", 2),
+                "doc_token_limit": self.retriever_config.get(
+                    "doc_token_limit", 50000
+                ),
+                "model_id": self.model_id,
+                "user_api_key": self.agent_config["user_api_key"],
+                "agent_id": self.agent_id,
+                "llm_name": provider or settings.LLM_PROVIDER,
+                "api_key": system_api_key,
+                "decoded_token": self.decoded_token,
+            }
+
+        elif agent_type == "workflow":
             workflow_config = self.agent_config.get("workflow")
             if isinstance(workflow_config, str):
                 agent_kwargs["workflow_id"] = workflow_config

application/api/user/agents/routes.py

@@ -104,6 +104,8 @@ AGENT_TYPE_SCHEMAS = {
 }
 
 AGENT_TYPE_SCHEMAS["react"] = AGENT_TYPE_SCHEMAS["classic"]
+AGENT_TYPE_SCHEMAS["agentic"] = AGENT_TYPE_SCHEMAS["classic"]
+AGENT_TYPE_SCHEMAS["research"] = AGENT_TYPE_SCHEMAS["classic"]
 AGENT_TYPE_SCHEMAS["openai"] = AGENT_TYPE_SCHEMAS["classic"]
 
 

application/api/user/base.py

@@ -145,14 +145,22 @@ def resolve_tool_details(tool_ids):
     Returns:
         List of tool details with id, name, and display_name
     """
+    valid_ids = []
+    for tid in tool_ids:
+        try:
+            valid_ids.append(ObjectId(tid))
+        except Exception:
+            continue
     tools = user_tools_collection.find(
-        {"_id": {"$in": [ObjectId(tid) for tid in tool_ids]}}
-    )
+        {"_id": {"$in": valid_ids}}
+    ) if valid_ids else []
     return [
         {
             "id": str(tool["_id"]),
             "name": tool.get("name", ""),
-            "display_name": tool.get("displayName", tool.get("name", "")),
+            "display_name": tool.get("customName")
+            or tool.get("displayName")
+            or tool.get("name", ""),
         }
         for tool in tools
     ]

application/app.py

@@ -72,6 +72,11 @@ def home():
         return "Welcome to DocsGPT Backend!"
 
 
+@app.route("/api/health")
+def health():
+    return jsonify({"status": "ok"})
+
+
 @app.route("/api/config")
 def get_config():
     response = {

application/core/model_configs.py

@@ -27,6 +27,8 @@ ANTHROPIC_ATTACHMENTS = IMAGE_ATTACHMENTS
 
 OPENROUTER_ATTACHMENTS = IMAGE_ATTACHMENTS
 
+NOVITA_ATTACHMENTS = IMAGE_ATTACHMENTS
+
 
 OPENAI_MODELS = [
     AvailableModel(
@@ -193,6 +195,46 @@ OPENROUTER_MODELS = [
     ),
 ]
 
+NOVITA_MODELS = [
+    AvailableModel(
+        id="moonshotai/kimi-k2.5",
+        provider=ModelProvider.NOVITA,
+        display_name="Kimi K2.5",
+        description="MoE model with function calling, structured output, reasoning, and vision",
+        capabilities=ModelCapabilities(
+            supports_tools=True,
+            supports_structured_output=True,
+            supported_attachment_types=NOVITA_ATTACHMENTS,
+            context_window=262144,
+        ),
+    ),
+    AvailableModel(
+        id="zai-org/glm-5",
+        provider=ModelProvider.NOVITA,
+        display_name="GLM-5",
+        description="MoE model with function calling, structured output, and reasoning",
+        capabilities=ModelCapabilities(
+            supports_tools=True,
+            supports_structured_output=True,
+            supported_attachment_types=[],
+            context_window=202800,
+        ),
+    ),
+    AvailableModel(
+        id="minimax/minimax-m2.5",
+        provider=ModelProvider.NOVITA,
+        display_name="MiniMax M2.5",
+        description="MoE model with function calling, structured output, and reasoning",
+        capabilities=ModelCapabilities(
+            supports_tools=True,
+            supports_structured_output=True,
+            supported_attachment_types=[],
+            context_window=204800,
+        ),
+    ),
+]
+
+
 AZURE_OPENAI_MODELS = [
     AvailableModel(
         id="azure-gpt-4",

application/core/model_settings.py

@@ -114,6 +114,10 @@ class ModelRegistry:
             settings.LLM_PROVIDER == "openrouter" and settings.API_KEY
         ):
             self._add_openrouter_models(settings)
+        if settings.NOVITA_API_KEY or (
+            settings.LLM_PROVIDER == "novita" and settings.API_KEY
+        ):
+            self._add_novita_models(settings)
         if settings.HUGGINGFACE_API_KEY or (
             settings.LLM_PROVIDER == "huggingface" and settings.API_KEY
         ):
@@ -245,6 +249,21 @@ class ModelRegistry:
         for model in OPENROUTER_MODELS:
             self.models[model.id] = model
 
+    def _add_novita_models(self, settings):
+        from application.core.model_configs import NOVITA_MODELS
+
+        if settings.NOVITA_API_KEY:
+            for model in NOVITA_MODELS:
+                self.models[model.id] = model
+            return
+        if settings.LLM_PROVIDER == "novita" and settings.LLM_NAME:
+            for model in NOVITA_MODELS:
+                if model.id == settings.LLM_NAME:
+                    self.models[model.id] = model
+                    return
+        for model in NOVITA_MODELS:
+            self.models[model.id] = model
+
     def _add_docsgpt_models(self, settings):
         model_id = "docsgpt-local"
         model = AvailableModel(

application/core/model_utils.py

@@ -10,6 +10,7 @@ def get_api_key_for_provider(provider: str) -> Optional[str]:
     provider_key_map = {
         "openai": settings.OPENAI_API_KEY,
         "openrouter": settings.OPEN_ROUTER_API_KEY,
+        "novita": settings.NOVITA_API_KEY,
         "anthropic": settings.ANTHROPIC_API_KEY,
         "google": settings.GOOGLE_API_KEY,
         "groq": settings.GROQ_API_KEY,

application/core/settings.py

@@ -5,9 +5,7 @@ from typing import Optional
 from pydantic import field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
-current_dir = os.path.dirname(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-)
+current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
 class Settings(BaseSettings):
@@ -15,15 +13,11 @@ class Settings(BaseSettings):
 
     AUTH_TYPE: Optional[str] = None  # simple_jwt, session_jwt, or None
     LLM_PROVIDER: str = "docsgpt"
-    LLM_NAME: Optional[str] = (
-        None  # if LLM_PROVIDER is openai, LLM_NAME can be gpt-4 or gpt-3.5-turbo
-    )
+    LLM_NAME: Optional[str] = None  # if LLM_PROVIDER is openai, LLM_NAME can be gpt-4 or gpt-3.5-turbo
     EMBEDDINGS_NAME: str = "huggingface_sentence-transformers/all-mpnet-base-v2"
     EMBEDDINGS_BASE_URL: Optional[str] = None  # Remote embeddings API URL (OpenAI-compatible)
-    EMBEDDINGS_KEY: Optional[str] = (
-        None  # api key for embeddings (if using openai, just copy API_KEY)
-    )
-    
+    EMBEDDINGS_KEY: Optional[str] = None  # api key for embeddings (if using openai, just copy API_KEY)
+
     CELERY_BROKER_URL: str = "redis://localhost:6379/0"
     CELERY_RESULT_BACKEND: str = "redis://localhost:6379/1"
     MONGO_URI: str = "mongodb://localhost:27017/docsgpt"
@@ -45,9 +39,7 @@ class Settings(BaseSettings):
     PARSE_IMAGE_REMOTE: bool = False
     DOCLING_OCR_ENABLED: bool = False  # Enable OCR for docling parsers (PDF, images)
     DOCLING_OCR_ATTACHMENTS_ENABLED: bool = False  # Enable OCR for docling when parsing attachments
-    VECTOR_STORE: str = (
-        "faiss"  #  "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb" or "pgvector"
-    )
+    VECTOR_STORE: str = "faiss"  #  "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb" or "pgvector"
     RETRIEVERS_ENABLED: list = ["classic_rag"]
     AGENT_NAME: str = "classic"
     FALLBACK_LLM_PROVIDER: Optional[str] = None  # provider for fallback llm
@@ -55,12 +47,8 @@ class Settings(BaseSettings):
     FALLBACK_LLM_API_KEY: Optional[str] = None  # api key for fallback llm
 
     # Google Drive integration
-    GOOGLE_CLIENT_ID: Optional[str] = (
-        None  # Replace with your actual Google OAuth client ID
-    )
-    GOOGLE_CLIENT_SECRET: Optional[str] = (
-        None  # Replace with your actual Google OAuth client secret
-    )
+    GOOGLE_CLIENT_ID: Optional[str] = None  # Replace with your actual Google OAuth client ID
+    GOOGLE_CLIENT_SECRET: Optional[str] = None  # Replace with your actual Google OAuth client secret
     CONNECTOR_REDIRECT_BASE_URI: Optional[str] = (
         "http://127.0.0.1:7091/api/connectors/callback"  ##add redirect url as it is to your provider's console(gcp)
     )
@@ -72,7 +60,7 @@ class Settings(BaseSettings):
     MICROSOFT_AUTHORITY: Optional[str] = None  # e.g., "https://login.microsoftonline.com/{tenant_id}"
 
     # GitHub source
-    GITHUB_ACCESS_TOKEN: Optional[str] = None # PAT token with read repo access
+    GITHUB_ACCESS_TOKEN: Optional[str] = None  # PAT token with read repo access
 
     # LLM Cache
     CACHE_REDIS_URL: str = "redis://localhost:6379/2"
@@ -90,16 +78,13 @@ class Settings(BaseSettings):
     GROQ_API_KEY: Optional[str] = None
     HUGGINGFACE_API_KEY: Optional[str] = None
     OPEN_ROUTER_API_KEY: Optional[str] = None
+    NOVITA_API_KEY: Optional[str] = None
 
     OPENAI_API_BASE: Optional[str] = None  # azure openai api base url
     OPENAI_API_VERSION: Optional[str] = None  # azure openai api version
     AZURE_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for answering
-    AZURE_EMBEDDINGS_DEPLOYMENT_NAME: Optional[str] = (
-        None  # azure deployment name for embeddings
-    )
-    OPENAI_BASE_URL: Optional[str] = (
-        None  # openai base url for open ai compatable models
-    )
+    AZURE_EMBEDDINGS_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for embeddings
+    OPENAI_BASE_URL: Optional[str] = None  # openai base url for open ai compatable models
 
     # elasticsearch
     ELASTIC_CLOUD_ID: Optional[str] = None  # cloud id for elasticsearch
@@ -141,9 +126,7 @@ class Settings(BaseSettings):
 
     # LanceDB vectorstore config
     LANCEDB_PATH: str = "./data/lancedb"  # Path where LanceDB stores its local data
-    LANCEDB_TABLE_NAME: Optional[str] = (
-        "docsgpts"  # Name of the table to use for storing vectors
-    )
+    LANCEDB_TABLE_NAME: Optional[str] = "docsgpts"  # Name of the table to use for storing vectors
 
     FLASK_DEBUG_MODE: bool = False
     STORAGE_TYPE: str = "local"  # local or s3
@@ -180,6 +163,7 @@ class Settings(BaseSettings):
         "GOOGLE_API_KEY",
         "GROQ_API_KEY",
         "HUGGINGFACE_API_KEY",
+        "NOVITA_API_KEY",
         "EMBEDDINGS_KEY",
         "FALLBACK_LLM_API_KEY",
         "QDRANT_API_KEY",

application/llm/base.py

@@ -16,22 +16,61 @@ class BaseLLM(ABC):
         agent_id=None,
         model_id=None,
         base_url=None,
+        backup_models=None,
     ):
         self.decoded_token = decoded_token
         self.agent_id = str(agent_id) if agent_id else None
         self.model_id = model_id
         self.base_url = base_url
         self.token_usage = {"prompt_tokens": 0, "generated_tokens": 0}
+        self._backup_models = backup_models or []
         self._fallback_llm = None
-        self._fallback_sequence_index = 0
 
     @property
     def fallback_llm(self):
-        """Lazy-loaded fallback LLM from FALLBACK_* settings."""
-        if self._fallback_llm is None and settings.FALLBACK_LLM_PROVIDER:
-            try:
-                from application.llm.llm_creator import LLMCreator
+        """Lazy-loaded fallback LLM: tries per-agent backup models first,
+        then the global FALLBACK_* settings."""
+        if self._fallback_llm is not None:
+            return self._fallback_llm
 
+        from application.llm.llm_creator import LLMCreator
+        from application.core.model_utils import (
+            get_provider_from_model_id,
+            get_api_key_for_provider,
+        )
+
+        # Try per-agent backup models first
+        for backup_model_id in self._backup_models:
+            try:
+                provider = get_provider_from_model_id(backup_model_id)
+                if not provider:
+                    logger.warning(
+                        f"Could not resolve provider for backup model: {backup_model_id}"
+                    )
+                    continue
+                api_key = get_api_key_for_provider(provider)
+                self._fallback_llm = LLMCreator.create_llm(
+                    provider,
+                    api_key=api_key,
+                    user_api_key=getattr(self, "user_api_key", None),
+                    decoded_token=self.decoded_token,
+                    model_id=backup_model_id,
+                    agent_id=self.agent_id,
+                )
+                logger.info(
+                    f"Fallback LLM initialized from agent backup model: "
+                    f"{provider}/{backup_model_id}"
+                )
+                return self._fallback_llm
+            except Exception as e:
+                logger.warning(
+                    f"Failed to initialize backup model {backup_model_id}: {str(e)}"
+                )
+                continue
+
+        # Fall back to global FALLBACK_* settings
+        if settings.FALLBACK_LLM_PROVIDER:
+            try:
                 self._fallback_llm = LLMCreator.create_llm(
                     settings.FALLBACK_LLM_PROVIDER,
                     api_key=settings.FALLBACK_LLM_API_KEY or settings.API_KEY,
@@ -41,12 +80,14 @@ class BaseLLM(ABC):
                     agent_id=self.agent_id,
                 )
                 logger.info(
-                    f"Fallback LLM initialized: {settings.FALLBACK_LLM_PROVIDER}/{settings.FALLBACK_LLM_NAME}"
+                    f"Fallback LLM initialized from global settings: "
+                    f"{settings.FALLBACK_LLM_PROVIDER}/{settings.FALLBACK_LLM_NAME}"
                 )
             except Exception as e:
                 logger.error(
                     f"Failed to initialize fallback LLM: {str(e)}", exc_info=True
                 )
+
         return self._fallback_llm
 
     @staticmethod
@@ -74,20 +115,60 @@ class BaseLLM(ABC):
                 method = decorator(method)
             return method(self, *args, **kwargs)
 
+        is_stream = "stream" in method_name
+
+        if is_stream:
+            return self._stream_with_fallback(
+                decorated_method, method_name, *args, **kwargs
+            )
+
         try:
             return decorated_method()
         except Exception as e:
             if not self.fallback_llm:
                 logger.error(f"Primary LLM failed and no fallback configured: {str(e)}")
                 raise
+            fallback = self.fallback_llm
             logger.warning(
-                f"Primary LLM failed. Falling back to {settings.FALLBACK_LLM_PROVIDER}/{settings.FALLBACK_LLM_NAME}. Error: {str(e)}"
+                f"Primary LLM failed. Falling back to "
+                f"{fallback.model_id}. Error: {str(e)}"
             )
 
             fallback_method = getattr(
-                self.fallback_llm, method_name.replace("_raw_", "")
+                fallback, method_name.replace("_raw_", "")
             )
-            return fallback_method(*args, **kwargs)
+            fallback_kwargs = {**kwargs, "model": fallback.model_id}
+            return fallback_method(*args, **fallback_kwargs)
+
+    def _stream_with_fallback(
+        self, decorated_method, method_name, *args, **kwargs
+    ):
+        """
+        Wrapper generator that catches mid-stream errors and falls back.
+
+        Unlike non-streaming calls where exceptions are raised immediately,
+        streaming generators raise exceptions during iteration. This wrapper
+        ensures that if the primary LLM fails at any point during streaming
+        (creation or mid-stream), we fall back to the backup model.
+        """
+        try:
+            yield from decorated_method()
+        except Exception as e:
+            if not self.fallback_llm:
+                logger.error(
+                    f"Primary LLM failed and no fallback configured: {str(e)}"
+                )
+                raise
+            fallback = self.fallback_llm
+            logger.warning(
+                f"Primary LLM failed mid-stream. Falling back to "
+                f"{fallback.model_id}. Error: {str(e)}"
+            )
+            fallback_method = getattr(
+                fallback, method_name.replace("_raw_", "")
+            )
+            fallback_kwargs = {**kwargs, "model": fallback.model_id}
+            yield from fallback_method(*args, **fallback_kwargs)
 
     def gen(self, model, messages, stream=False, tools=None, *args, **kwargs):
         decorators = [gen_token_usage, gen_cache]

application/llm/handlers/handler_creator.py

@@ -7,6 +7,7 @@ class LLMHandlerCreator:
     handlers = {
         "openai": OpenAILLMHandler,
         "google": GoogleLLMHandler,
+        "novita": OpenAILLMHandler,  # Novita uses OpenAI-compatible API
         "default": OpenAILLMHandler,
     }
 

application/llm/llm_creator.py

@@ -38,6 +38,7 @@ class LLMCreator:
         decoded_token,
         model_id=None,
         agent_id=None,
+        backup_models=None,
         *args,
         **kwargs,
     ):
@@ -59,6 +60,7 @@ class LLMCreator:
             model_id=model_id,
             agent_id=agent_id,
             base_url=base_url,
+            backup_models=backup_models,
             *args,
             **kwargs,
         )

application/llm/novita.py

@@ -1,13 +1,13 @@
 from application.core.settings import settings
 from application.llm.openai import OpenAILLM
 
-NOVITA_BASE_URL = "https://api.novita.ai/v3/openai"
+NOVITA_BASE_URL = "https://api.novita.ai/openai"
 
 
 class NovitaLLM(OpenAILLM):
     def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs):
         super().__init__(
-            api_key=api_key or settings.API_KEY,
+            api_key=api_key or settings.NOVITA_API_KEY or settings.API_KEY,
             user_api_key=user_api_key,
             base_url=base_url or NOVITA_BASE_URL,
             *args,

application/llm/openai.py

@@ -101,9 +101,13 @@ class OpenAILLM(BaseLLM):
                     for item in content:
                         if "function_call" in item:
                             # Function calls need their own message
-                            cleaned_args = self._remove_null_values(
-                                item["function_call"]["args"]
-                            )
+                            args = item["function_call"]["args"]
+                            if isinstance(args, str):
+                                try:
+                                    args = json.loads(args)
+                                except (json.JSONDecodeError, TypeError):
+                                    pass
+                            cleaned_args = self._remove_null_values(args)
                             tool_call = {
                                 "id": item["function_call"]["call_id"],
                                 "type": "function",

application/prompts/agentic/creative.txt

@@ -0,0 +1,16 @@
+You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you,
+be proactive and fill in missing information.
+Users can Upload documents for your context as attachments or sources via UI using the Conversation input box.
+If appropriate, your answers can include code examples, formatted as follows:
+```(language)
+(code)
+```
+Users are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses.
+Try to respond with mermaid charts if visualization helps with users queries.
+You effectively utilize chat history, ensuring relevant and tailored responses.
+You have access to a search tool that searches the user's uploaded documents and knowledge base.
+Use the search_internal tool to find relevant information before answering questions.
+You may search multiple times with different queries if needed.
+Do not guess when documents are available — search first, then answer based on what you find.
+If no relevant documents are found, use your general knowledge and tool capabilities.
+Allow yourself to be very creative and use your imagination.

application/prompts/agentic/default.txt

@@ -0,0 +1,15 @@
+You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you,
+be proactive and fill in missing information.
+Users can Upload documents for your context as attachments or sources via UI using the Conversation input box.
+If appropriate, your answers can include code examples, formatted as follows:
+```(language)
+(code)
+```
+Users are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses.
+Try to respond with mermaid charts if visualization helps with users queries.
+You effectively utilize chat history, ensuring relevant and tailored responses.
+You have access to a search tool that searches the user's uploaded documents and knowledge base.
+Use the search_internal tool to find relevant information before answering questions.
+You may search multiple times with different queries if needed.
+Do not guess when documents are available — search first, then answer based on what you find.
+If no relevant documents are found, use your general knowledge and tool capabilities.

application/prompts/agentic/strict.txt

@@ -0,0 +1,16 @@
+You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you,
+be proactive and fill in missing information.
+Users can Upload documents for your context as attachments or sources via UI using the Conversation input box.
+If appropriate, your answers can include code examples, formatted as follows:
+```(language)
+(code)
+```
+Users are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses.
+Try to respond with mermaid charts if visualization helps with users queries.
+You effectively utilize chat history, ensuring relevant and tailored responses.
+You have access to a search tool that searches the user's uploaded documents and knowledge base.
+Use the search_internal tool to find relevant information before answering questions.
+You may search multiple times with different queries if needed.
+You MUST search before answering any factual question. Do not guess or use general knowledge when documents are available.
+If you dont have enough information from the search results or tools, answer "I don't know" or "I don't have enough information".
+Never make up information or provide false information!

application/prompts/react_final_prompt.txt

@@ -1,3 +0,0 @@
-Query: {query}
-Observations: {observations}
-Now, using the insights from the observations, formulate a well-structured and precise final answer.

application/prompts/react_planning_prompt.txt

@@ -1,13 +0,0 @@
-You are an AI assistant and talk like you're thinking out loud. Given the following query, outline a concise thought process that includes key steps and considerations necessary for effective analysis and response. Avoid pointwise formatting. The goal is to break down the query into manageable components without excessive detail, focusing on clarity and logical progression.
-
-Include the following elements in your thought and execution process:
-1. Identify the main objective of the query.
-2. Determine any relevant context or background information needed to understand the query.
-3. List potential approaches or methods to address the query.
-4. Highlight any critical factors or constraints that may influence the outcome.
-5. Plan with available tools to help you with the analysis but dont execute them. Tools will be executed by another AI.
-
-Query: {query}  
-Summaries: {summaries}
-Prompt: {prompt}
-Observations(potentially previous tool calls): {observations}

application/prompts/research/clarification.txt

@@ -0,0 +1,23 @@
+You are a research assistant evaluating whether a user's question is clear enough to begin in-depth research.
+
+Decide whether the question can be researched as-is, or whether you need to ask clarifying questions first.
+
+Proceed WITHOUT clarification if:
+- The question has a clear topic and intent
+- There is enough context to form a research plan
+- The question is broad but can be broken into sub-topics
+- Documents/sources are available to search
+
+Ask for clarification ONLY if:
+- The question is critically ambiguous (e.g., "review the thing" with no context about what "thing" means)
+- Key information is missing that would make research impossible (not just imperfect)
+- The user seems to reference something specific but hasn't provided it
+
+Err on the side of proceeding. Most questions are clear enough to start researching.
+
+You MUST respond with ONLY a valid JSON object (no markdown, no code fences):
+{
+    "needs_clarification": true or false,
+    "questions": ["question 1", "question 2"] (only if needs_clarification is true, max 3 questions),
+    "reason": "brief explanation of why clarification is or isn't needed"
+}

application/prompts/research/planning.txt

@@ -0,0 +1,23 @@
+You are a research planner. Your job is to analyze the user's question and create a focused research plan.
+
+IMPORTANT: Every step must be a concrete research action — something you can search for and find information about. Never generate steps that ask the user for more information or request documents. Work with what you have.
+
+Assess the question's complexity:
+- "simple": Can be answered with 1-2 focused searches (e.g., "What is our refund policy?")
+- "moderate": Needs 3-4 searches across different topics (e.g., "Compare our pricing across product lines")
+- "complex": Requires 5-6 searches with synthesis (e.g., "Audit our compliance docs against regulation X")
+
+Break the question into sub-questions appropriate to its complexity. Don't over-decompose simple questions.
+
+Consider:
+- What distinct aspects of the question need separate investigation?
+- What order should they be investigated in?
+- Are there any dependencies between sub-questions?
+
+You MUST respond with ONLY a valid JSON object in this exact format (no markdown, no code fences):
+{
+    "complexity": "simple|moderate|complex",
+    "steps": [
+        {"query": "specific sub-question to investigate", "rationale": "why this step is needed"}
+    ]
+}

application/prompts/research/step.txt

@@ -0,0 +1,13 @@
+You are a research agent investigating a specific topic. Your goal is to find comprehensive, accurate information.
+
+Your current research task: {step_query}
+
+Instructions:
+1. Use the available tools to search for and gather relevant information.
+2. If you have a search_internal tool, use it to search the knowledge base. You may search multiple times with different queries.
+3. If you have a reason_think tool, use it to analyze what you've found before drawing conclusions.
+4. After gathering sufficient information, provide a detailed summary of your findings.
+5. Cite specific documents and passages you found. Reference sources by their titles or filenames.
+6. If you cannot find relevant information through tools, use your general knowledge but clearly indicate this.
+
+Be thorough — prefer completeness over brevity. Include all relevant details you find.

application/prompts/research/synthesis.txt

@@ -0,0 +1,22 @@
+You are compiling a comprehensive research report based on multiple research steps.
+
+Original question: {question}
+
+Research plan:
+{plan_summary}
+
+Intermediate findings:
+{findings}
+
+Write a well-structured, thorough report that:
+1. Directly addresses the user's original question
+2. Synthesizes findings from all research steps into a coherent narrative
+3. Uses inline citations [N] for every factual claim, where N maps to the source list below
+4. Highlights key insights and connections across different research steps
+5. Notes any gaps or areas where information was insufficient
+6. Includes a "References" section at the end listing all cited sources
+
+Available sources for citation:
+{references}
+
+Format the report with clear headings and sections. Be comprehensive but well-organized.

application/seed/config/premade_agents.yaml

@@ -18,7 +18,7 @@ agents:
   - name: "Researcher"
     description: "A specialized research agent that performs deep dives into subjects."
     image: "https://d3dg1063dc54p9.cloudfront.net/imgs/agents/agent-researcher.svg"
-    agent_type: "react"
+    agent_type: "classic"
     prompt:
       name: "Researcher-Agent"
       content: |

application/templates/template_engine.py

@@ -3,12 +3,12 @@ from typing import Any, Dict, List, Optional, Set
 
 from jinja2 import (
     ChainableUndefined,
-    Environment,
     nodes,
     select_autoescape,
     TemplateSyntaxError,
 )
-from jinja2.exceptions import UndefinedError
+from jinja2.exceptions import SecurityError, UndefinedError
+from jinja2.sandbox import SandboxedEnvironment
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +23,7 @@ class TemplateEngine:
     """Jinja2-based template engine for dynamic prompt rendering"""
 
     def __init__(self):
-        self._env = Environment(
+        self._env = SandboxedEnvironment(
             undefined=ChainableUndefined,
             trim_blocks=True,
             lstrip_blocks=True,
@@ -57,6 +57,10 @@ class TemplateEngine:
             error_msg = f"Undefined variable in template: {e.message}"
             logger.error(error_msg)
             raise TemplateRenderError(error_msg) from e
+        except SecurityError as e:
+            error_msg = f"Template security violation: {str(e)}"
+            logger.error(error_msg)
+            raise TemplateRenderError(error_msg) from e
         except Exception as e:
             error_msg = f"Template rendering failed: {str(e)}"
             logger.error(error_msg)

docs/content/Agents/basics.mdx

@@ -44,36 +44,40 @@ The main set of instructions or system [prompt](/Guides/Customising-prompts) tha
 
 ## Understanding Agent Types
 
-DocsGPT allows for different "types" of agents, each with a distinct way of processing information and generating responses. The code for these agent types can be found in the `application/agents/` directory.
+DocsGPT supports several agent types, each with a distinct way of processing information. The code for these can be found in the `application/agents/` directory.
 
-### 1. Classic Agent (`classic_agent.py`)
+### 1. Classic Agent
 
-**How it works:** The Classic Agent follows a traditional Retrieval Augmented Generation (RAG) approach.
-    1.  **Retrieve:** When a query is made, it first searches the selected Source documents for relevant information.
-    2.  **Augment:** This retrieved data is then added to the context, along with the main Prompt and the user's query.
-    3.  **Generate:** The LLM generates a response based on this augmented context. It can also utilize any configured tools if the LLM decides they are necessary.
+The Classic Agent follows a traditional Retrieval Augmented Generation (RAG) approach: it retrieves relevant document chunks, augments the prompt context with them, and generates a response. It can also use configured tools if the LLM decides they are necessary.
 
-**Best for:**
-    * Direct question-answering over a specific set of documents.
-    * Tasks where the primary goal is to extract and synthesize information from the provided sources.
-    * Simpler tool integrations where the decision to use a tool is straightforward.
+**Best for:** Direct question-answering over a specific set of documents and straightforward tool use.
 
-### 2. ReAct Agent (`react_agent.py`)
+### 2. Agentic Agent
 
-**How it works:** The ReAct Agent employs a more sophisticated "Reason and Act" framework. This involves a multi-step process:
-    1.  **Plan (Thought):** Based on the query, its prompt, and available tools/sources, the LLM first generates a plan or a sequence of thoughts on how to approach the problem. You might see this output as a "thought" process during generation.
-    2.  **Act:** The agent then executes actions based on this plan. This might involve querying its sources, using a tool, or performing internal reasoning.
-    3.  **Observe:** It gathers observations from the results of its actions (e.g., data from a tool, snippets from documents).
-    4.  **Repeat (if necessary):** Steps 2 and 3 can be repeated as the agent refines its approach or gathers more information.
-    5.  **Conclude:** Finally, it generates the final answer based on the initial query and all accumulated observations.
+Unlike Classic which pre-fetches documents into the prompt, the Agentic Agent gives the LLM an `internal_search` tool so it can decide **when, what, and whether** to search. This means the LLM controls its own retrieval — it can search multiple times, refine queries, or skip retrieval entirely if the question doesn't need it.
 
-**Best for:**
-    * More complex tasks that require multi-step reasoning or problem-solving.
-    * Scenarios where the agent needs to dynamically decide which tools to use and in what order, based on intermediate results.
-    * Interactive tasks where the agent needs to "think" through a problem.
+**Best for:** Tasks where the agent needs to dynamically decide how to gather information, use multiple tools in sequence, or combine retrieval with external tool calls.
+
+### 3. Research Agent
+
+A multi-phase agent designed for in-depth research tasks:
+1. **Clarification** — Determines if the question needs clarification before proceeding.
+2. **Planning** — Decomposes the question into research steps with adaptive depth based on complexity.
+3. **Research** — Executes each step, calling tools and refining queries as needed.
+4. **Synthesis** — Compiles findings into a final cited report.
+
+Includes budget controls for max steps, timeout, and token limits to keep research bounded.
+
+**Best for:** Complex questions that require multi-step investigation, gathering information from multiple sources, and producing structured reports with citations.
+
+### 4. Workflow Agent
+
+Executes predefined workflows composed of connected nodes (AI Agent, Set State, Condition). See the [Workflow Nodes](/Agents/nodes) page for details on building workflows.
+
+**Best for:** Structured, multi-step processes with branching logic and shared state between steps.
 
 <Callout type="info">
-Developers looking to introduce new agent architectures can explore the `application/agents/` directory. `classic_agent.py` and `react_agent.py` serve as excellent starting points, demonstrating how to inherit from `BaseAgent` and structure agent logic.
+The legacy "ReAct" agent type is still accepted for backwards compatibility but maps to the Classic Agent internally. New agents should use Classic, Agentic, or Research instead.
 </Callout>
 
 ## Navigating and Managing Agents in DocsGPT

docs/content/Deploying/Amazon-Lightsail.mdx

@@ -70,9 +70,9 @@ Inside the DocsGPT folder create a `.env` file and copy the contents of `.env_sa
 Make sure your `.env` file looks like this:
 
 ```
-OPENAI_API_KEY=(Your OpenAI API key)
+API_KEY=<Your LLM API key>
+LLM_NAME=docsgpt
 VITE_API_STREAMING=true
-SELF_HOSTED_MODEL=false
 ```
 
 To save the file, press CTRL+X, then Y, and then ENTER.

docs/content/Deploying/DocsGPT-Settings.mdx

@@ -104,7 +104,7 @@ DocsGPT can transcribe audio in two places:
 - Voice input in the chat.
 - Audio file ingestion. Uploaded `.wav`, `.mp3`, `.m4a`, `.ogg`, and `.webm` files are transcribed first and then passed through the normal parser, chunking, embedding, and indexing pipeline.
 
-For an end-to-end walkthrough, see the [Speech and Audio Guide](/Guides/speech-and-audio).
+The settings below control speech-to-text behaviour for both voice input and audio file ingestion.
 
 | Setting | Purpose | Typical values |
 | --- | --- | --- |
@@ -214,6 +214,31 @@ If you have configured `AUTH_TYPE=simple_jwt`, the DocsGPT frontend will prompt
   }}
 />
 
+## S3 Storage Backend
+
+By default DocsGPT stores files locally. Set `STORAGE_TYPE=s3` to use Amazon S3 instead.
+
+| Setting | Description | Default |
+| --- | --- | --- |
+| `STORAGE_TYPE` | `local` or `s3` | `local` |
+| `S3_BUCKET_NAME` | S3 bucket name | `docsgpt-test-bucket` |
+| `SAGEMAKER_ACCESS_KEY` | AWS access key ID | — |
+| `SAGEMAKER_SECRET_KEY` | AWS secret access key | — |
+| `SAGEMAKER_REGION` | AWS region | — |
+| `URL_STRATEGY` | `backend` (proxy through API) or `s3` (direct S3 URLs) | `backend` |
+
+The S3 credentials use `SAGEMAKER_*` variable names because they are shared with the SageMaker integration.
+
+```env
+STORAGE_TYPE=s3
+S3_BUCKET_NAME=your-bucket-name
+SAGEMAKER_ACCESS_KEY=your-aws-access-key-id
+SAGEMAKER_SECRET_KEY=your-aws-secret-access-key
+SAGEMAKER_REGION=us-east-1
+```
+
+Your IAM user needs these permissions on the bucket: `s3:PutObject`, `s3:GetObject`, `s3:DeleteObject`, `s3:ListBucket`, `s3:HeadObject`.
+
 ## Exploring More Settings
 
 These are just the basic settings to get you started. The `settings.py` file contains many more advanced options that you can explore to further customize DocsGPT, such as:

docs/content/Deploying/Railway.mdx

@@ -86,13 +86,9 @@ Make sure your `.env` file looks like this:
   
 
 ```
-
-OPENAI_API_KEY=(Your OpenAI API key)
-
+API_KEY=<Your LLM API key>
+LLM_NAME=docsgpt
 VITE_API_STREAMING=true
-
-SELF_HOSTED_MODEL=false
-
 ```
 
   

docs/content/Extensions/api-key-guide.mdx

@@ -11,18 +11,18 @@ DocsGPT API keys are essential for developers and users who wish to integrate th
 
 After uploading your document, you can obtain an API key either through the graphical user interface or via an API call:
 
-- **Graphical User Interface:** Navigate to the Settings section of the DocsGPT web app, find the API Keys option, and press 'Create New' to generate your key.
-- **API Call:** Alternatively, you can use the `/api/create_api_key` endpoint to create a new API key. For detailed instructions, visit [DocsGPT API Documentation](https://gptcloud.arc53.com/).
+- **Graphical User Interface:** Navigate to the Settings section of the DocsGPT web app, find the Agents option, and press 'Create New' to generate a new agent (which includes an API key).
+- **API Call:** Alternatively, you can use the `/api/create_agent` endpoint to create a new agent. An API key is automatically generated for each agent. For detailed instructions, visit [DocsGPT API Documentation](https://gptcloud.arc53.com/).
 
 ## Understanding Key Variables
 
-Upon creating your API key, you will encounter several key variables. Each serves a specific purpose:
+Upon creating your agent, you will encounter several key variables. Each serves a specific purpose:
 
-- **Name:** Assign a name to your API key for easy identification.
-- **Source:** Indicates the source document(s) linked to your API key, which DocsGPT will use to generate responses.
-- **ID:** A unique identifier for your API key. You can view this by making a call to `/api/get_api_keys`.
-- **Key:** The API key itself, which will be used in your application to authenticate API requests.
+- **Name:** Assign a name to your agent for easy identification.
+- **Source:** Indicates the source document(s) linked to your agent, which DocsGPT will use to generate responses.
+- **ID:** A unique identifier for your agent. You can view this by making a call to `/api/get_agents`.
+- **Key:** The API key for the agent, which will be used in your application to authenticate API requests.
 
-With your API key ready, you can now integrate DocsGPT into your application, such as the DocsGPT Widget or any other software, via `/api/answer` or `/stream` endpoints. The source document is preset with the API key, allowing you to bypass fields like `selectDocs` and `active_docs` during implementation.
+With your API key ready, you can now integrate DocsGPT into your application, such as the DocsGPT Widget or any other software, via `/api/answer` or `/stream` endpoints. The source document is preset with the agent, allowing you to bypass fields like `selectDocs` and `active_docs` during implementation.
 
 Congratulations on taking the first step towards enhancing your applications with DocsGPT!

docs/content/Guides/Architecture.mdx

@@ -64,7 +64,7 @@ flowchart LR
 *   **Technology:** Supports multiple vector databases.
 *   **Responsibility:** Vector Stores are used to store and retrieve vector embeddings of document chunks. This enables semantic search and retrieval of relevant document snippets in response to user queries.
 *   **Key Features:**
-    *   Supports vector databases including FAISS, Elasticsearch, Qdrant, Milvus, and LanceDB.
+    *   Supports vector databases including FAISS, Elasticsearch, Qdrant, Milvus, MongoDB Atlas Vector Search, and pgvector.
     *   Provides storage and indexing of high-dimensional vector embeddings.
     *   Enables editing and updating of vector indexes including specific chunks.
 

docs/content/Guides/How-to-train-on-other-documentation.mdx

@@ -16,7 +16,7 @@ Training on other documentation sources can greatly enhance the versatility and
 Make sure you have the document on which you want to train on  ready with you on the device which you are using .You can also use links to the documentation to train on.
 
 <Callout type="warning" emoji="⚠️">
- Note: The document should be either of the given file formats .pdf, .txt, .rst, .docx, .md, .zip and  limited to 25mb.You can also train using the link of the documentation.
+ Note: Supported file formats include .pdf, .txt, .rst, .docx, .md, .mdx, .csv, .epub, .html, .json, .xlsx, .pptx, .png, .jpg, .jpeg, and audio files (.wav, .mp3, .m4a, .ogg, .webm). You can also train using the link of the documentation.
 
 </Callout>
 

docs/content/Guides/How-to-use-different-LLM.mdx

@@ -35,8 +35,34 @@ Choose the LLM of your choice.
 For open source version please edit `LLM_PROVIDER`, `LLM_NAME` and others in the .env file. Refer to [⚙️ App Configuration](/Deploying/DocsGPT-Settings) for more information.
 ### Step 2
 Visit [☁️ Cloud Providers](/Models/cloud-providers) for the updated list of online models. Make sure you have the right API_KEY and correct LLM_PROVIDER.
-For self-hosted please visit [🖥️ Local Inference](/Models/local-inference). 
+For self-hosted please visit [🖥️ Local Inference](/Models/local-inference).
 </Steps>
 
+## Fallback LLM
+
+DocsGPT can automatically switch to a fallback LLM when the primary model fails, including mid-stream. This works with both streaming and non-streaming requests.
+
+**Fallback order:**
+1. Per-agent backup models (other models configured on the same agent)
+2. Global fallback (`FALLBACK_LLM_*` env vars below)
+3. Error returned if all fail
+
+| Setting | Description | Default |
+| --- | --- | --- |
+| `FALLBACK_LLM_PROVIDER` | Provider name (e.g., `openai`, `anthropic`, `google`) | — |
+| `FALLBACK_LLM_NAME` | Model name (e.g., `gpt-4o`, `claude-sonnet-4-20250514`) | — |
+| `FALLBACK_LLM_API_KEY` | API key for the fallback provider | Falls back to `API_KEY` |
+
+All three (`FALLBACK_LLM_PROVIDER`, `FALLBACK_LLM_NAME`, and an API key) must resolve for the global fallback to activate.
+
+```env
+FALLBACK_LLM_PROVIDER=anthropic
+FALLBACK_LLM_NAME=claude-sonnet-4-20250514
+FALLBACK_LLM_API_KEY=sk-ant-your-anthropic-key
+```
+
+<Callout type="info">
+For maximum resilience, use a fallback provider from a different cloud than your primary. Each agent can also have multiple models configured — the other models are tried first before the global fallback.
+</Callout>
 
 

docs/content/Guides/Integrations/_meta.js

@@ -2,5 +2,13 @@ export default {
   "google-drive-connector": {
     "title": "🔗 Google Drive",
     "href": "/Guides/Integrations/google-drive-connector"
+  },
+  "sharepoint-connector": {
+    "title": "🔗 SharePoint / OneDrive",
+    "href": "/Guides/Integrations/sharepoint-connector"
+  },
+  "mcp-tool-integration": {
+    "title": "🔗 MCP Tools",
+    "href": "/Guides/Integrations/mcp-tool-integration"
   }
 }

docs/content/Guides/Integrations/mcp-tool-integration.mdx

@@ -0,0 +1,66 @@
+---
+title: MCP Tool Integration
+description: Connect external tools to DocsGPT agents using the Model Context Protocol (MCP) standard.
+---
+
+import { Callout } from 'nextra/components'
+import { Steps } from 'nextra/components'
+
+# MCP Tool Integration
+
+The [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) integration lets you connect external tool servers to DocsGPT. Your agents can then discover and call tools provided by those servers during conversations — for example, querying a CRM, running code, or accessing a database.
+
+## Setup
+
+<Steps>
+
+### Step 1: Configure Environment Variables (Optional)
+
+Only needed if your MCP servers use OAuth authentication:
+
+```env
+MCP_OAUTH_REDIRECT_URI=https://yourdomain.com/api/mcp_server/callback
+```
+
+If not set, falls back to `API_URL/api/mcp_server/callback`.
+
+### Step 2: Add an MCP Server
+
+Go to **Settings** > **Tools** > **Add Tool** > **MCP Server**. Enter the server URL, select an auth type, and click **Test Connection** to verify, then **Save**.
+
+### Step 3: Enable for Your Agent
+
+In your agent configuration, enable the MCP tools you want the agent to use.
+
+</Steps>
+
+## Authentication Types
+
+| Auth Type | Config Fields |
+|-----------|---------------|
+| **None** | — |
+| **Bearer** | `bearer_token` |
+| **API Key** | `api_key`, `api_key_header` (default: `X-API-Key`) |
+| **Basic** | `username`, `password` |
+| **OAuth** | `oauth_scopes` (optional) |
+
+<Callout type="warning">
+For OAuth in production, `MCP_OAUTH_REDIRECT_URI` must be a publicly accessible URL pointing to your DocsGPT backend.
+</Callout>
+
+## API Endpoints
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/api/mcp_server/test` | POST | Test a connection without saving |
+| `/api/mcp_server/save` | POST | Save or update a server configuration |
+| `/api/mcp_server/callback` | GET | OAuth callback handler |
+| `/api/mcp_server/oauth_status/<task_id>` | GET | Poll OAuth flow status |
+| `/api/mcp_server/auth_status` | GET | Batch check auth status for all MCP tools |
+
+## Troubleshooting
+
+- **Connection refused** — Verify the URL and that the server is reachable from your backend.
+- **403 Forbidden** — Check credentials and permissions.
+- **Timed out** — Default is 30s; increase timeout in tool config (max 300s).
+- **OAuth "needs_auth" persists** — Verify `MCP_OAUTH_REDIRECT_URI` is correct and Redis is running.

docs/content/Guides/Integrations/sharepoint-connector.mdx

@@ -0,0 +1,63 @@
+---
+title: SharePoint / OneDrive Connector
+description: Connect your Microsoft SharePoint or OneDrive as an external knowledge base to upload and process files directly.
+---
+
+import { Callout } from 'nextra/components'
+import { Steps } from 'nextra/components'
+
+# SharePoint / OneDrive Connector
+
+Connect your SharePoint or OneDrive account to upload and process files directly as an external knowledge base. Supports Office files, PDFs, text files, CSVs, images, and more. Authentication is handled via Microsoft Entra ID (Azure AD) with automatic token refresh.
+
+## Setup
+
+<Steps>
+
+### Step 1: Create an App Registration in Azure
+
+1. Go to the [Azure Portal](https://portal.azure.com/) > **Microsoft Entra ID** > **App registrations** > **New registration**
+2. Set **Redirect URI** (Web) to:
+   - Local: `http://localhost:7091/api/connectors/callback?provider=share_point`
+   - Production: `https://yourdomain.com/api/connectors/callback?provider=share_point`
+
+### Step 2: Configure API Permissions
+
+In your App Registration, go to **API permissions** > **Add a permission** > **Microsoft Graph** > **Delegated permissions** and add: `Files.Read`, `Files.Read.All`, `Sites.Read.All`. Grant admin consent if possible.
+
+### Step 3: Create a Client Secret
+
+Go to **Certificates & secrets** > **New client secret**. Copy the secret value immediately (it won't be shown again).
+
+### Step 4: Configure Environment Variables
+
+Add to your `.env` file:
+
+```env
+MICROSOFT_CLIENT_ID=your-azure-ad-client-id
+MICROSOFT_CLIENT_SECRET=your-azure-ad-client-secret
+MICROSOFT_TENANT_ID=your-azure-ad-tenant-id
+```
+
+| Variable | Description | Required | Default |
+|----------|-------------|----------|---------|
+| `MICROSOFT_CLIENT_ID` | Application (client) ID from App Registration overview | Yes | — |
+| `MICROSOFT_CLIENT_SECRET` | Client secret value | Yes | — |
+| `MICROSOFT_TENANT_ID` | Directory (tenant) ID | No | `common` |
+| `MICROSOFT_AUTHORITY` | Login endpoint override | No | Auto-constructed |
+
+<Callout type="warning">
+`MICROSOFT_TENANT_ID=common` (the default) allows any Microsoft account to authenticate. Set this to your specific tenant ID in production.
+</Callout>
+
+### Step 5: Restart and Use
+
+Restart your application, then go to the upload section in DocsGPT and select **SharePoint / OneDrive** as the source. You'll be redirected to Microsoft to sign in, then can browse and select files to process.
+
+</Steps>
+
+## Troubleshooting
+
+- **Option not appearing** — Verify `MICROSOFT_CLIENT_ID` and `MICROSOFT_CLIENT_SECRET` are set, then restart.
+- **Authentication failed** — Check that the redirect URI matches exactly, including `?provider=share_point`.
+- **Permission denied** — Ensure admin consent is granted and the user has access to the target files.

docs/content/Guides/My-AI-answers-questions-using-external-knowledge.mdx

@@ -7,20 +7,10 @@ description:
 
 If your AI uses external knowledge and is not explicit enough, it is ok, because we try to make DocsGPT friendly.
 
-But if you want to adjust it, here is a simple way:-
-
-- Got to `application/prompts/chat_combine_prompt.txt`
-
-- And change it to
+But if you want to adjust it, prompts are now managed through the UI and API using a template-based system. See the [Customising Prompts](/Guides/Customising-prompts) guide for details.
 
+To make the AI stricter about staying on-topic, edit your active prompt template (via **Sidebar → Settings → Active Prompt**) to include instructions like:
 
 ```
-
-You are a DocsGPT, friendly and helpful AI assistant by Arc53 that provides help with documents. You give thorough answers with code examples, if possible.
-Write an answer for the question below based on the provided context.
 If the context provides insufficient information, reply "I cannot answer".
-You have access to chat history and can use it to help answer the question.
-----------------
-{summaries}
-
 ```

docs/content/Guides/_meta.js

@@ -29,7 +29,7 @@ export default {
     "title": "OCR",
     "href": "/Guides/ocr"
   },
-  "Integrations": {
+"Integrations": {
     "title": "🔗 Integrations"
   }
 }

docs/content/quickstart.mdx

@@ -70,7 +70,7 @@ The easiest way to launch DocsGPT is using the provided `setup.sh` script. This
    To stop DocsGPT, simply open a new terminal in the `DocsGPT` directory and run:
 
    ```bash
-   docker compose -f deployment/docker-compose.yaml down
+   docker compose -f deployment/docker-compose-hub.yaml down
    ```
    (or the specific `docker compose` command shown at the end of the `setup.sh` execution, which may include optional compose files depending on your choices).
 

extensions/react-widget/custom.d.ts

@@ -1,9 +0,0 @@
-declare module "*.svg" {
-    import * as React from "react";
-  
-    const ReactComponent: React.FunctionComponent<
-      React.SVGProps<SVGSVGElement> & { title?: string }
-    >;
-  
-    export default ReactComponent;
-  }

extensions/react-widget/eslint.config.mjs

@@ -0,0 +1,72 @@
+import js from '@eslint/js'
+import tsParser from '@typescript-eslint/parser'
+import tsPlugin from '@typescript-eslint/eslint-plugin'
+import react from 'eslint-plugin-react'
+import unusedImports from 'eslint-plugin-unused-imports'
+import prettier from 'eslint-plugin-prettier'
+import globals from 'globals'
+
+export default [
+  {
+    ignores: [
+      'node_modules/',
+      'dist/',
+      'prettier.config.cjs',
+      'custom.d.ts',
+      'package-lock.json',
+      'package.json',
+    ],
+  },
+  {
+    files: ['**/*.{js,jsx,ts,tsx}'],
+    languageOptions: {
+      ecmaVersion: 'latest',
+      sourceType: 'module',
+      parser: tsParser,
+      parserOptions: {
+        ecmaFeatures: {
+          jsx: true,
+        },
+      },
+      globals: {
+        ...globals.browser,
+        ...globals.es2021,
+        ...globals.node,
+      },
+    },
+    plugins: {
+      '@typescript-eslint': tsPlugin,
+      react,
+      'unused-imports': unusedImports,
+      prettier,
+    },
+    rules: {
+      ...js.configs.recommended.rules,
+      ...tsPlugin.configs.recommended.rules,
+      ...react.configs.recommended.rules,
+      ...prettier.configs.recommended.rules,
+      'react/prop-types': 'off',
+      'unused-imports/no-unused-imports': 'error',
+      'react/react-in-jsx-scope': 'off',
+      'no-undef': 'off',
+      '@typescript-eslint/no-explicit-any': 'warn',
+      '@typescript-eslint/no-unused-vars': [
+        'warn',
+        { varsIgnorePattern: '^_', argsIgnorePattern: '^_' },
+      ],
+      '@typescript-eslint/no-unused-expressions': 'warn',
+      'prettier/prettier': [
+        'error',
+        {
+          endOfLine: 'auto',
+        },
+      ],
+    },
+    settings: {
+      react: {
+        version: 'detect',
+      },
+    },
+  },
+]
+

extensions/react-widget/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docsgpt",
-  "version": "0.5.1",
+  "version": "0.6.3",
   "private": false,
   "description": "DocsGPT 🦖 is an innovative open-source tool designed to simplify the retrieval of information from project documentation using advanced GPT models 🤖.",
   "source": "./src/index.html",
@@ -35,7 +35,9 @@
     "serve": "parcel serve -p 3000",
     "dev": "parcel -p 3000",
     "test": "jest",
-    "lint": "eslint",
+    "lint": "eslint ./src --ext .jsx,.js,.ts,.tsx",
+    "lint-fix": "eslint ./src --ext .jsx,.js,.ts,.tsx --fix",
+    "format": "prettier ./src --write",
     "check": "tsc --noEmit",
     "ci": "yarn build && yarn test && yarn lint && yarn check"
   },
@@ -50,7 +52,7 @@
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
     "dompurify": "^3.1.5",
-    "flow-bin": "^0.305.0",
+    "flow-bin": "^0.306.0",
     "markdown-it": "^14.1.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
@@ -60,14 +62,24 @@
     "@babel/core": "^7.24.0",
     "@babel/preset-env": "^7.24.0",
     "@babel/preset-react": "^7.23.3",
+    "@eslint/js": "^9.39.4",
     "@parcel/packager-ts": "^2.16.4",
     "@parcel/transformer-typescript-types": "^2.16.4",
     "@types/dompurify": "^3.0.5",
     "@types/markdown-it": "^14.1.2",
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
+    "@typescript-eslint/eslint-plugin": "^8.57.2",
+    "@typescript-eslint/parser": "^8.57.2",
     "babel-loader": "^10.1.1",
+    "eslint": "^9.39.4",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-prettier": "^5.5.5",
+    "eslint-plugin-react": "^7.37.5",
+    "eslint-plugin-unused-imports": "^4.4.1",
+    "globals": "^15.15.0",
     "parcel": "^2.16.4",
+    "prettier": "^3.8.1",
     "process": "^0.11.10",
     "svgo": "^3.3.3",
     "typescript": "^5.3.3"

extensions/react-widget/prettier.config.cjs

@@ -0,0 +1,8 @@
+module.exports = {
+  trailingComma: 'all',
+  tabWidth: 2,
+  semi: true,
+  singleQuote: true,
+  printWidth: 80,
+};
+

extensions/react-widget/src/App.tsx

@@ -1,11 +1,11 @@
-import React from "react"
-import {DocsGPTWidget} from "./components/DocsGPTWidget"
-import {SearchBar} from "./components/SearchBar"
+import React from 'react';
+import { DocsGPTWidget } from './components/DocsGPTWidget';
+import { SearchBar } from './components/SearchBar';
 export const App = () => {
   return (
     <div>
-      <SearchBar/>
-      <DocsGPTWidget/>
+      <SearchBar />
+      <DocsGPTWidget />
     </div>
-  )
-}
+  );
+};

extensions/react-widget/src/assets/dislike.svg

@@ -1,4 +0,0 @@
-<svg width="14" height="14" viewBox="0 0 16 16" fill="current" xmlns="http://www.w3.org/2000/svg">
-<path d="M6.37776 10.1001V12.9C6.37776 13.457 6.599 13.9911 6.99282 14.3849C7.38664 14.7788 7.92077 15 8.47772 15L11.2777 8.70011V1.00025H3.38181C3.04419 0.996436 2.71656 1.11477 2.45929 1.33344C2.20203 1.55212 2.03246 1.8564 1.98184 2.19023L1.01585 8.49012C0.985398 8.69076 0.998931 8.89563 1.05551 9.09053C1.1121 9.28543 1.21038 9.46569 1.34355 9.61884C1.47671 9.77198 1.64159 9.89434 1.82674 9.97744C2.01189 10.0605 2.2129 10.1024 2.41583 10.1001H6.37776ZM11.2777 1.00025H13.1466C13.5428 0.993247 13.9277 1.13195 14.2284 1.39002C14.5291 1.64809 14.7245 2.00758 14.7776 2.40023V7.30014C14.7245 7.69279 14.5291 8.05227 14.2284 8.31035C13.9277 8.56842 13.5428 8.70712 13.1466 8.70011H11.2777" fill="none"/>
-<path d="M11.2777 8.70011L8.47772 15C7.92077 15 7.38664 14.7788 6.99282 14.3849C6.599 13.9911 6.37776 13.457 6.37776 12.9V10.1001H2.41583C2.2129 10.1024 2.01189 10.0605 1.82674 9.97744C1.64159 9.89434 1.47671 9.77198 1.34355 9.61884C1.21038 9.46569 1.1121 9.28543 1.05551 9.09053C0.998931 8.89563 0.985398 8.69076 1.01585 8.49012L1.98184 2.19023C2.03246 1.8564 2.20203 1.55212 2.45929 1.33344C2.71656 1.11477 3.04419 0.996436 3.38181 1.00025H11.2777M11.2777 8.70011V1.00025M11.2777 8.70011H13.1466C13.5428 8.70712 13.9277 8.56842 14.2284 8.31035C14.5291 8.05227 14.7245 7.69279 14.7776 7.30014V2.40023C14.7245 2.00758 14.5291 1.64809 14.2284 1.39002C13.9277 1.13195 13.5428 0.993247 13.1466 1.00025H11.2777" stroke="current" stroke-width="1.4" stroke-linecap="round" stroke-linejoin="round"/>
-</svg>

extensions/react-widget/src/assets/like.svg

@@ -1,4 +0,0 @@
-<svg width="14" height="14" viewBox="0 0 16 16" fill="current" xmlns="http://www.w3.org/2000/svg">
-<path d="M9.39995 5.89997V3.09999C9.39995 2.54304 9.1787 2.0089 8.78487 1.61507C8.39105 1.22125 7.85691 1 7.29996 1L4.49998 7.29996V14.9999H12.3959C12.7336 15.0037 13.0612 14.8854 13.3185 14.6667C13.5757 14.448 13.7453 14.1437 13.7959 13.8099L14.7619 7.50996C14.7924 7.30931 14.7788 7.10444 14.7222 6.90954C14.6657 6.71464 14.5674 6.53437 14.4342 6.38123C14.301 6.22808 14.1362 6.10572 13.951 6.02262C13.7659 5.93952 13.5649 5.89767 13.3619 5.89997H9.39995ZM4.49998 14.9999H2.39999C2.02869 14.9999 1.6726 14.8524 1.41005 14.5899C1.1475 14.3273 1 13.9712 1 13.5999V8.69995C1 8.32865 1.1475 7.97256 1.41005 7.71001C1.6726 7.44746 2.02869 7.29996 2.39999 7.29996H4.49998" fill="none"/>
-<path d="M4.49998 7.29996L7.29996 1C7.85691 1 8.39105 1.22125 8.78487 1.61507C9.1787 2.0089 9.39995 2.54304 9.39995 3.09999V5.89997H13.3619C13.5649 5.89767 13.7659 5.93952 13.951 6.02262C14.1362 6.10572 14.301 6.22808 14.4342 6.38123C14.5674 6.53437 14.6657 6.71464 14.7223 6.90954C14.7788 7.10444 14.7924 7.30931 14.7619 7.50996L13.7959 13.8099C13.7453 14.1437 13.5757 14.448 13.3185 14.6667C13.0612 14.8854 12.7336 15.0037 12.3959 14.9999H4.49998M4.49998 7.29996V14.9999M4.49998 7.29996H2.39999C2.02869 7.29996 1.6726 7.44746 1.41005 7.71001C1.1475 7.97256 1 8.32865 1 8.69995V13.5999C1 13.9712 1.1475 14.3273 1.41005 14.5899C1.6726 14.8524 2.02869 14.9999 2.39999 14.9999H4.49998" stroke="current" stroke-width="1.39999" stroke-linecap="round" stroke-linejoin="round"/>
-</svg>

extensions/react-widget/src/browser.tsx

@@ -1,10 +1,10 @@
 //exports browser ready methods
 
-import { createRoot } from "react-dom/client";
+import { createRoot } from 'react-dom/client';
 
 import { DocsGPTWidget } from './components/DocsGPTWidget';
 import { SearchBar } from './components/SearchBar';
-import React from "react";
+import React from 'react';
 if (typeof window !== 'undefined') {
   const renderWidget = (elementId: string, props = {}) => {
     const root = createRoot(document.getElementById(elementId) as HTMLElement);
@@ -14,9 +14,11 @@ if (typeof window !== 'undefined') {
     const root = createRoot(document.getElementById(elementId) as HTMLElement);
     root.render(<SearchBar {...props} />);
   };
-  (window as any).renderDocsGPTWidget = renderWidget;
+  (window as unknown as Record<string, unknown>).renderDocsGPTWidget =
+    renderWidget;
 
-  (window as any).renderSearchBar = renderSearchBar;
+  (window as unknown as Record<string, unknown>).renderSearchBar =
+    renderSearchBar;
 }
 
 export { DocsGPTWidget, SearchBar };

extensions/react-widget/src/index.ts

@@ -1,3 +1,3 @@
 //exports methods for React
-export {SearchBar} from "./components/SearchBar"
-export { DocsGPTWidget } from "./components/DocsGPTWidget";
+export { SearchBar } from './components/SearchBar';
+export { DocsGPTWidget } from './components/DocsGPTWidget';

extensions/react-widget/src/main.tsx

@@ -1,8 +1,7 @@
-
 //development
-import { createRoot } from "react-dom/client";
-import { App } from "./App";
-import React from "react";
-const container = document.getElementById("app") as HTMLElement;
-const root = createRoot(container)
+import { createRoot } from 'react-dom/client';
+import { App } from './App';
+import React from 'react';
+const container = document.getElementById('app') as HTMLElement;
+const root = createRoot(container);
 root.render(<App />);

extensions/react-widget/src/requests/searchAPI.ts

@@ -1,20 +1,24 @@
-import { Result } from "@/types";
-
-async function getSearchResults(question: string, apiKey: string, apiHost: string, signal: AbortSignal): Promise<Result[]> {
+import { Result } from '@/types';
 
+async function getSearchResults(
+  question: string,
+  apiKey: string,
+  apiHost: string,
+  signal: AbortSignal,
+): Promise<Result[]> {
   const payload = {
     question,
-    api_key: apiKey
+    api_key: apiKey,
   };
 
   try {
     const response = await fetch(`${apiHost}/api/search`, {
-      method: "POST",
+      method: 'POST',
       headers: {
-        "Content-Type": "application/json",
+        'Content-Type': 'application/json',
       },
       body: JSON.stringify(payload),
-      signal: signal
+      signal: signal,
     });
 
     if (!response.ok) {
@@ -23,15 +27,12 @@ async function getSearchResults(question: string, apiKey: string, apiHost: strin
 
     const data: Result[] = await response.json();
     return data;
-
   } catch (error) {
-    if (!(error instanceof DOMException && error.name == "AbortError")) {
-      console.error("Failed to fetch documents:", error);
+    if (!(error instanceof DOMException && error.name == 'AbortError')) {
+      console.error('Failed to fetch documents:', error);
     }
     throw error;
   }
 }
 
-export {
-  getSearchResults
-}
+export { getSearchResults };

extensions/react-widget/src/requests/streamingApi.ts

@@ -1,5 +1,3 @@
-import { FEEDBACK } from "@/types";
-
 interface HistoryItem {
   prompt: string;
   response?: string;
@@ -30,7 +28,9 @@ export function fetchAnswerStreaming({
   history = [],
   conversationId = null,
   apiHost = '',
-  onEvent = () => { console.log("Event triggered, but no handler provided."); }
+  onEvent = () => {
+    console.log('Event triggered, but no handler provided.');
+  },
 }: FetchAnswerStreamingProps): Promise<void> {
   return new Promise<void>((resolve, reject) => {
     const body = {
@@ -38,7 +38,7 @@ export function fetchAnswerStreaming({
       history: JSON.stringify(history),
       conversation_id: conversationId,
       model: 'default',
-      api_key: apiKey
+      api_key: apiKey,
     };
     fetch(apiHost + '/stream', {
       method: 'POST',
@@ -52,7 +52,7 @@ export function fetchAnswerStreaming({
 
         const reader = response.body.getReader();
         const decoder = new TextDecoder('utf-8');
-        let counter = 0;
+        let counter = 0; // eslint-disable-line @typescript-eslint/no-unused-vars
         const processStream = ({
           done,
           value,
@@ -95,12 +95,14 @@ export function fetchAnswerStreaming({
   });
 }
 
-
-export  const sendFeedback = (payload: FeedbackPayload, apiHost: string): Promise<Response> => {
+export const sendFeedback = (
+  payload: FeedbackPayload,
+  apiHost: string,
+): Promise<Response> => {
   return fetch(`${apiHost}/api/feedback`, {
     method: 'POST',
     headers: {
-      'Content-Type': 'application/json'
+      'Content-Type': 'application/json',
     },
     body: JSON.stringify({
       question: payload.question,
@@ -108,7 +110,7 @@ export  const sendFeedback = (payload: FeedbackPayload, apiHost: string): Promis
       feedback: payload.feedback,
       api_key: payload.apikey,
       conversation_id: payload.conversation_id,
-      question_index: payload.question_index
+      question_index: payload.question_index,
     }),
   });
 };

extensions/react-widget/src/types/index.ts

@@ -1,3 +1,30 @@
+import 'styled-components';
+
+declare module 'styled-components' {
+  export interface DefaultTheme {
+    bg: string;
+    text: string;
+    primary: {
+      text: string;
+      bg: string;
+    };
+    secondary: {
+      text: string;
+      bg: string;
+    };
+    /** Present only in SearchBar theme */
+    name?: string;
+    /** Present only in DocsGPTWidget theme (always provided when these styled components render) */
+    dimensions?: {
+      size: string;
+      width: string;
+      height: string;
+      maxWidth?: string;
+      maxHeight?: string;
+    };
+  }
+}
+
 export type MESSAGE_TYPE = 'QUESTION' | 'ANSWER' | 'ERROR';
 
 export type Status = 'idle' | 'loading' | 'failed';
@@ -11,7 +38,7 @@ export interface Query {
   response?: string;
   feedback?: FEEDBACK;
   error?: string;
-  sources?: { title: string; text: string, source:string }[];
+  sources?: { title: string; text: string; source: string }[];
   conversationId?: string | null;
   title?: string | null;
 }
@@ -24,30 +51,34 @@ export interface WidgetProps {
   description?: string;
   heroTitle?: string;
   heroDescription?: string;
-  size?: 'small' | 'medium' | 'large' | {
-    custom: {
-      width: string;
-      height: string;
-      maxWidth?: string;
-      maxHeight?: string;
-    };
-  };
-  theme?:THEME,
-  buttonIcon?:string;
-  buttonText?:string;
-  buttonBg?:string;
-  collectFeedback?:boolean;
+  size?:
+    | 'small'
+    | 'medium'
+    | 'large'
+    | {
+        custom: {
+          width: string;
+          height: string;
+          maxWidth?: string;
+          maxHeight?: string;
+        };
+      };
+  theme?: THEME;
+  buttonIcon?: string;
+  buttonText?: string;
+  buttonBg?: string;
+  collectFeedback?: boolean;
   showSources?: boolean;
   defaultOpen?: boolean;
 }
-export interface WidgetCoreProps extends WidgetProps { 
-  widgetRef?:React.RefObject<HTMLDivElement> | null;
-  handleClose?:React.MouseEventHandler | undefined;
-  isOpen:boolean;
+export interface WidgetCoreProps extends WidgetProps {
+  widgetRef?: React.RefObject<HTMLDivElement> | null;
+  handleClose?: React.MouseEventHandler | undefined;
+  isOpen: boolean;
   prefilledQuery?: string;
 }
 
-export interface SearchBarProps { 
+export interface SearchBarProps {
   apiHost?: string;
   apiKey?: string;
   theme?: THEME;
@@ -57,7 +88,7 @@ export interface SearchBarProps {
 }
 
 export interface Result {
-  text:string;
-  title:string;
-  source:string;
+  text: string;
+  title: string;
+  source: string;
 }

extensions/react-widget/src/utils/helper.ts

@@ -30,11 +30,15 @@ interface ParsedElement {
   tag: string;
 }
 
-export const processMarkdownString = (markdown: string, keyword?: string): ParsedElement[] => {
+export const processMarkdownString = (
+  markdown: string,
+  keyword?: string,
+): ParsedElement[] => {
   const lines = markdown.trim().split('\n');
   const keywordLower = keyword?.toLowerCase();
 
-  const escapeRegExp = (str: string) => str.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
+  const escapeRegExp = (str: string) =>
+    str.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
   const escapedKeyword = keyword ? escapeRegExp(keyword) : '';
   const keywordRegex = keyword ? new RegExp(`(${escapedKeyword})`, 'gi') : null;
 
@@ -56,7 +60,7 @@ export const processMarkdownString = (markdown: string, keyword?: string): Parse
         const codeContent = codeBlockContent.join('\n');
         const parsedElement: ParsedElement = {
           content: codeContent,
-          tag: 'code'
+          tag: 'code',
         };
 
         if (!firstLine) {
@@ -64,7 +68,10 @@ export const processMarkdownString = (markdown: string, keyword?: string): Parse
         }
 
         if (keywordLower && codeContent.toLowerCase().includes(keywordLower)) {
-          parsedElement.content = parsedElement.content.replace(keywordRegex!, '<span class="highlight">$1</span>');
+          parsedElement.content = parsedElement.content.replace(
+            keywordRegex!,
+            '<span class="highlight">$1</span>',
+          );
           matchingLines.push(parsedElement);
         }
       }
@@ -89,30 +96,30 @@ export const processMarkdownString = (markdown: string, keyword?: string): Parse
       content = headingMatch[2];
       parsedElement = {
         content: content,
-        tag: 'heading'
+        tag: 'heading',
       };
     } else if (bulletMatch) {
       content = bulletMatch[1];
       parsedElement = {
         content: content,
-        tag: 'bulletList'
+        tag: 'bulletList',
       };
     } else if (numberedMatch) {
       content = numberedMatch[1];
       parsedElement = {
         content: content,
-        tag: 'numberedList'
+        tag: 'numberedList',
       };
     } else if (blockquoteMatch) {
       content = blockquoteMatch[1];
       parsedElement = {
         content: content,
-        tag: 'blockquote'
+        tag: 'blockquote',
       };
     } else {
       parsedElement = {
         content: content,
-        tag: 'text'
+        tag: 'text',
       };
     }
 
@@ -120,8 +127,14 @@ export const processMarkdownString = (markdown: string, keyword?: string): Parse
       firstLine = parsedElement;
     }
 
-    if (keywordLower && parsedElement.content.toLowerCase().includes(keywordLower)) {
-      parsedElement.content = parsedElement.content.replace(keywordRegex!, '<span class="highlight">$1</span>');
+    if (
+      keywordLower &&
+      parsedElement.content.toLowerCase().includes(keywordLower)
+    ) {
+      parsedElement.content = parsedElement.content.replace(
+        keywordRegex!,
+        '<span class="highlight">$1</span>',
+      );
       matchingLines.push(parsedElement);
     }
   }
@@ -130,7 +143,7 @@ export const processMarkdownString = (markdown: string, keyword?: string): Parse
     const codeContent = codeBlockContent.join('\n');
     const parsedElement: ParsedElement = {
       content: codeContent,
-      tag: 'code'
+      tag: 'code',
     };
 
     if (!firstLine) {
@@ -138,7 +151,10 @@ export const processMarkdownString = (markdown: string, keyword?: string): Parse
     }
 
     if (keywordLower && codeContent.toLowerCase().includes(keywordLower)) {
-      parsedElement.content = parsedElement.content.replace(keywordRegex!, '<span class="highlight">$1</span>');
+      parsedElement.content = parsedElement.content.replace(
+        keywordRegex!,
+        '<span class="highlight">$1</span>',
+      );
       matchingLines.push(parsedElement);
     }
   }

extensions/react-widget/tsconfig.json

@@ -1,9 +1,5 @@
 {
     "compilerOptions": {
-      "baseUrl": ".",
-      "paths": {
-        "@/*": ["src/*", "@/*"]
-      },
       "target": "ES2020",
       "useDefineForClassFields": true,
       "lib": ["ES2020", "DOM", "DOM.Iterable"],
@@ -23,8 +19,11 @@
       "noUnusedLocals": false,
       "noUnusedParameters": false,
       "noFallthroughCasesInSwitch": true,
-      /* The "typeRoots" configuration specifies the locations where 
-         TypeScript looks for type definitions (.d.ts files) to 
+      "paths": {
+        "@/*": ["./src/*"]
+      },
+      /* The "typeRoots" configuration specifies the locations where
+         TypeScript looks for type definitions (.d.ts files) to
          include in the compilation process.*/
          "typeRoots": ["./dist/index.d.ts", "node_modules/@types"]
         },

frontend/src/agents/NewAgent.tsx

@@ -28,6 +28,7 @@ import {
 import PromptsModal from '../preferences/PromptsModal';
 import Prompts from '../settings/Prompts';
 import { UserToolType } from '../settings/types';
+import { getToolDisplayName } from '../utils/toolUtils';
 import AgentPreview from './AgentPreview';
 import { Agent, ToolSummary } from './types';
 import WorkflowBuilder from './workflow/WorkflowBuilder';
@@ -137,7 +138,8 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
   const chunks = ['0', '2', '4', '6', '8', '10'];
   const agentTypes = [
     { label: t('agents.form.agentTypes.classic'), value: 'classic' },
-    { label: t('agents.form.agentTypes.react'), value: 'react' },
+    { label: 'Agentic', value: 'agentic' },
+    { label: 'Research', value: 'research' },
   ];
 
   const isPublishable = () => {
@@ -428,8 +430,9 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
       const data = await response.json();
       const tools: OptionType[] = data.tools.map((tool: UserToolType) => ({
         id: tool.id,
-        label: tool.customName ? tool.customName : tool.displayName,
+        label: getToolDisplayName(tool),
         icon: `/toolIcons/tool_${tool.name}.svg`,
+        name: tool.name,
       }));
       setUserTools(tools);
     };
@@ -946,7 +949,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
               >
                 {selectedTools.length > 0
                   ? selectedTools
-                      .map((tool) => tool.display_name || tool.name)
+                      .map((tool) => getToolDisplayName(tool))
                       .filter(Boolean)
                       .join(', ')
                   : t('agents.form.placeholders.selectTools')}
@@ -963,7 +966,10 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
                       .filter((tool) => newSelectedIds.has(tool.id))
                       .map((tool) => ({
                         id: String(tool.id),
-                        name: tool.label,
+                        name:
+                          typeof tool.name === 'string'
+                            ? tool.name
+                            : tool.label,
                         display_name: tool.label,
                       })),
                   )

frontend/src/agents/SharedAgentCard.tsx

@@ -1,4 +1,5 @@
 import AgentImage from '../components/AgentImage';
+import { getToolDisplayName } from '../utils/toolUtils';
 import { Agent } from './types';
 
 export default function SharedAgentCard({ agent }: { agent: Agent }) {
@@ -64,10 +65,10 @@ export default function SharedAgentCard({ agent }: { agent: Agent }) {
               >
                 <img
                   src={`/toolIcons/tool_${tool.name}.svg`}
-                  alt={`${tool.name} icon`}
+                  alt={`${getToolDisplayName(tool)} icon`}
                   className="h-3 w-3"
                 />{' '}
-                {tool.name}
+                {getToolDisplayName(tool)}
               </span>
             ))}
           </div>

frontend/src/agents/WorkflowBuilder.tsx

@@ -40,9 +40,12 @@ import {
 } from '@/components/ui/select';
 import { Sheet, SheetContent } from '@/components/ui/sheet';
 
+import { useSelector } from 'react-redux';
+
 import modelService from '../api/services/modelService';
 import userService from '../api/services/userService';
 import ArrowLeft from '../assets/arrow-left.svg';
+import { selectToken } from '../preferences/preferenceSlice';
 import { WorkflowNode } from './types/workflow';
 import {
   AgentNode,
@@ -55,7 +58,7 @@ import WorkflowPreview from './workflow/WorkflowPreview';
 
 import type { Model } from '../models/types';
 interface AgentNodeConfig {
-  agent_type: 'classic' | 'react';
+  agent_type: 'classic' | 'agentic' | 'research';
   llm_name?: string;
   model_id?: string;
   system_prompt: string;
@@ -77,6 +80,7 @@ interface UserTool {
 
 function WorkflowBuilderInner() {
   const navigate = useNavigate();
+  const token = useSelector(selectToken);
   const { agentId } = useParams<{ agentId?: string }>();
   const [searchParams] = useSearchParams();
   const folderId = searchParams.get('folder_id');
@@ -304,7 +308,7 @@ function WorkflowBuilderInner() {
           setAvailableModels(modelService.transformModels(modelsData.models));
         }
 
-        const toolsResponse = await userService.getUserTools(null);
+        const toolsResponse = await userService.getUserTools(token);
         if (toolsResponse.ok) {
           const toolsData = await toolsResponse.json();
           setAvailableTools(toolsData.tools);
@@ -881,7 +885,12 @@ function WorkflowBuilderInner() {
                                   <SelectItem value="classic">
                                     Classic
                                   </SelectItem>
-                                  <SelectItem value="react">ReAct</SelectItem>
+                                  <SelectItem value="agentic">
+                                    Agentic
+                                  </SelectItem>
+                                  <SelectItem value="research">
+                                    Research
+                                  </SelectItem>
                                 </SelectContent>
                               </Select>
                             </div>

frontend/src/agents/workflow/WorkflowBuilder.tsx

@@ -54,7 +54,11 @@ import { FileUpload } from '../../components/FileUpload';
 import AgentDetailsModal from '../../modals/AgentDetailsModal';
 import ConfirmationModal from '../../modals/ConfirmationModal';
 import { ActiveState } from '../../models/misc';
-import { selectToken } from '../../preferences/preferenceSlice';
+import {
+  selectSourceDocs,
+  selectToken,
+} from '../../preferences/preferenceSlice';
+import { getToolDisplayName } from '../../utils/toolUtils';
 import { Agent } from '../types';
 import { ConditionCase, WorkflowNode } from '../types/workflow';
 import MobileBlocker from './components/MobileBlocker';
@@ -74,7 +78,7 @@ import type { Model } from '../../models/types';
 const PRIMARY_ACTION_SPINNER_DELAY_MS = 180;
 
 interface AgentNodeConfig {
-  agent_type: 'classic' | 'react';
+  agent_type: 'classic' | 'agentic' | 'research';
   llm_name?: string;
   model_id?: string;
   system_prompt: string;
@@ -92,6 +96,7 @@ interface UserTool {
   id: string;
   name: string;
   displayName: string;
+  customName?: string;
 }
 
 function validateJsonSchemaConfig(schema: unknown): string | null {
@@ -298,6 +303,7 @@ function createWorkflowPayload(
 function WorkflowBuilderInner() {
   const navigate = useNavigate();
   const token = useSelector(selectToken);
+  const sourceDocs = useSelector(selectSourceDocs);
   const { agentId } = useParams<{ agentId?: string }>();
   const [searchParams] = useSearchParams();
   const folderId = searchParams.get('folder_id');
@@ -339,6 +345,14 @@ function WorkflowBuilderInner() {
   const [availableModels, setAvailableModels] = useState<Model[]>([]);
   const [defaultAgentModelId, setDefaultAgentModelId] = useState('');
   const [availableTools, setAvailableTools] = useState<UserTool[]>([]);
+  const sourceOptions = useMemo(
+    () =>
+      (sourceDocs ?? []).map((doc) => ({
+        value: doc.id ?? 'default',
+        label: doc.name,
+      })),
+    [sourceDocs],
+  );
   const [agentJsonSchemaDrafts, setAgentJsonSchemaDrafts] = useState<
     Record<string, string>
   >({});
@@ -385,31 +399,39 @@ function WorkflowBuilderInner() {
     [],
   );
 
-  const onConnect = useCallback((params: Connection) => {
-    setEdges((eds) => {
-      const exists = eds.some(
-        (e) =>
-          e.source === params.source &&
-          e.sourceHandle === params.sourceHandle &&
-          e.target === params.target &&
-          e.targetHandle === params.targetHandle,
-      );
-      if (exists) return eds;
-
-      const filtered = eds.filter(
-        (e) =>
-          !(
+  const onConnect = useCallback(
+    (params: Connection) => {
+      setEdges((eds) => {
+        const exists = eds.some(
+          (e) =>
             e.source === params.source &&
-            e.sourceHandle === (params.sourceHandle ?? null)
-          ) &&
-          !(
+            e.sourceHandle === params.sourceHandle &&
             e.target === params.target &&
-            e.targetHandle === (params.targetHandle ?? null)
-          ),
-      );
-      return addEdge(params, filtered);
-    });
-  }, []);
+            e.targetHandle === params.targetHandle,
+        );
+        if (exists) return eds;
+
+        const targetNode = nodes.find((n) => n.id === params.target);
+        const isEndNode = targetNode?.type === 'end';
+
+        const filtered = eds.filter(
+          (e) =>
+            !(
+              e.source === params.source &&
+              e.sourceHandle === (params.sourceHandle ?? null)
+            ) &&
+            // End nodes accept multiple incoming edges
+            (isEndNode ||
+              !(
+                e.target === params.target &&
+                e.targetHandle === (params.targetHandle ?? null)
+              )),
+        );
+        return addEdge(params, filtered);
+      });
+    },
+    [nodes],
+  );
 
   const onEdgeClick = useCallback((_event: React.MouseEvent, edge: Edge) => {
     setEdges((eds) => eds.filter((e) => e.id !== edge.id));
@@ -699,7 +721,7 @@ function WorkflowBuilderInner() {
           setDefaultAgentModelId(preferredDefaultModel);
         }
 
-        const toolsResponse = await userService.getUserTools(null);
+        const toolsResponse = await userService.getUserTools(token);
         if (toolsResponse.ok) {
           const toolsData = await toolsResponse.json();
           setAvailableTools(toolsData.tools);
@@ -1269,8 +1291,8 @@ function WorkflowBuilderInner() {
 
   const handlePrimaryAction = useCallback(() => {
     if (isPrimaryActionDisabled) return;
-    void persistWorkflow(!canManageAgent);
-  }, [isPrimaryActionDisabled, persistWorkflow, canManageAgent]);
+    void persistWorkflow(false);
+  }, [isPrimaryActionDisabled, persistWorkflow]);
 
   const agentForDetails = useMemo<Agent>(
     () => ({
@@ -1746,8 +1768,11 @@ function WorkflowBuilderInner() {
                                       <SelectItem value="classic">
                                         Classic
                                       </SelectItem>
-                                      <SelectItem value="react">
-                                        ReAct
+                                      <SelectItem value="agentic">
+                                        Agentic
+                                      </SelectItem>
+                                      <SelectItem value="research">
+                                        Research
                                       </SelectItem>
                                     </SelectContent>
                                   </Select>
@@ -1887,7 +1912,7 @@ function WorkflowBuilderInner() {
                                   <MultiSelect
                                     options={availableTools.map((tool) => ({
                                       value: tool.id,
-                                      label: tool.displayName,
+                                      label: getToolDisplayName(tool),
                                     }))}
                                     selected={
                                       selectedNode.data.config?.tools || []
@@ -1905,6 +1930,28 @@ function WorkflowBuilderInner() {
                                     emptyText="No tools available"
                                   />
                                 </div>
+                                <div>
+                                  <label className="mb-2 block text-sm font-medium text-gray-700 dark:text-gray-300">
+                                    Sources
+                                  </label>
+                                  <MultiSelect
+                                    options={sourceOptions}
+                                    selected={
+                                      selectedNode.data.config?.sources || []
+                                    }
+                                    onChange={(newSources) =>
+                                      handleUpdateNodeData({
+                                        config: {
+                                          ...(selectedNode.data.config || {}),
+                                          sources: newSources,
+                                        },
+                                      })
+                                    }
+                                    placeholder="Select sources..."
+                                    searchPlaceholder="Search sources..."
+                                    emptyText="No sources available"
+                                  />
+                                </div>
                                 <div>
                                   <label className="mb-2 block text-sm font-medium text-gray-700 dark:text-gray-300">
                                     Structured Output (JSON Schema)

frontend/src/assets/research.svg

@@ -0,0 +1,18 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M11 2C15.968 2 20 6.032 20 11C20 13.1245 19.2637 15.0768 18.0319 16.6175L21.7071 20.2929C22.0976 20.6834 22.0976 21.3166 21.7071 21.7071C21.3166 22.0976 20.6834 22.0976 20.2929 21.7071L16.6175 18.0319C15.0768 19.2637 13.1245 20 11 20C6.032 20 2 15.968 2 11C2 6.032 6.032 2 11 2ZM11 4C7.13401 4 4 7.13401 4 11C4 14.866 7.13401 18 11 18C14.866 18 18 14.866 18 11C18 7.13401 14.866 4 11 4Z" fill="url(#paint0_research)"/>
+<path d="M11 7C11.5523 7 12 7.44772 12 8V10H14C14.5523 10 15 10.4477 15 11C15 11.5523 14.5523 12 14 12H12V14C12 14.5523 11.5523 15 11 15C10.4477 15 10 14.5523 10 14V12H8C7.44772 12 7 11.5523 7 11C7 10.4477 7.44772 10 8 10H10V8C10 7.44772 10.4477 7 11 7Z" fill="url(#paint1_research)"/>
+<defs>
+<linearGradient id="paint0_research" x1="2" y1="12" x2="22" y2="12" gradientUnits="userSpaceOnUse">
+<stop stop-color="#70FDF7"/>
+<stop offset="0.325" stop-color="#747696"/>
+<stop offset="0.68" stop-color="#BD5372"/>
+<stop offset="1" stop-color="#F5A06C"/>
+</linearGradient>
+<linearGradient id="paint1_research" x1="7" y1="11" x2="15" y2="11" gradientUnits="userSpaceOnUse">
+<stop stop-color="#70FDF7"/>
+<stop offset="0.325" stop-color="#747696"/>
+<stop offset="0.68" stop-color="#BD5372"/>
+<stop offset="1" stop-color="#F5A06C"/>
+</linearGradient>
+</defs>
+</svg>

frontend/src/conversation/ConversationBubble.tsx

@@ -37,7 +37,8 @@ import {
   selectSelectedDocs,
 } from '../preferences/preferenceSlice';
 import classes from './ConversationBubble.module.css';
-import { FEEDBACK, MESSAGE_TYPE } from './conversationModels';
+import { FEEDBACK, MESSAGE_TYPE, ResearchState } from './conversationModels';
+import ResearchProgress from './ResearchProgress';
 import { ToolCallsType } from './types';
 
 const DisableSourceFE = import.meta.env.VITE_DISABLE_SOURCE_FE || false;
@@ -53,6 +54,7 @@ const ConversationBubble = forwardRef<
     thought?: string;
     sources?: { title: string; text: string; link: string }[];
     toolCalls?: ToolCallsType[];
+    research?: ResearchState;
     retryBtn?: React.ReactElement;
     questionNumber?: number;
     isStreaming?: boolean;
@@ -74,6 +76,7 @@ const ConversationBubble = forwardRef<
     thought,
     sources,
     toolCalls,
+    research,
     retryBtn,
     questionNumber,
     isStreaming,
@@ -261,7 +264,15 @@ const ConversationBubble = forwardRef<
       return inlineProcessedContent;
     };
     const processMarkdownContent = (content: string) => {
-      const processedContent = preprocessLaTeX(content);
+      let processedContent = preprocessLaTeX(content);
+
+      // Convert citation references [N] into markdown links [N](#cite-N)
+      // so ReactMarkdown renders them as <a> tags we can style.
+      // Avoid matching inside code blocks or existing links.
+      processedContent = processedContent.replace(
+        /(?<!\[)\[(\d+)\](?!\()/g,
+        (_, num) => `[${num}](#cite-${num})`,
+      );
 
       const contentSegments: Array<{
         type: 'text' | 'mermaid';
@@ -290,6 +301,7 @@ const ConversationBubble = forwardRef<
 
       return contentSegments;
     };
+
     bubble = (
       <div
         ref={ref}
@@ -320,7 +332,7 @@ const ConversationBubble = forwardRef<
                 <div className="fade-in mr-5 ml-3 max-w-[90vw] md:max-w-[70vw] lg:max-w-[50vw]">
                   <div className="grid grid-cols-2 gap-2 lg:grid-cols-4">
                     {sources?.slice(0, 3)?.map((source, index) => (
-                      <div key={index} className="relative">
+                      <div key={index} id={`source-${index}`} className="relative transition-all duration-300">
                         <div
                           className="bg-muted hover:bg-accent dark:bg-answer-bubble dark:hover:bg-muted h-28 cursor-pointer rounded-4xl p-4"
                           onMouseOver={() => setActiveTooltip(index)}
@@ -393,6 +405,7 @@ const ConversationBubble = forwardRef<
                 </div>
               </div>
             )}
+        {research && <ResearchProgress research={research} />}
         {toolCalls && toolCalls.length > 0 && (
           <ToolCalls toolCalls={toolCalls} />
         )}
@@ -474,6 +487,30 @@ const ConversationBubble = forwardRef<
                             remarkPlugins={[remarkGfm, remarkMath]}
                             rehypePlugins={[rehypeKatex]}
                             components={{
+                              a({ href, children }) {
+                                if (href?.startsWith('#cite-')) {
+                                  const num = href.replace('#cite-', '');
+                                  const sourceIdx = parseInt(num, 10) - 1;
+                                  return (
+                                    <button
+                                      type="button"
+                                      onClick={() => {
+                                        const el = document.getElementById(`source-${sourceIdx}`);
+                                        if (el) {
+                                          el.scrollIntoView({ behavior: 'smooth', block: 'center' });
+                                          el.classList.add('ring-2', 'ring-purple-500');
+                                          setTimeout(() => el.classList.remove('ring-2', 'ring-purple-500'), 2000);
+                                        }
+                                      }}
+                                      className="mx-0.5 inline-flex h-5 min-w-5 items-center justify-center rounded-full bg-purple-100 px-1.5 text-xs font-semibold text-purple-700 transition-colors hover:bg-purple-200 dark:bg-purple-900/40 dark:text-purple-300 dark:hover:bg-purple-900/60"
+                                      title={`Jump to source ${num}`}
+                                    >
+                                      {num}
+                                    </button>
+                                  );
+                                }
+                                return <a href={href} target="_blank" rel="noopener noreferrer">{children}</a>;
+                              },
                               code(props) {
                                 const {
                                   children,
@@ -649,6 +686,29 @@ const ConversationBubble = forwardRef<
                     <div className="relative mr-2 block items-center justify-center">
                       <CopyButton textToCopy={message} />
                     </div>
+                    {research && message && (
+                      <div className="relative mr-2 block items-center justify-center">
+                        <button
+                          type="button"
+                          onClick={() => {
+                            const blob = new Blob([message], { type: 'text/markdown' });
+                            const url = URL.createObjectURL(blob);
+                            const link = document.createElement('a');
+                            link.href = url;
+                            link.download = `research-report.md`;
+                            link.click();
+                            URL.revokeObjectURL(url);
+                          }}
+                          className="bg-white-3000 dark:hover:bg-purple-taupe flex cursor-pointer items-center justify-center rounded-full p-2 hover:bg-[#EEEEEE] dark:bg-transparent"
+                          aria-label="Export as Markdown"
+                          title="Export as Markdown"
+                        >
+                          <svg className="h-5 w-5 stroke-gray-4000" fill="none" viewBox="0 0 24 24" strokeWidth={1.5}>
+                            <path strokeLinecap="round" strokeLinejoin="round" d="M3 16.5v2.25A2.25 2.25 0 005.25 21h13.5A2.25 2.25 0 0021 18.75V16.5M16.5 12L12 16.5m0 0L7.5 12m4.5 4.5V3" />
+                          </svg>
+                        </button>
+                      </div>
+                    )}
                     <div className="relative mr-2 block items-center justify-center">
                       <SpeakButton text={message} />
                     </div>

frontend/src/conversation/ConversationMessages.tsx

@@ -140,7 +140,7 @@ export default function ConversationMessages({
       ? LAST_BUBBLE_MARGIN
       : DEFAULT_BUBBLE_MARGIN;
 
-    if (query.thought || query.response || query.tool_calls) {
+    if (query.thought || query.response || query.tool_calls || query.research) {
       const isCurrentlyStreaming =
         status === 'loading' && index === queries.length - 1;
       return (
@@ -152,6 +152,7 @@ export default function ConversationMessages({
           thought={query.thought}
           sources={query.sources}
           toolCalls={query.tool_calls}
+          research={query.research}
           onOpenArtifact={onOpenArtifact}
           feedback={query.feedback}
           isStreaming={isCurrentlyStreaming}

frontend/src/conversation/ResearchProgress.tsx

@@ -0,0 +1,147 @@
+import { useEffect, useState } from 'react';
+import ResearchIcon from '../assets/research.svg';
+import Avatar from '../components/Avatar';
+import { ResearchState } from './conversationModels';
+
+const SmallCheck = () => (
+  <svg className="h-3 w-3 text-green-500" fill="none" viewBox="0 0 24 24" stroke="currentColor" strokeWidth={3}>
+    <path strokeLinecap="round" strokeLinejoin="round" d="M5 13l4 4L19 7" />
+  </svg>
+);
+
+const SmallSpinner = () => (
+  <svg className="h-3 w-3 animate-spin text-purple-500" fill="none" viewBox="0 0 24 24">
+    <circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4" />
+    <path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z" />
+  </svg>
+);
+
+const SmallPending = () => (
+  <div className="h-2.5 w-2.5 rounded-full border-[1.5px] border-gray-400 dark:border-gray-500" />
+);
+
+function StatusText({ status, elapsed }: { status: string; elapsed?: number }) {
+  const labels: Record<string, string> = {
+    planning: 'Planning research...',
+    researching: 'Researching...',
+    synthesizing: 'Writing report...',
+    complete: 'Complete',
+  };
+  const elapsed_str = elapsed ? ` \u00B7 ${Math.round(elapsed)}s` : '';
+  return (
+    <span className="text-xs text-gray-500 dark:text-gray-400">
+      {status === 'complete' ? (
+        <>
+          <span className="text-green-600 dark:text-green-400">{labels.complete}</span>
+          {elapsed_str}
+        </>
+      ) : (
+        <>
+          {labels[status] || status}
+          {elapsed_str}
+        </>
+      )}
+    </span>
+  );
+}
+
+export default function ResearchProgress({
+  research,
+}: {
+  research: ResearchState;
+}) {
+  const { plan, status, elapsed_seconds } = research;
+  const [isExpanded, setIsExpanded] = useState(true);
+
+  // Auto-collapse when synthesis starts (report is streaming)
+  useEffect(() => {
+    if (status === 'synthesizing' || status === 'complete') {
+      setIsExpanded(false);
+    }
+  }, [status]);
+
+  if (!plan && !status) return null;
+
+  const completedSteps = plan?.filter((s) => s.status === 'complete').length ?? 0;
+  const totalSteps = plan?.length ?? 0;
+
+  // Collapsed: single-line summary
+  const summaryText = totalSteps > 0
+    ? `Researched ${completedSteps} topic${completedSteps !== 1 ? 's' : ''}`
+    : 'Research';
+
+  return (
+    <div className="mb-4 flex w-full flex-col flex-wrap items-start self-start lg:flex-nowrap">
+      {/* Header row — matches Reasoning / Sources pattern */}
+      <div className="my-2 flex flex-row items-center gap-3">
+        <Avatar
+          className="h-[26px] w-[30px] text-xl"
+          avatar={
+            <img
+              src={ResearchIcon}
+              alt="Research"
+              className="h-full w-full object-fill"
+            />
+          }
+        />
+        <button
+          className="flex flex-row items-center gap-2"
+          onClick={() => setIsExpanded(!isExpanded)}
+        >
+          <p className="text-sm font-semibold">
+            {isExpanded ? 'Research' : summaryText}
+          </p>
+          <svg
+            className={`h-4 w-4 text-gray-500 transition-transform duration-200 dark:text-gray-400 ${isExpanded ? 'rotate-180' : ''}`}
+            fill="none" viewBox="0 0 24 24" stroke="currentColor" strokeWidth={2}
+          >
+            <path strokeLinecap="round" strokeLinejoin="round" d="M19 9l-7 7-7-7" />
+          </svg>
+        </button>
+        {status && <StatusText status={status} elapsed={elapsed_seconds} />}
+      </div>
+
+      {/* Expanded: vertical timeline of steps */}
+      {isExpanded && plan && plan.length > 0 && (
+        <div className="fade-in mr-5 ml-[42px] max-w-[90vw] md:max-w-[70vw] lg:max-w-[50vw]">
+          <div className="space-y-0">
+            {plan.map((step, i) => {
+              const isLast = i === plan.length - 1;
+              return (
+                <div key={i} className="flex items-stretch gap-3">
+                  {/* Timeline: dot + vertical line */}
+                  <div className="flex flex-col items-center pt-1">
+                    <div className="flex h-4 w-4 flex-shrink-0 items-center justify-center">
+                      {step.status === 'complete' ? (
+                        <SmallCheck />
+                      ) : step.status === 'researching' ? (
+                        <SmallSpinner />
+                      ) : (
+                        <SmallPending />
+                      )}
+                    </div>
+                    {!isLast && (
+                      <div className="mt-1 w-px flex-1 bg-gray-300 dark:bg-gray-600" />
+                    )}
+                  </div>
+                  {/* Step content */}
+                  <div className={`pb-3 ${isLast ? '' : ''}`}>
+                    <p className={`text-sm ${
+                      step.status === 'complete'
+                        ? 'text-gray-700 dark:text-gray-300'
+                        : step.status === 'researching'
+                          ? 'font-medium text-purple-700 dark:text-purple-300'
+                          : 'text-gray-500 dark:text-gray-500'
+                    }`}>
+                      {step.query}
+                    </p>
+                  </div>
+                </div>
+              );
+            })}
+          </div>
+        </div>
+      )}
+    </div>
+  );
+}

frontend/src/conversation/conversationModels.ts

@@ -18,6 +18,20 @@ export interface Attachment {
   token_count?: number;
 }
 
+export interface ResearchStep {
+  query: string;
+  rationale?: string;
+  status: 'pending' | 'researching' | 'complete';
+}
+
+export interface ResearchState {
+  plan?: ResearchStep[];
+  complexity?: string;
+  status?: string;
+  elapsed_seconds?: number;
+  tokens_used?: number;
+}
+
 export interface ConversationState {
   queries: Query[];
   status: Status;
@@ -50,6 +64,7 @@ export interface Query {
   attachments?: { id: string; fileName: string }[];
   structured?: boolean;
   schema?: object;
+  research?: ResearchState;
 }
 
 export interface RetrievalPayload {

frontend/src/conversation/conversationSlice.ts

@@ -11,7 +11,13 @@ import {
   handleFetchAnswer,
   handleFetchAnswerSteaming,
 } from './conversationHandlers';
-import { Answer, ConversationState, Query, Status } from './conversationModels';
+import {
+  Answer,
+  ConversationState,
+  Query,
+  ResearchStep,
+  Status,
+} from './conversationModels';
 import { ToolCallsType } from './types';
 
 const initialState: ConversationState = {
@@ -71,6 +77,16 @@ export const fetchAnswer = createAsyncThunk<
           if (currentConversationId === state.conversation.conversationId) {
             if (data.type === 'end') {
               dispatch(conversationSlice.actions.setStatus('idle'));
+              // Only update research status if this query has research data
+              const currentState = getState() as RootState;
+              if (currentState.conversation.queries[targetIndex]?.research) {
+                dispatch(
+                  updateResearchProgress({
+                    index: targetIndex,
+                    progress: { status: 'complete' },
+                  }),
+                );
+              }
               getConversations(state.preference.token)
                 .then((fetchedConversations) => {
                   dispatch(setConversations(fetchedConversations));
@@ -122,6 +138,21 @@ export const fetchAnswer = createAsyncThunk<
                   tool_call: data.data as ToolCallsType,
                 }),
               );
+            } else if (data.type === 'research_plan') {
+              dispatch(
+                updateResearchPlan({
+                  index: targetIndex,
+                  plan: data.data.steps,
+                  complexity: data.data.complexity,
+                }),
+              );
+            } else if (data.type === 'research_progress') {
+              dispatch(
+                updateResearchProgress({
+                  index: targetIndex,
+                  progress: data.data,
+                }),
+              );
             } else if (data.type === 'error') {
               // set status to 'failed'
               dispatch(conversationSlice.actions.setStatus('failed'));
@@ -256,6 +287,7 @@ export const conversationSlice = createSlice({
       delete state.queries[index].structured;
       delete state.queries[index].schema;
       delete state.queries[index].feedback;
+      delete state.queries[index].research;
     },
     updateStreamingQuery(
       state,
@@ -337,6 +369,77 @@ export const conversationSlice = createSlice({
         };
       } else state.queries[index].tool_calls.push(tool_call);
     },
+    updateResearchPlan(
+      state,
+      action: PayloadAction<{
+        index: number;
+        plan: ResearchStep[];
+        complexity?: string;
+      }>,
+    ) {
+      const { index, plan, complexity } = action.payload;
+      if (!state.queries[index].research) {
+        state.queries[index].research = {};
+      }
+      state.queries[index].research!.plan = plan.map((step) => ({
+        ...step,
+        status: 'pending',
+      }));
+      if (complexity) {
+        state.queries[index].research!.complexity = complexity;
+      }
+    },
+    updateResearchProgress(
+      state,
+      action: PayloadAction<{
+        index: number;
+        progress: {
+          status?: string;
+          step?: number;
+          total?: number;
+          query?: string;
+          elapsed_seconds?: number;
+          tokens_used?: number;
+        };
+      }>,
+    ) {
+      const { index, progress } = action.payload;
+      if (!state.queries[index].research) {
+        state.queries[index].research = {};
+      }
+      const research = state.queries[index].research!;
+      if (progress.elapsed_seconds !== undefined) {
+        research.elapsed_seconds = progress.elapsed_seconds;
+      }
+      if (progress.tokens_used !== undefined) {
+        research.tokens_used = progress.tokens_used;
+      }
+      // Update individual step status when step number is present
+      if (progress.step !== undefined) {
+        if (!research.plan) {
+          research.plan = [];
+        }
+        const stepIndex = progress.step - 1;
+        // Dynamically add step if it doesn't exist yet
+        while (research.plan.length <= stepIndex) {
+          research.plan.push({
+            query: progress.query || `Step ${research.plan.length + 1}`,
+            status: 'pending',
+          });
+        }
+        if (progress.status === 'researching' || progress.status === 'complete') {
+          research.plan[stepIndex].status = progress.status;
+        }
+        if (progress.query) {
+          research.plan[stepIndex].query = progress.query;
+        }
+        // Keep top-level status as "researching" while steps are running
+        research.status = 'researching';
+      } else if (progress.status) {
+        // Top-level status updates (planning, synthesizing)
+        research.status = progress.status;
+      }
+    },
     updateQuery(
       state,
       action: PayloadAction<{ index: number; query: Partial<Query> }>,
@@ -405,6 +508,8 @@ export const {
   updateThought,
   updateStreamingSource,
   updateToolCall,
+  updateResearchPlan,
+  updateResearchProgress,
   setConversation,
   setStatus,
   raiseError,

frontend/src/utils/toolUtils.ts

@@ -0,0 +1,17 @@
+type ToolLabelSource = {
+  customName?: string | null;
+  displayName?: string | null;
+  display_name?: string | null;
+  name?: string | null;
+};
+
+const isNonEmptyString = (value: unknown): value is string =>
+  typeof value === 'string' && value.trim().length > 0;
+
+export function getToolDisplayName(tool: ToolLabelSource): string {
+  if (isNonEmptyString(tool.customName)) return tool.customName.trim();
+  if (isNonEmptyString(tool.displayName)) return tool.displayName.trim();
+  if (isNonEmptyString(tool.display_name)) return tool.display_name.trim();
+  if (isNonEmptyString(tool.name)) return tool.name.trim();
+  return '';
+}

pytest.ini

@@ -3,7 +3,7 @@ testpaths = tests
 python_files = test_*.py
 python_classes = Test*
 python_functions = test_*
-addopts = 
+addopts =
     -v
     --strict-markers
     --tb=short
@@ -11,6 +11,7 @@ addopts =
     --cov-report=html
     --cov-report=term-missing
     --cov-report=xml
+    --ignore=tests/integration
 markers =
     unit: Unit tests
     integration: Integration tests

setup.ps1

@@ -977,8 +977,8 @@ function Connect-CloudAPIProvider {
             }
             "7" {  # Novita
                 $script:provider_name = "Novita"
-                $script:llm_name = "novita"
-                $script:model_name = "deepseek/deepseek-r1"
+                $script:llm_provider = "novita"
+                $script:model_name = "moonshotai/kimi-k2.5"
                 Get-APIKey
                 break
             }

setup.sh

@@ -704,7 +704,7 @@ connect_cloud_api_provider() {
             7) # Novita
                 provider_name="Novita"
                 llm_provider="novita"
-                model_name="deepseek/deepseek-r1"
+                model_name="moonshotai/kimi-k2.5"
                 get_api_key
                 break ;;
             b|B) clear; return 1 ;; # Clear screen and Back to Main Menu

tests/agents/test_agent_creator.py

@@ -1,7 +1,6 @@
 import pytest
 from application.agents.agent_creator import AgentCreator
 from application.agents.classic_agent import ClassicAgent
-from application.agents.react_agent import ReActAgent
 
 
 @pytest.mark.unit
@@ -14,11 +13,9 @@ class TestAgentCreator:
         assert agent.llm_name == agent_base_params["llm_name"]
         assert agent.model_id == agent_base_params["model_id"]
 
-    def test_create_react_agent(self, agent_base_params):
+    def test_create_react_falls_back_to_classic(self, agent_base_params):
         agent = AgentCreator.create_agent("react", **agent_base_params)
-        assert isinstance(agent, ReActAgent)
-        assert agent.endpoint == agent_base_params["endpoint"]
-        assert agent.llm_name == agent_base_params["llm_name"]
+        assert isinstance(agent, ClassicAgent)
 
     def test_create_agent_case_insensitive(self, agent_base_params):
         agent_upper = AgentCreator.create_agent("CLASSIC", **agent_base_params)
@@ -35,7 +32,6 @@ class TestAgentCreator:
         assert "classic" in AgentCreator.agents
         assert "react" in AgentCreator.agents
         assert AgentCreator.agents["classic"] == ClassicAgent
-        assert AgentCreator.agents["react"] == ReActAgent
 
     def test_create_agent_with_optional_params(self, agent_base_params):
         agent_base_params["user_api_key"] = "user_key_123"

tests/agents/test_agentic_agent.py

@@ -0,0 +1,139 @@
+"""Tests for AgenticAgent — LLM-controlled retrieval agent."""
+
+from unittest.mock import Mock
+
+import pytest
+from application.agents.agentic_agent import AgenticAgent
+
+
+@pytest.mark.unit
+class TestAgenticAgentInit:
+
+    def test_initialization(
+        self, agent_base_params, mock_llm_creator, mock_llm_handler_creator
+    ):
+        agent = AgenticAgent(**agent_base_params)
+        assert isinstance(agent, AgenticAgent)
+        assert agent.retriever_config == {}
+
+    def test_initialization_with_retriever_config(
+        self, agent_base_params, mock_llm_creator, mock_llm_handler_creator
+    ):
+        rc = {"source": {"active_docs": ["abc"]}, "retriever_name": "classic"}
+        agent = AgenticAgent(retriever_config=rc, **agent_base_params)
+        assert agent.retriever_config == rc
+
+    def test_inherits_base_properties(
+        self, agent_base_params, mock_llm_creator, mock_llm_handler_creator
+    ):
+        agent = AgenticAgent(**agent_base_params)
+        assert agent.endpoint == agent_base_params["endpoint"]
+        assert agent.llm_name == agent_base_params["llm_name"]
+        assert agent.model_id == agent_base_params["model_id"]
+
+
+@pytest.mark.unit
+class TestAgenticAgentGenInner:
+
+    def test_basic_flow_yields_sources_and_tool_calls(
+        self,
+        agent_base_params,
+        mock_llm,
+        mock_llm_handler,
+        mock_llm_creator,
+        mock_llm_handler_creator,
+        mock_mongo_db,
+        log_context,
+    ):
+        mock_llm.gen_stream = Mock(return_value=iter(["Answer"]))
+
+        def mock_handler(*args, **kwargs):
+            yield "Processed"
+
+        mock_llm_handler.process_message_flow = Mock(side_effect=mock_handler)
+
+        agent = AgenticAgent(**agent_base_params)
+        results = list(agent._gen_inner("Test query", log_context))
+
+        sources = [r for r in results if "sources" in r]
+        tool_calls = [r for r in results if "tool_calls" in r]
+        assert len(sources) == 1
+        assert len(tool_calls) == 1
+
+    def test_logs_agent_component(
+        self,
+        agent_base_params,
+        mock_llm,
+        mock_llm_handler,
+        mock_llm_creator,
+        mock_llm_handler_creator,
+        mock_mongo_db,
+        log_context,
+    ):
+        mock_llm.gen_stream = Mock(return_value=iter(["Answer"]))
+
+        def mock_handler(*args, **kwargs):
+            yield "Done"
+
+        mock_llm_handler.process_message_flow = Mock(side_effect=mock_handler)
+
+        agent = AgenticAgent(**agent_base_params)
+        list(agent._gen_inner("Query", log_context))
+
+        agent_logs = [s for s in log_context.stacks if s["component"] == "agent"]
+        assert len(agent_logs) == 1
+        assert "tool_calls" in agent_logs[0]["data"]
+
+    def test_no_pre_fetched_docs_in_messages(
+        self,
+        agent_base_params,
+        mock_llm,
+        mock_llm_handler,
+        mock_llm_creator,
+        mock_llm_handler_creator,
+        mock_mongo_db,
+        log_context,
+    ):
+        mock_llm.gen_stream = Mock(return_value=iter(["Answer"]))
+
+        def mock_handler(*args, **kwargs):
+            yield "Done"
+
+        mock_llm_handler.process_message_flow = Mock(side_effect=mock_handler)
+
+        agent = AgenticAgent(**agent_base_params)
+        list(agent._gen_inner("Query", log_context))
+
+        call_kwargs = mock_llm.gen_stream.call_args[1]
+        messages = call_kwargs["messages"]
+        # System prompt should not contain {summaries} replacement
+        assert messages[0]["role"] == "system"
+        assert messages[-1]["role"] == "user"
+        assert messages[-1]["content"] == "Query"
+
+
+@pytest.mark.unit
+class TestAgenticAgentCollectSources:
+
+    def test_collect_internal_sources_from_cache(
+        self, agent_base_params, mock_llm_creator, mock_llm_handler_creator
+    ):
+        agent = AgenticAgent(**agent_base_params)
+
+        mock_tool = Mock()
+        mock_tool.retrieved_docs = [
+            {"text": "Found", "title": "Doc", "source": "test"},
+        ]
+        cache_key = f"internal_search:internal:{agent.user or ''}"
+        agent.tool_executor._loaded_tools[cache_key] = mock_tool
+
+        agent._collect_internal_sources()
+        assert len(agent.retrieved_docs) == 1
+        assert agent.retrieved_docs[0]["title"] == "Doc"
+
+    def test_collect_internal_sources_no_cache(
+        self, agent_base_params, mock_llm_creator, mock_llm_handler_creator
+    ):
+        agent = AgenticAgent(**agent_base_params)
+        agent._collect_internal_sources()
+        assert agent.retrieved_docs == []

tests/agents/test_api_body_serializer.py

@@ -0,0 +1,202 @@
+"""Tests for application/agents/tools/api_body_serializer.py"""
+
+import json
+
+import pytest
+
+from application.agents.tools.api_body_serializer import (
+    ContentType,
+    RequestBodySerializer,
+)
+
+
+@pytest.mark.unit
+class TestContentTypeEnum:
+    def test_json_value(self):
+        assert ContentType.JSON == "application/json"
+
+    def test_form_urlencoded_value(self):
+        assert ContentType.FORM_URLENCODED == "application/x-www-form-urlencoded"
+
+    def test_multipart_value(self):
+        assert ContentType.MULTIPART_FORM_DATA == "multipart/form-data"
+
+    def test_text_plain_value(self):
+        assert ContentType.TEXT_PLAIN == "text/plain"
+
+    def test_xml_value(self):
+        assert ContentType.XML == "application/xml"
+
+    def test_octet_stream_value(self):
+        assert ContentType.OCTET_STREAM == "application/octet-stream"
+
+
+@pytest.mark.unit
+class TestSerializeJson:
+    def test_basic_json(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"key": "value"}, ContentType.JSON
+        )
+        assert json.loads(body) == {"key": "value"}
+        assert headers["Content-Type"] == "application/json"
+
+    def test_nested_json(self):
+        data = {"user": {"name": "Alice", "age": 30}}
+        body, headers = RequestBodySerializer.serialize(data, ContentType.JSON)
+        assert json.loads(body) == data
+
+    def test_empty_body_returns_none(self):
+        body, headers = RequestBodySerializer.serialize({}, ContentType.JSON)
+        assert body is None
+        assert headers == {}
+
+    def test_none_body(self):
+        body, headers = RequestBodySerializer.serialize(None, ContentType.JSON)
+        assert body is None
+
+    def test_unknown_content_type_falls_back_to_json(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"k": "v"}, "application/vnd.custom+json"
+        )
+        assert json.loads(body) == {"k": "v"}
+
+    def test_content_type_with_charset_suffix(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"k": "v"}, "application/json; charset=utf-8"
+        )
+        assert json.loads(body) == {"k": "v"}
+
+
+@pytest.mark.unit
+class TestSerializeFormUrlencoded:
+    def test_basic_form(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"name": "Alice", "age": "30"}, ContentType.FORM_URLENCODED
+        )
+        assert "name=Alice" in body
+        assert "age=30" in body
+        assert headers["Content-Type"] == "application/x-www-form-urlencoded"
+
+    def test_none_values_skipped(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"name": "Alice", "skip": None}, ContentType.FORM_URLENCODED
+        )
+        assert "name=Alice" in body
+        assert "skip" not in body
+
+    def test_list_explode_true(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"tags": ["a", "b"]},
+            ContentType.FORM_URLENCODED,
+            encoding_rules={"tags": {"style": "form", "explode": True}},
+        )
+        assert "tags=a" in body
+        assert "tags=b" in body
+
+    def test_list_explode_false(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"tags": ["a", "b"]},
+            ContentType.FORM_URLENCODED,
+            encoding_rules={"tags": {"style": "form", "explode": False}},
+        )
+        # Value is percent-encoded by _serialize_form_value then urlencoded again
+        assert "tags=" in body
+        assert "a" in body and "b" in body
+
+    def test_dict_value_json_content_type(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"metadata": {"key": "val"}},
+            ContentType.FORM_URLENCODED,
+            encoding_rules={"metadata": {"contentType": "application/json"}},
+        )
+        assert "metadata" in body
+
+
+@pytest.mark.unit
+class TestSerializeTextPlain:
+    def test_single_value(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"message": "hello"}, ContentType.TEXT_PLAIN
+        )
+        assert body == "hello"
+        assert headers["Content-Type"] == "text/plain"
+
+    def test_multiple_values(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"name": "Alice", "age": 30}, ContentType.TEXT_PLAIN
+        )
+        assert "name: Alice" in body
+        assert "age: 30" in body
+
+
+@pytest.mark.unit
+class TestSerializeXml:
+    def test_basic_xml(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"name": "Alice"}, ContentType.XML
+        )
+        assert '<?xml version="1.0"' in body
+        assert "<name>Alice</name>" in body
+        assert headers["Content-Type"] == "application/xml"
+
+    def test_nested_xml(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"user": {"name": "Alice"}}, ContentType.XML
+        )
+        assert "<user>" in body
+        assert "<name>Alice</name>" in body
+
+    def test_xml_escapes_special_chars(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"data": "<script>alert('xss')</script>"}, ContentType.XML
+        )
+        assert "&lt;script&gt;" in body
+
+
+@pytest.mark.unit
+class TestSerializeOctetStream:
+    def test_dict_body(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"key": "val"}, ContentType.OCTET_STREAM
+        )
+        assert isinstance(body, bytes)
+        assert headers["Content-Type"] == "application/octet-stream"
+
+
+@pytest.mark.unit
+class TestSerializeMultipartFormData:
+    def test_basic_multipart(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"field": "value"}, ContentType.MULTIPART_FORM_DATA
+        )
+        assert isinstance(body, bytes)
+        assert "multipart/form-data" in headers["Content-Type"]
+        assert "boundary=" in headers["Content-Type"]
+
+    def test_none_values_skipped(self):
+        body, headers = RequestBodySerializer.serialize(
+            {"field": "value", "empty": None}, ContentType.MULTIPART_FORM_DATA
+        )
+        body_str = body.decode("utf-8", errors="replace")
+        assert "field" in body_str
+        assert "empty" not in body_str
+
+
+@pytest.mark.unit
+class TestHelpers:
+    def test_percent_encode(self):
+        assert RequestBodySerializer._percent_encode("hello world") == "hello%20world"
+        assert RequestBodySerializer._percent_encode("a/b") == "a%2Fb"
+        assert RequestBodySerializer._percent_encode("safe", safe_chars="/") == "safe"
+
+    def test_escape_xml(self):
+        assert "&amp;" in RequestBodySerializer._escape_xml("&")
+        assert "&lt;" in RequestBodySerializer._escape_xml("<")
+        assert "&gt;" in RequestBodySerializer._escape_xml(">")
+        assert "&quot;" in RequestBodySerializer._escape_xml('"')
+        assert "&apos;" in RequestBodySerializer._escape_xml("'")
+
+    def test_dict_to_xml_list(self):
+        xml = RequestBodySerializer._dict_to_xml({"items": [1, 2, 3]})
+        assert "<item>1</item>" in xml
+        assert "<item>2</item>" in xml

tests/agents/test_api_tool.py

@@ -0,0 +1,282 @@
+"""Tests for application/agents/tools/api_tool.py"""
+
+import json
+from unittest.mock import MagicMock, patch
+
+import pytest
+import requests
+
+from application.agents.tools.api_tool import APITool
+
+
+@pytest.fixture
+def tool():
+    return APITool(
+        config={
+            "url": "https://api.example.com/data",
+            "method": "GET",
+            "headers": {"Accept": "application/json"},
+            "query_params": {},
+        }
+    )
+
+
+@pytest.fixture
+def post_tool():
+    return APITool(
+        config={
+            "url": "https://api.example.com/items",
+            "method": "POST",
+            "headers": {},
+            "query_params": {},
+        }
+    )
+
+
+@pytest.mark.unit
+class TestAPIToolInit:
+    def test_default_values(self):
+        tool = APITool(config={})
+        assert tool.url == ""
+        assert tool.method == "GET"
+        assert tool.headers == {}
+        assert tool.query_params == {}
+
+
+@pytest.mark.unit
+class TestMakeApiCall:
+    @patch("application.agents.tools.api_tool.validate_url")
+    @patch("application.agents.tools.api_tool.requests.get")
+    def test_successful_get(self, mock_get, mock_validate, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.headers = {"Content-Type": "application/json"}
+        mock_resp.json.return_value = {"result": "ok"}
+        mock_resp.content = b'{"result":"ok"}'
+        mock_get.return_value = mock_resp
+
+        result = tool.execute_action("any_action")
+
+        assert result["status_code"] == 200
+        assert result["data"] == {"result": "ok"}
+        assert result["message"] == "API call successful."
+
+    @patch("application.agents.tools.api_tool.validate_url")
+    @patch("application.agents.tools.api_tool.requests.post")
+    def test_successful_post(self, mock_post, mock_validate, post_tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 201
+        mock_resp.headers = {"Content-Type": "application/json"}
+        mock_resp.json.return_value = {"id": 1}
+        mock_resp.content = b'{"id":1}'
+        mock_post.return_value = mock_resp
+
+        result = post_tool.execute_action("create", name="test")
+
+        assert result["status_code"] == 201
+
+    @patch("application.agents.tools.api_tool.validate_url")
+    def test_ssrf_blocked(self, mock_validate, tool):
+        from application.core.url_validation import SSRFError
+
+        mock_validate.side_effect = SSRFError("blocked")
+
+        result = tool.execute_action("any")
+
+        assert result["status_code"] is None
+        assert "URL validation error" in result["message"]
+
+    @patch("application.agents.tools.api_tool.validate_url")
+    @patch("application.agents.tools.api_tool.requests.get")
+    def test_timeout_error(self, mock_get, mock_validate, tool):
+        mock_get.side_effect = requests.exceptions.Timeout()
+
+        result = tool.execute_action("any")
+
+        assert result["status_code"] is None
+        assert "timeout" in result["message"].lower()
+
+    @patch("application.agents.tools.api_tool.validate_url")
+    @patch("application.agents.tools.api_tool.requests.get")
+    def test_connection_error(self, mock_get, mock_validate, tool):
+        mock_get.side_effect = requests.exceptions.ConnectionError("refused")
+
+        result = tool.execute_action("any")
+
+        assert result["status_code"] is None
+        assert "Connection error" in result["message"]
+
+    @patch("application.agents.tools.api_tool.validate_url")
+    @patch("application.agents.tools.api_tool.requests.get")
+    def test_http_error(self, mock_get, mock_validate, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 404
+        mock_resp.text = "Not Found"
+        mock_resp.json.side_effect = json.JSONDecodeError("", "", 0)
+        mock_resp.raise_for_status.side_effect = requests.exceptions.HTTPError(
+            response=mock_resp
+        )
+        mock_get.return_value = mock_resp
+
+        result = tool.execute_action("any")
+
+        assert result["status_code"] == 404
+        assert "HTTP Error" in result["message"]
+
+    @patch("application.agents.tools.api_tool.validate_url")
+    def test_unsupported_method(self, mock_validate):
+        tool = APITool(
+            config={"url": "https://example.com", "method": "CUSTOM"}
+        )
+        result = tool.execute_action("any")
+        assert result["status_code"] is None
+        assert "Unsupported" in result["message"]
+
+    @patch("application.agents.tools.api_tool.validate_url")
+    @patch("application.agents.tools.api_tool.requests.put")
+    def test_put_method(self, mock_put, mock_validate):
+        tool = APITool(config={"url": "https://example.com/item/1", "method": "PUT"})
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.headers = {"Content-Type": "application/json"}
+        mock_resp.json.return_value = {}
+        mock_resp.content = b'{}'
+        mock_put.return_value = mock_resp
+
+        result = tool.execute_action("update", name="new")
+        assert result["status_code"] == 200
+
+    @patch("application.agents.tools.api_tool.validate_url")
+    @patch("application.agents.tools.api_tool.requests.delete")
+    def test_delete_method(self, mock_delete, mock_validate):
+        tool = APITool(config={"url": "https://example.com/item/1", "method": "DELETE"})
+        mock_resp = MagicMock()
+        mock_resp.status_code = 204
+        mock_resp.headers = {"Content-Type": "text/plain"}
+        mock_resp.content = b''
+        mock_delete.return_value = mock_resp
+
+        result = tool.execute_action("delete")
+        assert result["status_code"] == 204
+
+    @patch("application.agents.tools.api_tool.validate_url")
+    @patch("application.agents.tools.api_tool.requests.patch")
+    def test_patch_method(self, mock_patch, mock_validate):
+        tool = APITool(config={"url": "https://example.com/item/1", "method": "PATCH"})
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.headers = {"Content-Type": "application/json"}
+        mock_resp.json.return_value = {"patched": True}
+        mock_resp.content = b'{"patched":true}'
+        mock_patch.return_value = mock_resp
+
+        result = tool.execute_action("patch", field="val")
+        assert result["status_code"] == 200
+
+    @patch("application.agents.tools.api_tool.validate_url")
+    @patch("application.agents.tools.api_tool.requests.head")
+    def test_head_method(self, mock_head, mock_validate):
+        tool = APITool(config={"url": "https://example.com", "method": "HEAD"})
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.headers = {"Content-Type": "text/html"}
+        mock_resp.content = b''
+        mock_head.return_value = mock_resp
+
+        result = tool.execute_action("check")
+        assert result["status_code"] == 200
+
+    @patch("application.agents.tools.api_tool.validate_url")
+    @patch("application.agents.tools.api_tool.requests.options")
+    def test_options_method(self, mock_options, mock_validate):
+        tool = APITool(config={"url": "https://example.com", "method": "OPTIONS"})
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.headers = {"Content-Type": "text/plain"}
+        mock_resp.content = b''
+        mock_options.return_value = mock_resp
+
+        result = tool.execute_action("options")
+        assert result["status_code"] == 200
+
+
+@pytest.mark.unit
+class TestPathParamSubstitution:
+    @patch("application.agents.tools.api_tool.validate_url")
+    @patch("application.agents.tools.api_tool.requests.get")
+    def test_path_params_substituted(self, mock_get, mock_validate):
+        tool = APITool(
+            config={
+                "url": "https://api.example.com/users/{user_id}/posts/{post_id}",
+                "method": "GET",
+                "query_params": {"user_id": "42", "post_id": "7"},
+            }
+        )
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.headers = {"Content-Type": "application/json"}
+        mock_resp.json.return_value = []
+        mock_resp.content = b'[]'
+        mock_get.return_value = mock_resp
+
+        tool.execute_action("get")
+
+        called_url = mock_get.call_args[0][0]
+        assert "/users/42/posts/7" in called_url
+        assert "{user_id}" not in called_url
+
+
+@pytest.mark.unit
+class TestParseResponse:
+    def test_json_response(self, tool):
+        mock_resp = MagicMock()
+        mock_resp.headers = {"Content-Type": "application/json"}
+        mock_resp.json.return_value = {"key": "val"}
+        mock_resp.content = b'{"key":"val"}'
+
+        result = tool._parse_response(mock_resp)
+        assert result == {"key": "val"}
+
+    def test_text_response(self, tool):
+        mock_resp = MagicMock()
+        mock_resp.headers = {"Content-Type": "text/plain"}
+        mock_resp.text = "plain text"
+        mock_resp.content = b"plain text"
+
+        result = tool._parse_response(mock_resp)
+        assert result == "plain text"
+
+    def test_xml_response(self, tool):
+        mock_resp = MagicMock()
+        mock_resp.headers = {"Content-Type": "application/xml"}
+        mock_resp.text = "<root><item>1</item></root>"
+        mock_resp.content = b"<root><item>1</item></root>"
+
+        result = tool._parse_response(mock_resp)
+        assert "<root>" in result
+
+    def test_empty_content(self, tool):
+        mock_resp = MagicMock()
+        mock_resp.headers = {"Content-Type": "application/json"}
+        mock_resp.content = b""
+
+        result = tool._parse_response(mock_resp)
+        assert result is None
+
+    def test_html_response(self, tool):
+        mock_resp = MagicMock()
+        mock_resp.headers = {"Content-Type": "text/html"}
+        mock_resp.text = "<html><body>Hi</body></html>"
+        mock_resp.content = b"<html><body>Hi</body></html>"
+
+        result = tool._parse_response(mock_resp)
+        assert "<html>" in result
+
+
+@pytest.mark.unit
+class TestAPIToolMetadata:
+    def test_actions_metadata_empty(self, tool):
+        assert tool.get_actions_metadata() == []
+
+    def test_config_requirements_empty(self, tool):
+        assert tool.get_config_requirements() == {}

tests/agents/test_brave_tool.py

@@ -0,0 +1,132 @@
+"""Tests for application/agents/tools/brave.py"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from application.agents.tools.brave import BraveSearchTool
+
+
+@pytest.fixture
+def tool():
+    return BraveSearchTool(config={"token": "test_api_key"})
+
+
+@pytest.mark.unit
+class TestBraveExecuteAction:
+    def test_unknown_action_raises(self, tool):
+        with pytest.raises(ValueError, match="Unknown action"):
+            tool.execute_action("invalid")
+
+    @patch("application.agents.tools.brave.requests.get")
+    def test_web_search_success(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.json.return_value = {"web": {"results": [{"title": "Result"}]}}
+        mock_get.return_value = mock_resp
+
+        result = tool.execute_action("brave_web_search", query="python")
+
+        assert result["status_code"] == 200
+        assert "results" in result
+        assert "successfully" in result["message"]
+
+        call_kwargs = mock_get.call_args
+        assert call_kwargs[1]["headers"]["X-Subscription-Token"] == "test_api_key"
+
+    @patch("application.agents.tools.brave.requests.get")
+    def test_web_search_failure(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 429
+        mock_get.return_value = mock_resp
+
+        result = tool.execute_action("brave_web_search", query="test")
+
+        assert result["status_code"] == 429
+        assert "failed" in result["message"].lower()
+
+    @patch("application.agents.tools.brave.requests.get")
+    def test_image_search_success(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.json.return_value = {"results": [{"url": "https://img.com/1.jpg"}]}
+        mock_get.return_value = mock_resp
+
+        result = tool.execute_action("brave_image_search", query="cats")
+
+        assert result["status_code"] == 200
+        assert "results" in result
+
+    @patch("application.agents.tools.brave.requests.get")
+    def test_image_search_failure(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 500
+        mock_get.return_value = mock_resp
+
+        result = tool.execute_action("brave_image_search", query="cats")
+
+        assert result["status_code"] == 500
+
+    @patch("application.agents.tools.brave.requests.get")
+    def test_count_capped_at_20(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.json.return_value = {}
+        mock_get.return_value = mock_resp
+
+        tool.execute_action("brave_web_search", query="test", count=100)
+
+        params = mock_get.call_args[1]["params"]
+        assert params["count"] == 20
+
+    @patch("application.agents.tools.brave.requests.get")
+    def test_image_count_capped_at_100(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.json.return_value = {}
+        mock_get.return_value = mock_resp
+
+        tool.execute_action("brave_image_search", query="test", count=500)
+
+        params = mock_get.call_args[1]["params"]
+        assert params["count"] == 100
+
+    @patch("application.agents.tools.brave.requests.get")
+    def test_freshness_param(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.json.return_value = {}
+        mock_get.return_value = mock_resp
+
+        tool.execute_action("brave_web_search", query="news", freshness="pd")
+
+        params = mock_get.call_args[1]["params"]
+        assert params["freshness"] == "pd"
+
+    @patch("application.agents.tools.brave.requests.get")
+    def test_offset_capped(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.json.return_value = {}
+        mock_get.return_value = mock_resp
+
+        tool.execute_action("brave_web_search", query="test", offset=100)
+
+        params = mock_get.call_args[1]["params"]
+        assert params["offset"] == 9
+
+
+@pytest.mark.unit
+class TestBraveMetadata:
+    def test_actions_metadata(self, tool):
+        meta = tool.get_actions_metadata()
+        assert len(meta) == 2
+        names = {a["name"] for a in meta}
+        assert "brave_web_search" in names
+        assert "brave_image_search" in names
+
+    def test_config_requirements(self, tool):
+        reqs = tool.get_config_requirements()
+        assert "token" in reqs
+        assert reqs["token"]["secret"] is True
+        assert reqs["token"]["required"] is True

tests/agents/test_cel_evaluator.py

@@ -0,0 +1,169 @@
+"""Tests for application/agents/workflows/cel_evaluator.py"""
+
+import pytest
+
+from application.agents.workflows.cel_evaluator import (
+    CelEvaluationError,
+    _convert_value,
+    build_activation,
+    cel_to_python,
+    evaluate_cel,
+)
+import celpy.celtypes
+
+
+class TestConvertValue:
+
+    @pytest.mark.unit
+    def test_bool_true(self):
+        result = _convert_value(True)
+        assert isinstance(result, celpy.celtypes.BoolType)
+        assert bool(result) is True
+
+    @pytest.mark.unit
+    def test_bool_false(self):
+        result = _convert_value(False)
+        assert isinstance(result, celpy.celtypes.BoolType)
+        assert bool(result) is False
+
+    @pytest.mark.unit
+    def test_int(self):
+        result = _convert_value(42)
+        assert isinstance(result, celpy.celtypes.IntType)
+        assert int(result) == 42
+
+    @pytest.mark.unit
+    def test_float(self):
+        result = _convert_value(3.14)
+        assert isinstance(result, celpy.celtypes.DoubleType)
+        assert float(result) == pytest.approx(3.14)
+
+    @pytest.mark.unit
+    def test_string(self):
+        result = _convert_value("hello")
+        assert isinstance(result, celpy.celtypes.StringType)
+        assert str(result) == "hello"
+
+    @pytest.mark.unit
+    def test_list(self):
+        result = _convert_value([1, "two", 3.0])
+        assert isinstance(result, celpy.celtypes.ListType)
+
+    @pytest.mark.unit
+    def test_dict(self):
+        result = _convert_value({"key": "value"})
+        assert isinstance(result, celpy.celtypes.MapType)
+
+    @pytest.mark.unit
+    def test_none(self):
+        result = _convert_value(None)
+        assert isinstance(result, celpy.celtypes.BoolType)
+        assert bool(result) is False
+
+    @pytest.mark.unit
+    def test_other_type_converts_to_string(self):
+        result = _convert_value(object())
+        assert isinstance(result, celpy.celtypes.StringType)
+
+
+class TestBuildActivation:
+
+    @pytest.mark.unit
+    def test_converts_dict_values(self):
+        state = {"name": "Alice", "age": 30, "active": True}
+        result = build_activation(state)
+        assert "name" in result
+        assert "age" in result
+        assert "active" in result
+
+    @pytest.mark.unit
+    def test_empty_state(self):
+        assert build_activation({}) == {}
+
+
+class TestEvaluateCel:
+
+    @pytest.mark.unit
+    def test_simple_comparison(self):
+        assert evaluate_cel("x > 5", {"x": 10}) is True
+        assert evaluate_cel("x > 5", {"x": 3}) is False
+
+    @pytest.mark.unit
+    def test_string_comparison(self):
+        assert evaluate_cel('name == "Alice"', {"name": "Alice"}) is True
+        assert evaluate_cel('name == "Alice"', {"name": "Bob"}) is False
+
+    @pytest.mark.unit
+    def test_arithmetic(self):
+        assert evaluate_cel("x + y", {"x": 3, "y": 4}) == 7
+
+    @pytest.mark.unit
+    def test_boolean_logic(self):
+        assert evaluate_cel("a && b", {"a": True, "b": True}) is True
+        assert evaluate_cel("a && b", {"a": True, "b": False}) is False
+        assert evaluate_cel("a || b", {"a": False, "b": True}) is True
+
+    @pytest.mark.unit
+    def test_empty_expression_raises(self):
+        with pytest.raises(CelEvaluationError, match="Empty expression"):
+            evaluate_cel("", {})
+
+    @pytest.mark.unit
+    def test_whitespace_expression_raises(self):
+        with pytest.raises(CelEvaluationError, match="Empty expression"):
+            evaluate_cel("   ", {})
+
+    @pytest.mark.unit
+    def test_invalid_expression_raises(self):
+        with pytest.raises(CelEvaluationError):
+            evaluate_cel("invalid!!!", {})
+
+    @pytest.mark.unit
+    def test_missing_variable_raises(self):
+        with pytest.raises(CelEvaluationError):
+            evaluate_cel("undefined_var > 5", {})
+
+
+class TestCelToPython:
+
+    @pytest.mark.unit
+    def test_bool(self):
+        result = cel_to_python(celpy.celtypes.BoolType(True))
+        assert result is True
+
+    @pytest.mark.unit
+    def test_int(self):
+        result = cel_to_python(celpy.celtypes.IntType(42))
+        assert result == 42
+
+    @pytest.mark.unit
+    def test_double(self):
+        result = cel_to_python(celpy.celtypes.DoubleType(3.14))
+        assert result == pytest.approx(3.14)
+
+    @pytest.mark.unit
+    def test_string(self):
+        result = cel_to_python(celpy.celtypes.StringType("hello"))
+        assert result == "hello"
+
+    @pytest.mark.unit
+    def test_list(self):
+        cel_list = celpy.celtypes.ListType([
+            celpy.celtypes.IntType(1),
+            celpy.celtypes.IntType(2),
+        ])
+        result = cel_to_python(cel_list)
+        assert result == [1, 2]
+
+    @pytest.mark.unit
+    def test_map(self):
+        cel_map = celpy.celtypes.MapType({
+            celpy.celtypes.StringType("key"): celpy.celtypes.StringType("value"),
+        })
+        result = cel_to_python(cel_map)
+        assert result == {"key": "value"}
+
+    @pytest.mark.unit
+    def test_unknown_type_passthrough(self):
+        result = cel_to_python("raw_value")
+        assert result == "raw_value"

tests/agents/test_cryptoprice_tool.py

@@ -0,0 +1,85 @@
+"""Tests for application/agents/tools/cryptoprice.py"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from application.agents.tools.cryptoprice import CryptoPriceTool
+
+
+@pytest.fixture
+def tool():
+    return CryptoPriceTool(config={})
+
+
+@pytest.mark.unit
+class TestCryptoPriceExecuteAction:
+    def test_unknown_action_raises(self, tool):
+        with pytest.raises(ValueError, match="Unknown action"):
+            tool.execute_action("invalid_action")
+
+    @patch("application.agents.tools.cryptoprice.requests.get")
+    def test_successful_price_fetch(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.json.return_value = {"USD": 65000}
+        mock_get.return_value = mock_resp
+
+        result = tool.execute_action("cryptoprice_get", symbol="BTC", currency="USD")
+
+        assert result["status_code"] == 200
+        assert result["price"] == 65000
+        assert "successfully" in result["message"]
+
+    @patch("application.agents.tools.cryptoprice.requests.get")
+    def test_currency_not_found(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.json.return_value = {"EUR": 60000}
+        mock_get.return_value = mock_resp
+
+        result = tool.execute_action("cryptoprice_get", symbol="BTC", currency="USD")
+
+        assert result["status_code"] == 200
+        assert "Couldn't find" in result["message"]
+        assert "price" not in result
+
+    @patch("application.agents.tools.cryptoprice.requests.get")
+    def test_api_failure(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 500
+        mock_get.return_value = mock_resp
+
+        result = tool.execute_action("cryptoprice_get", symbol="BTC", currency="USD")
+
+        assert result["status_code"] == 500
+        assert "Failed" in result["message"]
+
+    @patch("application.agents.tools.cryptoprice.requests.get")
+    def test_symbol_case_insensitive(self, mock_get, tool):
+        mock_resp = MagicMock()
+        mock_resp.status_code = 200
+        mock_resp.json.return_value = {"USD": 100}
+        mock_get.return_value = mock_resp
+
+        tool.execute_action("cryptoprice_get", symbol="btc", currency="usd")
+
+        called_url = mock_get.call_args[0][0]
+        assert "fsym=BTC" in called_url
+        assert "tsyms=USD" in called_url
+
+
+@pytest.mark.unit
+class TestCryptoPriceMetadata:
+    def test_actions_metadata(self, tool):
+        meta = tool.get_actions_metadata()
+        assert len(meta) == 1
+        assert meta[0]["name"] == "cryptoprice_get"
+        params = meta[0]["parameters"]
+        assert "symbol" in params["properties"]
+        assert "currency" in params["properties"]
+        assert "symbol" in params["required"]
+        assert "currency" in params["required"]
+
+    def test_config_requirements(self, tool):
+        assert tool.get_config_requirements() == {}

tests/agents/test_duckduckgo_tool.py

@@ -0,0 +1,145 @@
+"""Tests for application/agents/tools/duckduckgo.py"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from application.agents.tools.duckduckgo import DuckDuckGoSearchTool
+
+
+@pytest.fixture
+def tool():
+    return DuckDuckGoSearchTool(config={})
+
+
+@pytest.mark.unit
+class TestDuckDuckGoExecuteAction:
+    def test_unknown_action_raises(self, tool):
+        with pytest.raises(ValueError, match="Unknown action"):
+            tool.execute_action("invalid")
+
+    @patch.object(DuckDuckGoSearchTool, "_get_ddgs_client")
+    def test_web_search_success(self, mock_client_factory, tool):
+        mock_client = MagicMock()
+        mock_client.text.return_value = [
+            {"title": "Result 1", "href": "https://example.com", "body": "snippet"}
+        ]
+        mock_client_factory.return_value = mock_client
+
+        result = tool.execute_action("ddg_web_search", query="python")
+
+        assert result["status_code"] == 200
+        assert len(result["results"]) == 1
+        assert "successfully" in result["message"]
+
+    @patch.object(DuckDuckGoSearchTool, "_get_ddgs_client")
+    def test_image_search_success(self, mock_client_factory, tool):
+        mock_client = MagicMock()
+        mock_client.images.return_value = [{"image": "https://img.com/1.jpg"}]
+        mock_client_factory.return_value = mock_client
+
+        result = tool.execute_action("ddg_image_search", query="cats")
+
+        assert result["status_code"] == 200
+        assert len(result["results"]) == 1
+
+    @patch.object(DuckDuckGoSearchTool, "_get_ddgs_client")
+    def test_news_search_success(self, mock_client_factory, tool):
+        mock_client = MagicMock()
+        mock_client.news.return_value = [{"title": "News"}]
+        mock_client_factory.return_value = mock_client
+
+        result = tool.execute_action("ddg_news_search", query="tech")
+
+        assert result["status_code"] == 200
+        assert len(result["results"]) == 1
+
+    @patch.object(DuckDuckGoSearchTool, "_get_ddgs_client")
+    def test_search_error_returns_500(self, mock_client_factory, tool):
+        mock_client = MagicMock()
+        mock_client.text.side_effect = Exception("Network error")
+        mock_client_factory.return_value = mock_client
+
+        result = tool.execute_action("ddg_web_search", query="test")
+
+        assert result["status_code"] == 500
+        assert "failed" in result["message"].lower()
+        assert result["results"] == []
+
+    @patch.object(DuckDuckGoSearchTool, "_get_ddgs_client")
+    def test_max_results_capped_at_20(self, mock_client_factory, tool):
+        mock_client = MagicMock()
+        mock_client.text.return_value = []
+        mock_client_factory.return_value = mock_client
+
+        tool.execute_action("ddg_web_search", query="test", max_results=100)
+
+        call_kwargs = mock_client.text.call_args[1]
+        assert call_kwargs["max_results"] == 20
+
+    @patch.object(DuckDuckGoSearchTool, "_get_ddgs_client")
+    def test_image_max_results_capped_at_50(self, mock_client_factory, tool):
+        mock_client = MagicMock()
+        mock_client.images.return_value = []
+        mock_client_factory.return_value = mock_client
+
+        tool.execute_action("ddg_image_search", query="test", max_results=200)
+
+        call_kwargs = mock_client.images.call_args[1]
+        assert call_kwargs["max_results"] == 50
+
+    @patch("application.agents.tools.duckduckgo.time.sleep")
+    @patch.object(DuckDuckGoSearchTool, "_get_ddgs_client")
+    def test_rate_limit_retries(self, mock_client_factory, mock_sleep, tool):
+        mock_client = MagicMock()
+        mock_client.text.side_effect = [
+            Exception("RateLimit exceeded"),
+            [{"title": "Result"}],
+        ]
+        mock_client_factory.return_value = mock_client
+
+        result = tool.execute_action("ddg_web_search", query="test")
+
+        assert result["status_code"] == 200
+        assert len(result["results"]) == 1
+        mock_sleep.assert_called_once()
+
+    @patch.object(DuckDuckGoSearchTool, "_get_ddgs_client")
+    def test_empty_results(self, mock_client_factory, tool):
+        mock_client = MagicMock()
+        mock_client.text.return_value = []
+        mock_client_factory.return_value = mock_client
+
+        result = tool.execute_action("ddg_web_search", query="obscure query")
+
+        assert result["status_code"] == 200
+        assert result["results"] == []
+
+    @patch.object(DuckDuckGoSearchTool, "_get_ddgs_client")
+    def test_none_results(self, mock_client_factory, tool):
+        mock_client = MagicMock()
+        mock_client.text.return_value = None
+        mock_client_factory.return_value = mock_client
+
+        result = tool.execute_action("ddg_web_search", query="test")
+
+        assert result["status_code"] == 200
+        assert result["results"] == []
+
+
+@pytest.mark.unit
+class TestDuckDuckGoMetadata:
+    def test_actions_metadata(self, tool):
+        meta = tool.get_actions_metadata()
+        assert len(meta) == 3
+        names = {a["name"] for a in meta}
+        assert "ddg_web_search" in names
+        assert "ddg_image_search" in names
+        assert "ddg_news_search" in names
+
+    def test_config_requirements(self, tool):
+        assert tool.get_config_requirements() == {}
+
+    def test_custom_timeout(self):
+        tool = DuckDuckGoSearchTool(config={"timeout": 30})
+        assert tool.timeout == 30

tests/agents/test_internal_search_tool.py

@@ -0,0 +1,250 @@
+"""Tests for InternalSearchTool and its helper functions."""
+
+from unittest.mock import Mock, patch
+
+import pytest
+from application.agents.tools.internal_search import (
+    INTERNAL_TOOL_ID,
+    InternalSearchTool,
+    add_internal_search_tool,
+    build_internal_tool_config,
+    build_internal_tool_entry,
+)
+
+
+@pytest.mark.unit
+class TestInternalSearchToolSearch:
+
+    def _make_tool(self, **config_overrides):
+        config = {"source": {}, "retriever_name": "classic", "chunks": 2}
+        config.update(config_overrides)
+        return InternalSearchTool(config)
+
+    def test_search_no_query_returns_error(self):
+        tool = self._make_tool()
+        result = tool.execute_action("search", query="")
+        assert "required" in result.lower()
+
+    def test_search_returns_formatted_docs(self):
+        tool = self._make_tool()
+        mock_retriever = Mock()
+        mock_retriever.search.return_value = [
+            {"text": "Hello world", "title": "Doc1", "source": "test", "filename": "doc1.md"},
+        ]
+        tool._retriever = mock_retriever
+
+        result = tool.execute_action("search", query="hello")
+        assert "doc1.md" in result
+        assert "Hello world" in result
+        assert len(tool.retrieved_docs) == 1
+
+    def test_search_no_results(self):
+        tool = self._make_tool()
+        mock_retriever = Mock()
+        mock_retriever.search.return_value = []
+        tool._retriever = mock_retriever
+
+        result = tool.execute_action("search", query="nonexistent")
+        assert "No documents found" in result
+
+    def test_search_accumulates_docs(self):
+        tool = self._make_tool()
+        mock_retriever = Mock()
+        tool._retriever = mock_retriever
+
+        mock_retriever.search.return_value = [
+            {"text": "A", "title": "D1", "source": "s1"},
+        ]
+        tool.execute_action("search", query="first")
+
+        mock_retriever.search.return_value = [
+            {"text": "B", "title": "D2", "source": "s2"},
+        ]
+        tool.execute_action("search", query="second")
+
+        assert len(tool.retrieved_docs) == 2
+
+    def test_search_deduplicates_docs(self):
+        tool = self._make_tool()
+        doc = {"text": "Same", "title": "Same", "source": "same"}
+        mock_retriever = Mock()
+        mock_retriever.search.return_value = [doc]
+        tool._retriever = mock_retriever
+
+        tool.execute_action("search", query="q1")
+        tool.execute_action("search", query="q2")
+
+        assert len(tool.retrieved_docs) == 1
+
+    def test_search_with_path_filter(self):
+        tool = self._make_tool()
+        mock_retriever = Mock()
+        mock_retriever.search.return_value = [
+            {"text": "A", "title": "T", "source": "src/main.py", "filename": "main.py"},
+            {"text": "B", "title": "T", "source": "docs/readme.md", "filename": "readme.md"},
+        ]
+        tool._retriever = mock_retriever
+
+        result = tool.execute_action("search", query="code", path_filter="src/")
+        assert "main.py" in result
+        assert "readme.md" not in result
+
+    def test_search_path_filter_no_match(self):
+        tool = self._make_tool()
+        mock_retriever = Mock()
+        mock_retriever.search.return_value = [
+            {"text": "A", "title": "T", "source": "other/file.txt"},
+        ]
+        tool._retriever = mock_retriever
+
+        result = tool.execute_action("search", query="code", path_filter="src/")
+        assert "No documents found" in result
+
+    def test_search_retriever_error(self):
+        tool = self._make_tool()
+        mock_retriever = Mock()
+        mock_retriever.search.side_effect = Exception("Connection error")
+        tool._retriever = mock_retriever
+
+        result = tool.execute_action("search", query="test")
+        assert "failed" in result.lower() or "error" in result.lower()
+
+    def test_unknown_action(self):
+        tool = self._make_tool()
+        result = tool.execute_action("nonexistent")
+        assert "Unknown action" in result
+
+
+@pytest.mark.unit
+class TestInternalSearchToolListFiles:
+
+    def test_list_files_no_structure(self):
+        tool = InternalSearchTool({"source": {}})
+        tool._dir_structure_loaded = True
+        tool._directory_structure = None
+
+        result = tool.execute_action("list_files")
+        assert "No file structure" in result
+
+    def test_list_files_root(self):
+        tool = InternalSearchTool({"source": {}})
+        tool._dir_structure_loaded = True
+        tool._directory_structure = {
+            "src": {"main.py": {}},
+            "README.md": {"type": "md", "token_count": 100},
+        }
+
+        result = tool.execute_action("list_files")
+        assert "src/" in result
+        assert "README.md" in result
+
+    def test_list_files_nested_path(self):
+        tool = InternalSearchTool({"source": {}})
+        tool._dir_structure_loaded = True
+        tool._directory_structure = {
+            "src": {
+                "utils": {"helper.py": {}},
+            },
+        }
+
+        result = tool.execute_action("list_files", path="src")
+        assert "utils/" in result
+
+    def test_list_files_invalid_path(self):
+        tool = InternalSearchTool({"source": {}})
+        tool._dir_structure_loaded = True
+        tool._directory_structure = {"src": {}}
+
+        result = tool.execute_action("list_files", path="nonexistent")
+        assert "not found" in result
+
+
+@pytest.mark.unit
+class TestInternalSearchToolMetadata:
+
+    def test_actions_without_directory_structure(self):
+        tool = InternalSearchTool({"has_directory_structure": False})
+        meta = tool.get_actions_metadata()
+
+        action_names = [a["name"] for a in meta]
+        assert "search" in action_names
+        assert "list_files" not in action_names
+
+        # search should not have path_filter
+        search = meta[0]
+        assert "path_filter" not in search["parameters"]["properties"]
+
+    def test_actions_with_directory_structure(self):
+        tool = InternalSearchTool({"has_directory_structure": True})
+        meta = tool.get_actions_metadata()
+
+        action_names = [a["name"] for a in meta]
+        assert "search" in action_names
+        assert "list_files" in action_names
+
+        # search should have path_filter
+        search = next(a for a in meta if a["name"] == "search")
+        assert "path_filter" in search["parameters"]["properties"]
+
+
+@pytest.mark.unit
+class TestBuildHelpers:
+
+    def test_build_entry_without_directory_structure(self):
+        entry = build_internal_tool_entry(has_directory_structure=False)
+        assert entry["name"] == "internal_search"
+        action_names = [a["name"] for a in entry["actions"]]
+        assert "search" in action_names
+        assert "list_files" not in action_names
+
+    def test_build_entry_with_directory_structure(self):
+        entry = build_internal_tool_entry(has_directory_structure=True)
+        action_names = [a["name"] for a in entry["actions"]]
+        assert "list_files" in action_names
+
+    def test_build_config(self):
+        config = build_internal_tool_config(
+            source={"active_docs": ["abc"]},
+            retriever_name="semantic",
+            chunks=4,
+        )
+        assert config["source"] == {"active_docs": ["abc"]}
+        assert config["retriever_name"] == "semantic"
+        assert config["chunks"] == 4
+
+    def test_internal_tool_id(self):
+        assert INTERNAL_TOOL_ID == "internal"
+
+    def test_add_internal_search_tool_with_sources(self):
+        tools_dict = {}
+        retriever_config = {
+            "source": {"active_docs": ["abc"]},
+            "retriever_name": "classic",
+            "chunks": 2,
+            "model_id": "gpt-4",
+            "llm_name": "openai",
+            "api_key": "key",
+        }
+
+        with patch(
+            "application.agents.tools.internal_search.sources_have_directory_structure",
+            return_value=False,
+        ):
+            add_internal_search_tool(tools_dict, retriever_config)
+
+        assert INTERNAL_TOOL_ID in tools_dict
+        assert tools_dict[INTERNAL_TOOL_ID]["name"] == "internal_search"
+        assert "config" in tools_dict[INTERNAL_TOOL_ID]
+
+    def test_add_internal_search_tool_no_sources(self):
+        tools_dict = {}
+        retriever_config = {"source": {}}
+
+        add_internal_search_tool(tools_dict, retriever_config)
+
+        assert INTERNAL_TOOL_ID not in tools_dict
+
+    def test_add_internal_search_tool_empty_config(self):
+        tools_dict = {}
+        add_internal_search_tool(tools_dict, {})
+        assert INTERNAL_TOOL_ID not in tools_dict