tests: retriever and seeder

tests/test_namespaces.py

@@ -0,0 +1,279 @@
+from datetime import datetime, timezone
+from unittest.mock import patch
+
+import pytest
+
+from application.templates.namespaces import (
+    NamespaceBuilder,
+    NamespaceManager,
+    PassthroughNamespace,
+    SourceNamespace,
+    SystemNamespace,
+    ToolsNamespace,
+)
+
+
+# ── SystemNamespace ────────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestSystemNamespace:
+    def test_namespace_name(self):
+        ns = SystemNamespace()
+        assert ns.namespace_name == "system"
+
+    def test_build_returns_expected_keys(self):
+        ns = SystemNamespace()
+        result = ns.build()
+        assert "date" in result
+        assert "time" in result
+        assert "timestamp" in result
+        assert "request_id" in result
+        assert "user_id" in result
+
+    def test_build_with_request_id(self):
+        ns = SystemNamespace()
+        result = ns.build(request_id="req-123")
+        assert result["request_id"] == "req-123"
+
+    def test_build_with_user_id(self):
+        ns = SystemNamespace()
+        result = ns.build(user_id="user-456")
+        assert result["user_id"] == "user-456"
+
+    def test_build_generates_uuid_when_no_request_id(self):
+        ns = SystemNamespace()
+        result = ns.build()
+        assert len(result["request_id"]) == 36  # UUID format
+
+    def test_user_id_defaults_to_none(self):
+        ns = SystemNamespace()
+        result = ns.build()
+        assert result["user_id"] is None
+
+    def test_date_format(self):
+        ns = SystemNamespace()
+        fixed = datetime(2026, 1, 15, 10, 30, 45, tzinfo=timezone.utc)
+        with patch("application.templates.namespaces.datetime") as mock_dt:
+            mock_dt.now.return_value = fixed
+            mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw)
+            result = ns.build()
+        assert result["date"] == "2026-01-15"
+        assert result["time"] == "10:30:45"
+
+    def test_extra_kwargs_ignored(self):
+        ns = SystemNamespace()
+        result = ns.build(unknown_param="whatever")
+        assert "date" in result
+
+
+# ── PassthroughNamespace ───────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestPassthroughNamespace:
+    def test_namespace_name(self):
+        ns = PassthroughNamespace()
+        assert ns.namespace_name == "passthrough"
+
+    def test_none_data_returns_empty(self):
+        ns = PassthroughNamespace()
+        assert ns.build(passthrough_data=None) == {}
+
+    def test_no_data_returns_empty(self):
+        ns = PassthroughNamespace()
+        assert ns.build() == {}
+
+    def test_safe_types_pass_through(self):
+        ns = PassthroughNamespace()
+        data = {"s": "string", "i": 42, "f": 3.14, "b": True, "n": None}
+        result = ns.build(passthrough_data=data)
+        assert result == data
+
+    def test_non_serializable_types_filtered(self):
+        ns = PassthroughNamespace()
+        data = {"good": "ok", "bad": object()}
+        result = ns.build(passthrough_data=data)
+        assert result == {"good": "ok"}
+
+    def test_list_value_filtered(self):
+        ns = PassthroughNamespace()
+        data = {"list_val": [1, 2, 3]}
+        result = ns.build(passthrough_data=data)
+        assert result == {}
+
+    def test_dict_value_filtered(self):
+        ns = PassthroughNamespace()
+        data = {"nested": {"key": "val"}}
+        result = ns.build(passthrough_data=data)
+        assert result == {}
+
+
+# ── SourceNamespace ────────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestSourceNamespace:
+    def test_namespace_name(self):
+        ns = SourceNamespace()
+        assert ns.namespace_name == "source"
+
+    def test_no_docs_returns_empty(self):
+        ns = SourceNamespace()
+        assert ns.build() == {}
+
+    def test_with_docs(self):
+        ns = SourceNamespace()
+        docs = [{"text": "doc1"}, {"text": "doc2"}]
+        result = ns.build(docs=docs)
+        assert result["documents"] == docs
+        assert result["count"] == 2
+
+    def test_with_docs_together(self):
+        ns = SourceNamespace()
+        result = ns.build(docs_together="all content together")
+        assert result["content"] == "all content together"
+        assert result["docs_together"] == "all content together"
+        assert result["summaries"] == "all content together"
+
+    def test_with_both_docs_and_docs_together(self):
+        ns = SourceNamespace()
+        docs = [{"text": "doc1"}]
+        result = ns.build(docs=docs, docs_together="combined")
+        assert result["documents"] == docs
+        assert result["count"] == 1
+        assert result["content"] == "combined"
+
+    def test_empty_docs_list_returns_empty(self):
+        ns = SourceNamespace()
+        result = ns.build(docs=[])
+        assert "documents" not in result
+
+
+# ── ToolsNamespace ─────────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestToolsNamespace:
+    def test_namespace_name(self):
+        ns = ToolsNamespace()
+        assert ns.namespace_name == "tools"
+
+    def test_none_data_returns_empty(self):
+        ns = ToolsNamespace()
+        assert ns.build(tools_data=None) == {}
+
+    def test_no_data_returns_empty(self):
+        ns = ToolsNamespace()
+        assert ns.build() == {}
+
+    def test_safe_types_pass_through(self):
+        ns = ToolsNamespace()
+        data = {
+            "str_tool": "result",
+            "dict_tool": {"key": "val"},
+            "list_tool": [1, 2],
+            "int_tool": 42,
+            "float_tool": 3.14,
+            "bool_tool": True,
+            "none_tool": None,
+        }
+        result = ns.build(tools_data=data)
+        assert result == data
+
+    def test_non_serializable_filtered(self):
+        ns = ToolsNamespace()
+        data = {"good": "ok", "bad": object()}
+        result = ns.build(tools_data=data)
+        assert result == {"good": "ok"}
+
+
+# ── NamespaceBuilder ABC ──────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestNamespaceBuilderABC:
+    def test_cannot_instantiate(self):
+        with pytest.raises(TypeError):
+            NamespaceBuilder()
+
+    def test_subclass_must_implement_both(self):
+        class Incomplete(NamespaceBuilder):
+            pass
+
+        with pytest.raises(TypeError):
+            Incomplete()
+
+    def test_concrete_subclass_works(self):
+        class Complete(NamespaceBuilder):
+            @property
+            def namespace_name(self):
+                return "test"
+
+            def build(self, **kwargs):
+                return {"ok": True}
+
+        inst = Complete()
+        assert inst.namespace_name == "test"
+        assert inst.build() == {"ok": True}
+
+
+# ── NamespaceManager ──────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestNamespaceManager:
+    def test_build_context_contains_all_namespaces(self):
+        mgr = NamespaceManager()
+        ctx = mgr.build_context()
+        assert "system" in ctx
+        assert "passthrough" in ctx
+        assert "source" in ctx
+        assert "tools" in ctx
+
+    def test_system_namespace_populated(self):
+        mgr = NamespaceManager()
+        ctx = mgr.build_context(request_id="r1", user_id="u1")
+        assert ctx["system"]["request_id"] == "r1"
+        assert ctx["system"]["user_id"] == "u1"
+
+    def test_passthrough_namespace_populated(self):
+        mgr = NamespaceManager()
+        ctx = mgr.build_context(passthrough_data={"key": "val"})
+        assert ctx["passthrough"] == {"key": "val"}
+
+    def test_source_namespace_populated(self):
+        mgr = NamespaceManager()
+        docs = [{"text": "doc"}]
+        ctx = mgr.build_context(docs=docs)
+        assert ctx["source"]["count"] == 1
+
+    def test_tools_namespace_populated(self):
+        mgr = NamespaceManager()
+        ctx = mgr.build_context(tools_data={"search": "results"})
+        assert ctx["tools"] == {"search": "results"}
+
+    def test_empty_kwargs_all_namespaces_present(self):
+        mgr = NamespaceManager()
+        ctx = mgr.build_context()
+        for ns in ["system", "passthrough", "source", "tools"]:
+            assert ns in ctx
+            assert isinstance(ctx[ns], dict)
+
+    def test_builder_exception_returns_empty_namespace(self):
+        mgr = NamespaceManager()
+        with patch.object(
+            mgr._builders["system"], "build", side_effect=RuntimeError("boom")
+        ):
+            ctx = mgr.build_context()
+        assert ctx["system"] == {}
+        assert "passthrough" in ctx
+
+    def test_get_builder_existing(self):
+        mgr = NamespaceManager()
+        builder = mgr.get_builder("system")
+        assert isinstance(builder, SystemNamespace)
+
+    def test_get_builder_nonexistent(self):
+        mgr = NamespaceManager()
+        assert mgr.get_builder("nonexistent") is None

tests/test_retriever.py

@@ -0,0 +1,353 @@
+from unittest.mock import MagicMock, Mock, patch
+
+import pytest
+
+from application.retriever.base import BaseRetriever
+from application.retriever.retriever_creator import RetrieverCreator
+
+
+# ── BaseRetriever ──────────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestBaseRetriever:
+    def test_cannot_instantiate_directly(self):
+        with pytest.raises(TypeError):
+            BaseRetriever()
+
+    def test_subclass_must_implement_search(self):
+        class Incomplete(BaseRetriever):
+            pass
+
+        with pytest.raises(TypeError):
+            Incomplete()
+
+    def test_concrete_subclass_works(self):
+        class Concrete(BaseRetriever):
+            def search(self, *args, **kwargs):
+                return "ok"
+
+        instance = Concrete()
+        assert instance.search() == "ok"
+
+
+# ── RetrieverCreator ───────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestRetrieverCreator:
+    def test_create_classic(self):
+        mock_cls = Mock(return_value="rag_instance")
+        original = RetrieverCreator.retrievers.copy()
+        RetrieverCreator.retrievers["classic"] = mock_cls
+        try:
+            result = RetrieverCreator.create_retriever("classic", "arg1", key="val")
+            mock_cls.assert_called_once_with("arg1", key="val")
+            assert result == "rag_instance"
+        finally:
+            RetrieverCreator.retrievers.update(original)
+
+    def test_create_default(self):
+        mock_cls = Mock(return_value="rag_instance")
+        original = RetrieverCreator.retrievers.copy()
+        RetrieverCreator.retrievers["default"] = mock_cls
+        try:
+            result = RetrieverCreator.create_retriever("default")
+            mock_cls.assert_called_once_with()
+            assert result == "rag_instance"
+        finally:
+            RetrieverCreator.retrievers.update(original)
+
+    def test_create_none_type_uses_default(self):
+        mock_cls = Mock(return_value="rag_instance")
+        original = RetrieverCreator.retrievers.copy()
+        RetrieverCreator.retrievers["default"] = mock_cls
+        try:
+            result = RetrieverCreator.create_retriever(None)
+            mock_cls.assert_called_once()
+            assert result == "rag_instance"
+        finally:
+            RetrieverCreator.retrievers.update(original)
+
+    def test_case_insensitive(self):
+        mock_cls = Mock(return_value="rag_instance")
+        original = RetrieverCreator.retrievers.copy()
+        RetrieverCreator.retrievers["classic"] = mock_cls
+        try:
+            RetrieverCreator.create_retriever("CLASSIC")
+            mock_cls.assert_called_once()
+        finally:
+            RetrieverCreator.retrievers.update(original)
+
+    def test_invalid_type_raises(self):
+        with pytest.raises(ValueError, match="No retievers class found"):
+            RetrieverCreator.create_retriever("nonexistent")
+
+
+# ── ClassicRAG ─────────────────────────────────────────────────────────────────
+
+
+@pytest.fixture
+def _patch_llm_creator(mock_llm, monkeypatch):
+    """Patch LLMCreator.create_llm to return the shared mock_llm fixture."""
+    monkeypatch.setattr(
+        "application.retriever.classic_rag.LLMCreator.create_llm",
+        Mock(return_value=mock_llm),
+    )
+    return mock_llm
+
+
+def _make_rag(source=None, _patch_llm_creator=None, **overrides):
+    """Helper – builds a ClassicRAG with sensible defaults."""
+    from application.retriever.classic_rag import ClassicRAG
+
+    defaults = dict(
+        source=source or {"question": "hello"},
+        chat_history=None,
+        prompt="",
+        chunks=2,
+        doc_token_limit=50000,
+        model_id="test-model",
+        user_api_key=None,
+        agent_id=None,
+        llm_name="openai",
+        api_key="fake",
+        decoded_token={"sub": "user1"},
+    )
+    defaults.update(overrides)
+    return ClassicRAG(**defaults)
+
+
+@pytest.mark.unit
+class TestClassicRAGInit:
+    def test_basic_init(self, _patch_llm_creator):
+        rag = _make_rag()
+        assert rag.original_question == "hello"
+        assert rag.chunks == 2
+        assert rag.vectorstores == []
+
+    def test_active_docs_as_list(self, _patch_llm_creator):
+        rag = _make_rag(source={"question": "q", "active_docs": ["a", "b"]})
+        assert rag.vectorstores == ["a", "b"]
+
+    def test_active_docs_as_string(self, _patch_llm_creator):
+        rag = _make_rag(source={"question": "q", "active_docs": "single"})
+        assert rag.vectorstores == ["single"]
+
+    def test_active_docs_none(self, _patch_llm_creator):
+        rag = _make_rag(source={"question": "q", "active_docs": None})
+        assert rag.vectorstores == []
+
+    def test_chunks_string_converted(self, _patch_llm_creator):
+        rag = _make_rag(chunks="5")
+        assert rag.chunks == 5
+
+    def test_chunks_invalid_string_defaults(self, _patch_llm_creator):
+        rag = _make_rag(chunks="abc")
+        assert rag.chunks == 2
+
+    def test_decoded_token_none(self, _patch_llm_creator):
+        rag = _make_rag(decoded_token=None)
+        assert rag.decoded_token is None
+
+
+@pytest.mark.unit
+class TestClassicRAGValidateVectorstore:
+    def test_removes_empty_ids(self, _patch_llm_creator):
+        rag = _make_rag(source={"question": "q", "active_docs": ["ok", "", "  ", "good"]})
+        assert rag.vectorstores == ["ok", "good"]
+
+    def test_empty_vectorstores_no_error(self, _patch_llm_creator):
+        rag = _make_rag(source={"question": "q"})
+        assert rag.vectorstores == []
+
+
+@pytest.mark.unit
+class TestClassicRAGRephraseQuery:
+    def test_no_history_returns_original(self, _patch_llm_creator):
+        rag = _make_rag(
+            source={"question": "original", "active_docs": ["vs1"]},
+            chat_history=[],
+        )
+        assert rag.question == "original"
+
+    def test_no_vectorstores_returns_original(self, _patch_llm_creator):
+        rag = _make_rag(
+            source={"question": "original"},
+            chat_history=[{"prompt": "hi", "response": "hello"}],
+        )
+        assert rag.question == "original"
+
+    def test_chunks_zero_returns_original(self, _patch_llm_creator):
+        rag = _make_rag(
+            source={"question": "original", "active_docs": ["vs1"]},
+            chat_history=[{"prompt": "hi", "response": "hello"}],
+            chunks=0,
+        )
+        assert rag.question == "original"
+
+    def test_rephrase_called_with_history(self, _patch_llm_creator, mock_llm):
+        mock_llm.gen = Mock(return_value="rephrased question")
+        rag = _make_rag(
+            source={"question": "original", "active_docs": ["vs1"]},
+            chat_history=[{"prompt": "hi", "response": "hello"}],
+        )
+        assert rag.question == "rephrased question"
+        mock_llm.gen.assert_called_once()
+
+    def test_rephrase_llm_returns_empty_falls_back(self, _patch_llm_creator, mock_llm):
+        mock_llm.gen = Mock(return_value="")
+        rag = _make_rag(
+            source={"question": "original", "active_docs": ["vs1"]},
+            chat_history=[{"prompt": "hi", "response": "hello"}],
+        )
+        assert rag.question == "original"
+
+    def test_rephrase_llm_exception_falls_back(self, _patch_llm_creator, mock_llm):
+        mock_llm.gen = Mock(side_effect=RuntimeError("boom"))
+        rag = _make_rag(
+            source={"question": "original", "active_docs": ["vs1"]},
+            chat_history=[{"prompt": "hi", "response": "hello"}],
+        )
+        assert rag.question == "original"
+
+
+@pytest.mark.unit
+class TestClassicRAGGetData:
+    def test_chunks_zero_returns_empty(self, _patch_llm_creator):
+        rag = _make_rag(chunks=0)
+        assert rag._get_data() == []
+
+    def test_no_vectorstores_returns_empty(self, _patch_llm_creator):
+        rag = _make_rag(source={"question": "q"})
+        assert rag._get_data() == []
+
+    @patch("application.retriever.classic_rag.VectorCreator")
+    @patch("application.retriever.classic_rag.num_tokens_from_string", return_value=10)
+    def test_returns_docs_with_metadata(self, mock_tokens, mock_vc, _patch_llm_creator):
+        mock_docsearch = MagicMock()
+        mock_doc = MagicMock()
+        mock_doc.page_content = "content here"
+        mock_doc.metadata = {
+            "title": "path/to/Title",
+            "filename": "/docs/file.txt",
+            "source": "http://example.com",
+        }
+        mock_docsearch.search.return_value = [mock_doc]
+        mock_vc.create_vectorstore.return_value = mock_docsearch
+
+        rag = _make_rag(source={"question": "q", "active_docs": ["vs1"]})
+        docs = rag._get_data()
+
+        assert len(docs) == 1
+        assert docs[0]["text"] == "content here"
+        assert docs[0]["title"] == "Title"
+        assert docs[0]["filename"] == "file.txt"
+        assert docs[0]["source"] == "http://example.com"
+
+    @patch("application.retriever.classic_rag.VectorCreator")
+    @patch("application.retriever.classic_rag.num_tokens_from_string", return_value=10)
+    def test_dict_style_docs(self, mock_tokens, mock_vc, _patch_llm_creator):
+        mock_docsearch = MagicMock()
+        mock_docsearch.search.return_value = [
+            {"text": "dict content", "metadata": {"title": "Dict Title"}}
+        ]
+        mock_vc.create_vectorstore.return_value = mock_docsearch
+
+        rag = _make_rag(source={"question": "q", "active_docs": ["vs1"]})
+        docs = rag._get_data()
+
+        assert len(docs) == 1
+        assert docs[0]["text"] == "dict content"
+
+    @patch("application.retriever.classic_rag.VectorCreator")
+    @patch("application.retriever.classic_rag.num_tokens_from_string", return_value=100000)
+    def test_token_budget_respected(self, mock_tokens, mock_vc, _patch_llm_creator):
+        mock_docsearch = MagicMock()
+        mock_doc = MagicMock()
+        mock_doc.page_content = "big content"
+        mock_doc.metadata = {"title": "t"}
+        mock_docsearch.search.return_value = [mock_doc, mock_doc, mock_doc]
+        mock_vc.create_vectorstore.return_value = mock_docsearch
+
+        rag = _make_rag(
+            source={"question": "q", "active_docs": ["vs1"]},
+            doc_token_limit=100,
+        )
+        docs = rag._get_data()
+        # tokens (100000) exceed budget (90), so no docs should be added
+        assert len(docs) == 0
+
+    @patch("application.retriever.classic_rag.VectorCreator")
+    def test_vectorstore_error_continues(self, mock_vc, _patch_llm_creator):
+        mock_vc.create_vectorstore.side_effect = RuntimeError("connection failed")
+
+        rag = _make_rag(source={"question": "q", "active_docs": ["vs1"]})
+        docs = rag._get_data()
+        assert docs == []
+
+    @patch("application.retriever.classic_rag.VectorCreator")
+    @patch("application.retriever.classic_rag.num_tokens_from_string", return_value=10)
+    def test_multiple_vectorstores(self, mock_tokens, mock_vc, _patch_llm_creator):
+        mock_docsearch = MagicMock()
+        mock_doc = MagicMock()
+        mock_doc.page_content = "content"
+        mock_doc.metadata = {"title": "t", "source": "s"}
+        mock_docsearch.search.return_value = [mock_doc]
+        mock_vc.create_vectorstore.return_value = mock_docsearch
+
+        rag = _make_rag(source={"question": "q", "active_docs": ["vs1", "vs2"]})
+        docs = rag._get_data()
+        assert len(docs) == 2
+
+    @patch("application.retriever.classic_rag.VectorCreator")
+    @patch("application.retriever.classic_rag.num_tokens_from_string", return_value=10)
+    def test_doc_missing_filename_uses_title(self, mock_tokens, mock_vc, _patch_llm_creator):
+        mock_docsearch = MagicMock()
+        mock_doc = MagicMock()
+        mock_doc.page_content = "content"
+        mock_doc.metadata = {"title": "MyTitle"}
+        mock_docsearch.search.return_value = [mock_doc]
+        mock_vc.create_vectorstore.return_value = mock_docsearch
+
+        rag = _make_rag(source={"question": "q", "active_docs": ["vs1"]})
+        docs = rag._get_data()
+        assert docs[0]["filename"] == "MyTitle"
+
+    @patch("application.retriever.classic_rag.VectorCreator")
+    @patch("application.retriever.classic_rag.num_tokens_from_string", return_value=10)
+    def test_non_string_title_converted(self, mock_tokens, mock_vc, _patch_llm_creator):
+        mock_docsearch = MagicMock()
+        mock_doc = MagicMock()
+        mock_doc.page_content = "content"
+        mock_doc.metadata = {"title": 42}
+        mock_docsearch.search.return_value = [mock_doc]
+        mock_vc.create_vectorstore.return_value = mock_docsearch
+
+        rag = _make_rag(source={"question": "q", "active_docs": ["vs1"]})
+        docs = rag._get_data()
+        assert docs[0]["title"] == "42"
+
+
+@pytest.mark.unit
+class TestClassicRAGSearch:
+    @patch("application.retriever.classic_rag.VectorCreator")
+    @patch("application.retriever.classic_rag.num_tokens_from_string", return_value=10)
+    def test_search_with_query_override(self, mock_tokens, mock_vc, _patch_llm_creator, mock_llm):
+        mock_docsearch = MagicMock()
+        mock_doc = MagicMock()
+        mock_doc.page_content = "result"
+        mock_doc.metadata = {"title": "t"}
+        mock_docsearch.search.return_value = [mock_doc]
+        mock_vc.create_vectorstore.return_value = mock_docsearch
+        mock_llm.gen = Mock(return_value="")
+
+        rag = _make_rag(source={"question": "original", "active_docs": ["vs1"]})
+        docs = rag.search(query="override query")
+        assert rag.original_question == "override query"
+        assert len(docs) == 1
+
+    def test_search_without_query_uses_default(self, _patch_llm_creator):
+        rag = _make_rag(source={"question": "q"})
+        docs = rag.search()
+        assert docs == []

tests/test_seeder.py

@@ -0,0 +1,494 @@
+import os
+from unittest.mock import MagicMock, Mock, patch, mock_open
+
+import mongomock
+import pytest
+from bson import ObjectId
+from bson.dbref import DBRef
+
+from application.seed.seeder import DatabaseSeeder
+
+
+@pytest.fixture
+def mock_db():
+    client = mongomock.MongoClient()
+    return client["test_docsgpt"]
+
+
+@pytest.fixture
+def seeder(mock_db):
+    return DatabaseSeeder(mock_db)
+
+
+# ── __init__ ───────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestDatabaseSeederInit:
+    def test_collections_set(self, seeder, mock_db):
+        assert seeder.db is mock_db
+        assert seeder.tools_collection == mock_db["user_tools"]
+        assert seeder.sources_collection == mock_db["sources"]
+        assert seeder.agents_collection == mock_db["agents"]
+        assert seeder.prompts_collection == mock_db["prompts"]
+        assert seeder.system_user_id == "system"
+
+
+# ── _is_already_seeded ─────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestIsAlreadySeeded:
+    def test_not_seeded(self, seeder):
+        assert seeder._is_already_seeded() is False
+
+    def test_already_seeded(self, seeder, mock_db):
+        mock_db["agents"].insert_one({"user": "system", "name": "test"})
+        assert seeder._is_already_seeded() is True
+
+
+# ── _process_config ────────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestProcessConfig:
+    def test_env_var_substitution(self, seeder, monkeypatch):
+        monkeypatch.setenv("MY_SECRET", "secret_value")
+        result = seeder._process_config({"key": "${MY_SECRET}"})
+        assert result["key"] == "secret_value"
+
+    def test_missing_env_var_defaults_empty(self, seeder, monkeypatch):
+        monkeypatch.delenv("NONEXISTENT_VAR", raising=False)
+        result = seeder._process_config({"key": "${NONEXISTENT_VAR}"})
+        assert result["key"] == ""
+
+    def test_non_env_value_unchanged(self, seeder):
+        result = seeder._process_config({"key": "plain_value", "num": 42})
+        assert result == {"key": "plain_value", "num": 42}
+
+    def test_partial_env_syntax_unchanged(self, seeder):
+        result = seeder._process_config({"key": "${INCOMPLETE"})
+        assert result["key"] == "${INCOMPLETE"
+
+    def test_empty_config(self, seeder):
+        assert seeder._process_config({}) == {}
+
+
+# ── _handle_prompt ─────────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestHandlePrompt:
+    def test_no_prompt_returns_none(self, seeder):
+        assert seeder._handle_prompt({"name": "agent1"}) is None
+
+    def test_empty_content_returns_none(self, seeder):
+        config = {"name": "agent1", "prompt": {"name": "p", "content": ""}}
+        assert seeder._handle_prompt(config) is None
+
+    def test_creates_prompt(self, seeder, mock_db):
+        config = {
+            "name": "agent1",
+            "prompt": {"name": "My Prompt", "content": "You are helpful."},
+        }
+        result = seeder._handle_prompt(config)
+        assert result is not None
+        doc = mock_db["prompts"].find_one({"name": "My Prompt"})
+        assert doc is not None
+        assert doc["content"] == "You are helpful."
+        assert doc["user"] == "system"
+
+    def test_duplicate_prompt_returns_existing(self, seeder, mock_db):
+        config = {
+            "name": "agent1",
+            "prompt": {"name": "Dup Prompt", "content": "content"},
+        }
+        id1 = seeder._handle_prompt(config)
+        id2 = seeder._handle_prompt(config)
+        assert id1 == id2
+        assert mock_db["prompts"].count_documents({"name": "Dup Prompt"}) == 1
+
+    def test_default_prompt_name(self, seeder, mock_db):
+        config = {"name": "agent1", "prompt": {"content": "hello"}}
+        seeder._handle_prompt(config)
+        doc = mock_db["prompts"].find_one({"name": "agent1 Prompt"})
+        assert doc is not None
+
+    def test_exception_returns_none(self, seeder):
+        with patch.object(
+            seeder.prompts_collection, "find_one", side_effect=RuntimeError("db error")
+        ):
+            config = {
+                "name": "agent1",
+                "prompt": {"name": "p", "content": "c"},
+            }
+            assert seeder._handle_prompt(config) is None
+
+
+# ── _handle_tools ──────────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestHandleTools:
+    def test_no_tools_returns_empty(self, seeder):
+        assert seeder._handle_tools({"name": "agent1"}) == []
+
+    @patch("application.seed.seeder.tool_manager")
+    def test_creates_tool(self, mock_tm, seeder, mock_db):
+        mock_tool = MagicMock()
+        mock_tool.get_actions_metadata.return_value = [{"name": "act1"}]
+        mock_tm.tools = {"my_tool": mock_tool}
+
+        config = {
+            "name": "agent1",
+            "tools": [{"name": "my_tool", "description": "desc"}],
+        }
+        ids = seeder._handle_tools(config)
+        assert len(ids) == 1
+        doc = mock_db["user_tools"].find_one({"name": "my_tool"})
+        assert doc is not None
+        assert doc["user"] == "system"
+
+    @patch("application.seed.seeder.tool_manager")
+    def test_duplicate_tool_returns_existing(self, mock_tm, seeder, mock_db):
+        mock_tool = MagicMock()
+        mock_tool.get_actions_metadata.return_value = []
+        mock_tm.tools = {"my_tool": mock_tool}
+
+        config = {
+            "name": "agent1",
+            "tools": [{"name": "my_tool"}],
+        }
+        ids1 = seeder._handle_tools(config)
+        ids2 = seeder._handle_tools(config)
+        assert ids1 == ids2
+        assert mock_db["user_tools"].count_documents({"name": "my_tool"}) == 1
+
+    def test_tool_exception_continues(self, seeder):
+        config = {
+            "name": "agent1",
+            "tools": [{"name": "broken_tool"}],
+        }
+        # tool_manager.tools will KeyError on "broken_tool"
+        ids = seeder._handle_tools(config)
+        assert ids == []
+
+    @patch("application.seed.seeder.tool_manager")
+    def test_tool_config_env_expansion(self, mock_tm, seeder, monkeypatch):
+        monkeypatch.setenv("TOOL_KEY", "expanded_val")
+        mock_tool = MagicMock()
+        mock_tool.get_actions_metadata.return_value = []
+        mock_tm.tools = {"my_tool": mock_tool}
+
+        config = {
+            "name": "agent1",
+            "tools": [{"name": "my_tool", "config": {"api_key": "${TOOL_KEY}"}}],
+        }
+        seeder._handle_tools(config)
+        doc = seeder.tools_collection.find_one({"name": "my_tool"})
+        assert doc["config"]["api_key"] == "expanded_val"
+
+
+# ── _handle_source ─────────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestHandleSource:
+    def test_no_source_returns_none(self, seeder):
+        assert seeder._handle_source({"name": "a"}) is None
+
+    def test_existing_source_returns_id(self, seeder, mock_db):
+        inserted = mock_db["sources"].insert_one(
+            {"user": "system", "remote_data": "http://example.com"}
+        )
+        config = {
+            "name": "a",
+            "source": {"url": "http://example.com", "name": "src"},
+        }
+        result = seeder._handle_source(config)
+        assert result == inserted.inserted_id
+
+    @patch("application.seed.seeder.ingest_remote")
+    def test_new_source_ingestion(self, mock_ingest, seeder):
+        mock_task = MagicMock()
+        mock_task.get.return_value = {"id": "new_source_id"}
+        mock_task.successful.return_value = True
+        mock_ingest.delay.return_value = mock_task
+
+        config = {
+            "name": "a",
+            "source": {"url": "http://new.com", "name": "new_src", "loader": "web"},
+        }
+        result = seeder._handle_source(config)
+        assert result == "new_source_id"
+        mock_ingest.delay.assert_called_once_with(
+            source_data="http://new.com",
+            job_name="new_src",
+            user="system",
+            loader="web",
+        )
+
+    @patch("application.seed.seeder.ingest_remote")
+    def test_source_ingestion_failure_returns_false(self, mock_ingest, seeder):
+        mock_task = MagicMock()
+        mock_task.get.side_effect = RuntimeError("timeout")
+        mock_ingest.delay.return_value = mock_task
+
+        config = {
+            "name": "a",
+            "source": {"url": "http://fail.com", "name": "fail_src"},
+        }
+        result = seeder._handle_source(config)
+        assert result is False
+
+    @patch("application.seed.seeder.ingest_remote")
+    def test_source_missing_id_returns_false(self, mock_ingest, seeder):
+        mock_task = MagicMock()
+        mock_task.get.return_value = {"no_id_key": True}
+        mock_task.successful.return_value = True
+        mock_ingest.delay.return_value = mock_task
+
+        config = {
+            "name": "a",
+            "source": {"url": "http://bad.com", "name": "bad_src"},
+        }
+        result = seeder._handle_source(config)
+        assert result is False
+
+    @patch("application.seed.seeder.ingest_remote")
+    def test_default_loader(self, mock_ingest, seeder):
+        mock_task = MagicMock()
+        mock_task.get.return_value = {"id": "sid"}
+        mock_task.successful.return_value = True
+        mock_ingest.delay.return_value = mock_task
+
+        config = {
+            "name": "a",
+            "source": {"url": "http://x.com", "name": "s"},
+        }
+        seeder._handle_source(config)
+        call_kwargs = mock_ingest.delay.call_args[1]
+        assert call_kwargs["loader"] == "url"
+
+
+# ── seed_initial_data ──────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestSeedInitialData:
+    def test_already_seeded_skips(self, seeder, mock_db):
+        mock_db["agents"].insert_one({"user": "system", "name": "existing"})
+        with patch.object(seeder, "_seed_from_config") as mock_seed:
+            seeder.seed_initial_data()
+            mock_seed.assert_not_called()
+
+    def test_force_reseeds(self, seeder, mock_db):
+        mock_db["agents"].insert_one({"user": "system", "name": "existing"})
+        yaml_content = "agents: []"
+        with patch("builtins.open", mock_open(read_data=yaml_content)):
+            with patch.object(seeder, "_seed_from_config") as mock_seed:
+                seeder.seed_initial_data(force=True)
+                mock_seed.assert_called_once()
+
+    def test_config_file_not_found_raises(self, seeder):
+        with pytest.raises(Exception):
+            seeder.seed_initial_data(config_path="/nonexistent/path.yaml")
+
+    def test_custom_config_path(self, seeder):
+        yaml_content = "agents:\n  - name: test_agent"
+        with patch("builtins.open", mock_open(read_data=yaml_content)):
+            with patch.object(seeder, "_seed_from_config") as mock_seed:
+                seeder.seed_initial_data(config_path="/custom/path.yaml")
+                mock_seed.assert_called_once()
+
+
+# ── _seed_from_config ──────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestSeedFromConfig:
+    def test_no_agents_in_config(self, seeder):
+        seeder._seed_from_config({})
+        assert seeder.agents_collection.count_documents({}) == 0
+
+    def test_empty_agents_list(self, seeder):
+        seeder._seed_from_config({"agents": []})
+        assert seeder.agents_collection.count_documents({}) == 0
+
+    @patch.object(DatabaseSeeder, "_handle_source", return_value=None)
+    @patch.object(DatabaseSeeder, "_handle_tools", return_value=[])
+    @patch.object(DatabaseSeeder, "_handle_prompt", return_value=None)
+    def test_creates_agent(self, mock_prompt, mock_tools, mock_source, seeder, mock_db):
+        config = {
+            "agents": [
+                {
+                    "name": "TestAgent",
+                    "description": "A test agent",
+                    "agent_type": "classic",
+                }
+            ]
+        }
+        seeder._seed_from_config(config)
+        agent = mock_db["agents"].find_one({"name": "TestAgent"})
+        assert agent is not None
+        assert agent["user"] == "system"
+        assert agent["agent_type"] == "classic"
+        assert agent["status"] == "template"
+
+    @patch.object(DatabaseSeeder, "_handle_source", return_value=None)
+    @patch.object(DatabaseSeeder, "_handle_tools", return_value=[])
+    @patch.object(DatabaseSeeder, "_handle_prompt", return_value=None)
+    def test_updates_existing_agent(self, mock_prompt, mock_tools, mock_source, seeder, mock_db):
+        mock_db["agents"].insert_one(
+            {"user": "system", "name": "TestAgent", "description": "old"}
+        )
+        config = {
+            "agents": [
+                {
+                    "name": "TestAgent",
+                    "description": "updated",
+                    "agent_type": "classic",
+                }
+            ]
+        }
+        seeder._seed_from_config(config)
+        assert mock_db["agents"].count_documents({"name": "TestAgent"}) == 1
+        agent = mock_db["agents"].find_one({"name": "TestAgent"})
+        assert agent["description"] == "updated"
+
+    @patch.object(DatabaseSeeder, "_handle_source", return_value=False)
+    def test_source_failure_skips_agent(self, mock_source, seeder, mock_db):
+        config = {
+            "agents": [
+                {
+                    "name": "SkippedAgent",
+                    "description": "skip",
+                    "agent_type": "classic",
+                }
+            ]
+        }
+        seeder._seed_from_config(config)
+        assert mock_db["agents"].count_documents({"name": "SkippedAgent"}) == 0
+
+    @patch.object(DatabaseSeeder, "_handle_source", side_effect=KeyError("name"))
+    def test_agent_exception_continues(self, mock_source, seeder, mock_db):
+        config = {
+            "agents": [
+                {"name": "Bad", "description": "x", "agent_type": "y"},
+                {"name": "Good", "description": "x", "agent_type": "y"},
+            ]
+        }
+        with patch.object(seeder, "_handle_tools", return_value=[]):
+            with patch.object(seeder, "_handle_prompt", return_value=None):
+                seeder._seed_from_config(config)
+        # Both agents should be attempted; first errors, second might too
+        # Main assertion: no unhandled exception
+
+    @patch.object(DatabaseSeeder, "_handle_source", return_value=None)
+    @patch.object(DatabaseSeeder, "_handle_tools")
+    @patch.object(DatabaseSeeder, "_handle_prompt", return_value="prompt_id_123")
+    def test_agent_with_source_and_tools(self, mock_prompt, mock_tools, mock_source, seeder, mock_db):
+        tool_id = ObjectId()
+        mock_tools.return_value = [tool_id]
+
+        source_id = ObjectId()
+        mock_source.return_value = source_id
+
+        config = {
+            "agents": [
+                {
+                    "name": "FullAgent",
+                    "description": "full",
+                    "agent_type": "classic",
+                    "chunks": "5",
+                    "retriever": "classic",
+                    "image": "img.png",
+                }
+            ]
+        }
+        seeder._seed_from_config(config)
+        agent = mock_db["agents"].find_one({"name": "FullAgent"})
+        assert agent is not None
+        assert agent["prompt_id"] == "prompt_id_123"
+        assert str(tool_id) in agent["tools"]
+        assert agent["chunks"] == "5"
+        assert agent["image"] == "img.png"
+
+
+# ── initialize_from_env ────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestInitializeFromEnv:
+    @patch("application.seed.seeder.MongoClient")
+    def test_creates_seeder_from_env(self, mock_client_cls, monkeypatch):
+        monkeypatch.setenv("MONGO_URI", "mongodb://test:27017")
+        monkeypatch.setenv("MONGO_DB_NAME", "testdb")
+
+        mock_db = mongomock.MongoClient()["testdb"]
+        mock_client = MagicMock()
+        mock_client.__getitem__ = MagicMock(return_value=mock_db)
+        mock_client_cls.return_value = mock_client
+
+        seeder = DatabaseSeeder.initialize_from_env()
+        mock_client_cls.assert_called_once_with("mongodb://test:27017")
+        assert isinstance(seeder, DatabaseSeeder)
+
+    @patch("application.seed.seeder.MongoClient")
+    def test_default_env_values(self, mock_client_cls, monkeypatch):
+        monkeypatch.delenv("MONGO_URI", raising=False)
+        monkeypatch.delenv("MONGO_DB_NAME", raising=False)
+
+        mock_db = mongomock.MongoClient()["docsgpt"]
+        mock_client = MagicMock()
+        mock_client.__getitem__ = MagicMock(return_value=mock_db)
+        mock_client_cls.return_value = mock_client
+
+        DatabaseSeeder.initialize_from_env()
+        mock_client_cls.assert_called_once_with("mongodb://localhost:27017")
+
+
+# ── seed CLI commands ──────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestSeedCommands:
+    @patch("application.seed.commands.DatabaseSeeder")
+    @patch("application.seed.commands.MongoDB")
+    @patch("application.seed.commands.settings")
+    def test_init_command(self, mock_settings, mock_mongodb, mock_seeder_cls):
+        from click.testing import CliRunner
+        from application.seed.commands import seed
+
+        mock_settings.MONGO_DB_NAME = "testdb"
+        mock_client = MagicMock()
+        mock_client.__getitem__ = MagicMock(return_value="mock_db")
+        mock_mongodb.get_client.return_value = mock_client
+
+        mock_seeder = MagicMock()
+        mock_seeder_cls.return_value = mock_seeder
+
+        runner = CliRunner()
+        result = runner.invoke(seed, ["init"])
+        assert result.exit_code == 0
+        mock_seeder.seed_initial_data.assert_called_once_with(force=False)
+
+    @patch("application.seed.commands.DatabaseSeeder")
+    @patch("application.seed.commands.MongoDB")
+    @patch("application.seed.commands.settings")
+    def test_init_command_with_force(self, mock_settings, mock_mongodb, mock_seeder_cls):
+        from click.testing import CliRunner
+        from application.seed.commands import seed
+
+        mock_settings.MONGO_DB_NAME = "testdb"
+        mock_client = MagicMock()
+        mock_client.__getitem__ = MagicMock(return_value="mock_db")
+        mock_mongodb.get_client.return_value = mock_client
+
+        mock_seeder = MagicMock()
+        mock_seeder_cls.return_value = mock_seeder
+
+        runner = CliRunner()
+        result = runner.invoke(seed, ["init", "--force"])
+        assert result.exit_code == 0
+        mock_seeder.seed_initial_data.assert_called_once_with(force=True)

tests/test_template_engine.py

@@ -0,0 +1,186 @@
+from unittest.mock import patch
+
+import pytest
+
+from application.templates.template_engine import TemplateEngine, TemplateRenderError
+
+
+@pytest.fixture
+def engine():
+    return TemplateEngine()
+
+
+# ── render ─────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestRender:
+    def test_simple_variable(self, engine):
+        result = engine.render("Hello {{ name }}", {"name": "World"})
+        assert result == "Hello World"
+
+    def test_empty_template_returns_empty(self, engine):
+        assert engine.render("", {"x": 1}) == ""
+
+    def test_none_template_returns_empty(self, engine):
+        assert engine.render(None, {"x": 1}) == ""
+
+    def test_no_variables(self, engine):
+        assert engine.render("plain text", {}) == "plain text"
+
+    def test_multiple_variables(self, engine):
+        tpl = "{{ a }} and {{ b }}"
+        assert engine.render(tpl, {"a": "X", "b": "Y"}) == "X and Y"
+
+    def test_nested_dict_access(self, engine):
+        tpl = "{{ data.key }}"
+        assert engine.render(tpl, {"data": {"key": "value"}}) == "value"
+
+    def test_loop(self, engine):
+        tpl = "{% for i in items %}{{ i }} {% endfor %}"
+        result = engine.render(tpl, {"items": ["a", "b", "c"]})
+        assert result.strip() == "a b c"
+
+    def test_conditional(self, engine):
+        tpl = "{% if show %}yes{% else %}no{% endif %}"
+        assert engine.render(tpl, {"show": True}) == "yes"
+        assert engine.render(tpl, {"show": False}) == "no"
+
+    def test_syntax_error_raises_template_render_error(self, engine):
+        with pytest.raises(TemplateRenderError, match="syntax error"):
+            engine.render("{% if %}", {})
+
+    def test_undefined_variable_chainable(self, engine):
+        # ChainableUndefined should NOT raise; it silently returns empty
+        result = engine.render("{{ missing }}", {})
+        assert result == ""
+
+    def test_autoescape_html(self, engine):
+        result = engine.render("{{ content }}", {"content": "<script>alert(1)</script>"})
+        assert "<script>" not in result
+        assert "&lt;script&gt;" in result
+
+    def test_trim_blocks(self, engine):
+        tpl = "{% if True %}\nyes\n{% endif %}"
+        result = engine.render(tpl, {})
+        assert result.strip() == "yes"
+
+    def test_general_exception_raises_template_render_error(self, engine):
+        with patch.object(engine._env, "from_string", side_effect=RuntimeError("boom")):
+            with pytest.raises(TemplateRenderError, match="rendering failed"):
+                engine.render("{{ x }}", {"x": 1})
+
+
+# ── validate_template ──────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestValidateTemplate:
+    def test_valid_template(self, engine):
+        assert engine.validate_template("{{ name }}") is True
+
+    def test_empty_template_valid(self, engine):
+        assert engine.validate_template("") is True
+
+    def test_none_template_valid(self, engine):
+        assert engine.validate_template(None) is True
+
+    def test_invalid_syntax(self, engine):
+        assert engine.validate_template("{% if %}") is False
+
+    def test_plain_text_valid(self, engine):
+        assert engine.validate_template("just plain text") is True
+
+    def test_complex_valid_template(self, engine):
+        tpl = "{% for x in items %}{{ x.name }}{% endfor %}"
+        assert engine.validate_template(tpl) is True
+
+    def test_general_exception_returns_false(self, engine):
+        with patch.object(engine._env, "from_string", side_effect=RuntimeError("boom")):
+            assert engine.validate_template("{{ x }}") is False
+
+
+# ── extract_variables ──────────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestExtractVariables:
+    def test_empty_template(self, engine):
+        assert engine.extract_variables("") == set()
+
+    def test_none_template(self, engine):
+        assert engine.extract_variables(None) == set()
+
+    def test_syntax_error_returns_empty(self, engine):
+        assert engine.extract_variables("{% if %}") == set()
+
+    def test_general_exception_returns_empty(self, engine):
+        with patch.object(engine._env, "parse", side_effect=RuntimeError("boom")):
+            assert engine.extract_variables("{{ x }}") == set()
+
+
+# ── extract_tool_usages ───────────────────────────────────────────────────────
+
+
+@pytest.mark.unit
+class TestExtractToolUsages:
+    def test_empty_template(self, engine):
+        assert engine.extract_tool_usages("") == {}
+
+    def test_none_template(self, engine):
+        assert engine.extract_tool_usages(None) == {}
+
+    def test_syntax_error_returns_empty(self, engine):
+        assert engine.extract_tool_usages("{% if %}") == {}
+
+    def test_getattr_single_tool(self, engine):
+        tpl = "{{ tools.memory.notes }}"
+        result = engine.extract_tool_usages(tpl)
+        assert "memory" in result
+        assert "notes" in result["memory"]
+
+    def test_getattr_tool_without_action(self, engine):
+        tpl = "{{ tools.search }}"
+        result = engine.extract_tool_usages(tpl)
+        assert "search" in result
+        assert None in result["search"]
+
+    def test_getitem_bracket_notation(self, engine):
+        tpl = '{{ tools["calendar"]["events"] }}'
+        result = engine.extract_tool_usages(tpl)
+        assert "calendar" in result
+        assert "events" in result["calendar"]
+
+    def test_multiple_tools(self, engine):
+        tpl = "{{ tools.memory.notes }} {{ tools.search.query }}"
+        result = engine.extract_tool_usages(tpl)
+        assert "memory" in result
+        assert "search" in result
+
+    def test_same_tool_multiple_actions(self, engine):
+        tpl = "{{ tools.memory.notes }} {{ tools.memory.tasks }}"
+        result = engine.extract_tool_usages(tpl)
+        assert "memory" in result
+        assert "notes" in result["memory"]
+        assert "tasks" in result["memory"]
+
+    def test_non_tools_getattr_ignored(self, engine):
+        tpl = "{{ data.something }}"
+        result = engine.extract_tool_usages(tpl)
+        assert result == {}
+
+    def test_general_parse_error_returns_empty(self, engine):
+        with patch.object(engine._env, "parse", side_effect=RuntimeError("boom")):
+            assert engine.extract_tool_usages("{{ tools.x }}") == {}
+
+    def test_tools_in_loop(self, engine):
+        tpl = "{% for item in tools.memory.notes %}{{ item }}{% endfor %}"
+        result = engine.extract_tool_usages(tpl)
+        assert "memory" in result
+        assert "notes" in result["memory"]
+
+    def test_tools_in_conditional(self, engine):
+        tpl = "{% if tools.search.results %}found{% endif %}"
+        result = engine.extract_tool_usages(tpl)
+        assert "search" in result
+        assert "results" in result["search"]