Pg 4 (#2390)

* feat: postgres tests

* feat: mongo cutoff

* feat: mongo cutoff

* feat: adjust docs and compose files

* fix: mini code mongo removals

* fix: tests and k8s mongo stuff

* feat: test fixes

* fix: ruff

* fix: vale

* Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information'

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* fix: mini suggestions

* vale lint fix 2

* fix: codeql columns thing

* fix: test mongo

* fix: tests coverage

* feat: better tests 4

* feat: more tests

* feat: decent coverage

* fix: ruff fixes

* fix: remove mongo mock

* feat: enhance workflow engine and API routes; add document retrieval and source handling

* feat: e2e tests

* fix: mcp, mongo and more

* fix: mini codeql warning

* fix: agent chunk view

* fix: mini issues

* fix: more pg fixes

* feat: postgres prep on start

* feat: qa tests

* fix: mini improvements

* fix: tests

---------

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
Co-authored-by: Siddhant Rai <siddhant.rai.5686@gmail.com>

tests/api/user/attachments/test_routes.py

@@ -1,4 +1,5 @@
 import io
+from contextlib import contextmanager
 from types import SimpleNamespace
 from unittest.mock import MagicMock, patch
 
@@ -6,6 +7,39 @@ import pytest
 from flask import Flask, request
 
 
+class _FakeAgentsRepo:
+    """Post-PG migration replacement for the old Mongo `agents_collection`
+    mock. Tests set `_FakeAgentsRepo._row` to control what `find_by_key`
+    returns."""
+
+    _row = None
+
+    def __init__(self, *a, **kw):
+        pass
+
+    def find_by_key(self, key):
+        return self._row
+
+
+@contextmanager
+def _fake_readonly():
+    yield None
+
+
+def _patch_agents_repo(row):
+    _FakeAgentsRepo._row = row
+    return (
+        patch(
+            "application.api.user.attachments.routes.AgentsRepository",
+            _FakeAgentsRepo,
+        ),
+        patch(
+            "application.api.user.attachments.routes.db_readonly",
+            _fake_readonly,
+        ),
+    )
+
+
 def _get_response_status(response):
     if isinstance(response, tuple):
         return response[1]
@@ -36,7 +70,7 @@ class FakeRedis:
 class TestStoreAttachmentEndpoint:
     @patch("application.api.user.tasks.store_attachment.delay")
     def test_store_attachment_preserves_upload_indexes_for_partial_failures(
-        self, mock_store_attachment, flask_app, mock_mongo_db
+        self, mock_store_attachment, flask_app
     ):
         from application.api.user.attachments.routes import StoreAttachment
 
@@ -82,7 +116,7 @@ class TestStoreAttachmentEndpoint:
     @patch("application.api.user.tasks.store_attachment.delay")
     @patch("application.stt.upload_limits.settings")
     def test_store_attachment_rejects_oversized_audio_files(
-        self, mock_limit_settings, mock_store_attachment, flask_app, mock_mongo_db
+        self, mock_limit_settings, mock_store_attachment, flask_app
     ):
         from application.api.user.attachments.routes import StoreAttachment
 
@@ -111,7 +145,7 @@ class TestStoreAttachmentEndpoint:
 
 
 class TestSpeechToTextEndpoint:
-    def test_stt_returns_400_when_file_is_missing(self, flask_app, mock_mongo_db):
+    def test_stt_returns_400_when_file_is_missing(self, flask_app):
         from application.api.user.attachments.routes import SpeechToText
 
         app = Flask(__name__)
@@ -126,7 +160,7 @@ class TestSpeechToTextEndpoint:
             assert _get_response_json(response)["message"] == "Missing file"
 
     def test_stt_returns_401_when_authentication_is_missing(
-        self, flask_app, mock_mongo_db
+        self, flask_app
     ):
         from application.api.user.attachments.routes import SpeechToText
 
@@ -148,7 +182,7 @@ class TestSpeechToTextEndpoint:
 
     @patch("application.api.user.attachments.routes.STTCreator.create_stt")
     def test_stt_transcribes_audio_for_authenticated_user(
-        self, mock_create_stt, flask_app, mock_mongo_db
+        self, mock_create_stt, flask_app
     ):
         from application.api.user.attachments.routes import SpeechToText
 
@@ -189,7 +223,7 @@ class TestSpeechToTextEndpoint:
             mock_create_stt.assert_called_once()
             mock_stt.transcribe.assert_called_once()
 
-    def test_stt_rejects_unsupported_extension(self, flask_app, mock_mongo_db):
+    def test_stt_rejects_unsupported_extension(self, flask_app):
         from application.api.user.attachments.routes import SpeechToText
 
         app = Flask(__name__)
@@ -212,7 +246,7 @@ class TestSpeechToTextEndpoint:
 class TestLiveSpeechToTextEndpoint:
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_start_creates_session(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import LiveSpeechToTextStart
 
@@ -239,7 +273,7 @@ class TestLiveSpeechToTextEndpoint:
     @patch("application.api.user.attachments.routes.STTCreator.create_stt")
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_reconciles_transcript_progressively(
-        self, mock_get_redis, mock_create_stt, flask_app, mock_mongo_db
+        self, mock_get_redis, mock_create_stt, flask_app
     ):
         from application.api.user.attachments.routes import (
             LiveSpeechToTextChunk,
@@ -358,7 +392,7 @@ class TestLiveSpeechToTextEndpoint:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_rejects_missing_session(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import LiveSpeechToTextChunk
 
@@ -386,7 +420,7 @@ class TestLiveSpeechToTextEndpoint:
     @patch("application.api.user.attachments.routes.STTCreator.create_stt")
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_hides_internal_value_errors(
-        self, mock_get_redis, mock_create_stt, flask_app, mock_mongo_db
+        self, mock_get_redis, mock_create_stt, flask_app
     ):
         from application.api.user.attachments.routes import (
             LiveSpeechToTextChunk,
@@ -452,14 +486,13 @@ class TestResolveAuthenticatedUser:
             assert result is not None
             assert "jwt_user" in result
 
-    def test_returns_user_from_valid_api_key_form(self, flask_app, mock_mongo_db):
+    def test_returns_user_from_valid_api_key_form(self, flask_app):
         from application.api.user.attachments.routes import _resolve_authenticated_user
 
         app = Flask(__name__)
-        mock_agents = MagicMock()
-        mock_agents.find_one.return_value = {"key": "valid_key", "user": "apikey_user"}
+        p1, p2 = _patch_agents_repo({"key": "valid_key", "user_id": "apikey_user"})
 
-        with patch("application.api.user.base.agents_collection", mock_agents):
+        with p1, p2:
             with app.test_request_context(
                 "/api/store_attachment",
                 method="POST",
@@ -470,14 +503,13 @@ class TestResolveAuthenticatedUser:
                 assert result is not None
                 assert "apikey_user" in result
 
-    def test_returns_401_for_invalid_api_key(self, flask_app, mock_mongo_db):
+    def test_returns_401_for_invalid_api_key(self, flask_app):
         from application.api.user.attachments.routes import _resolve_authenticated_user
 
         app = Flask(__name__)
-        mock_agents = MagicMock()
-        mock_agents.find_one.return_value = None
+        p1, p2 = _patch_agents_repo(None)
 
-        with patch("application.api.user.base.agents_collection", mock_agents):
+        with p1, p2:
             with app.test_request_context(
                 "/api/store_attachment",
                 method="POST",
@@ -581,17 +613,14 @@ class TestStoreAttachmentAdditional:
     """Additional tests for StoreAttachment endpoint."""
 
     def test_store_attachment_returns_401_for_invalid_api_key(
-        self, flask_app, mock_mongo_db
+        self, flask_app
     ):
         from application.api.user.attachments.routes import StoreAttachment
 
         app = Flask(__name__)
-        mock_agents = MagicMock()
-        mock_agents.find_one.return_value = None
+        p1, p2 = _patch_agents_repo(None)
 
-        with patch(
-            "application.api.user.base.agents_collection", mock_agents
-        ):
+        with p1, p2:
             with app.test_request_context(
                 "/api/store_attachment",
                 method="POST",
@@ -607,7 +636,7 @@ class TestStoreAttachmentAdditional:
                 response = resource.post()
                 assert _get_response_status(response) == 401
 
-    def test_store_attachment_missing_file(self, flask_app, mock_mongo_db):
+    def test_store_attachment_missing_file(self, flask_app):
         from application.api.user.attachments.routes import StoreAttachment
 
         app = Flask(__name__)
@@ -623,7 +652,7 @@ class TestStoreAttachmentAdditional:
             response = resource.post()
             assert _get_response_status(response) == 400
 
-    def test_store_attachment_no_auth_returns_401(self, flask_app, mock_mongo_db):
+    def test_store_attachment_no_auth_returns_401(self, flask_app):
         from application.api.user.attachments.routes import StoreAttachment
 
         app = Flask(__name__)
@@ -641,7 +670,7 @@ class TestStoreAttachmentAdditional:
 
     @patch("application.api.user.tasks.store_attachment.delay")
     def test_store_attachment_single_file_response(
-        self, mock_store_attachment, flask_app, mock_mongo_db
+        self, mock_store_attachment, flask_app
     ):
         from application.api.user.attachments.routes import StoreAttachment
 
@@ -668,7 +697,7 @@ class TestStoreAttachmentAdditional:
 
     @patch("application.api.user.tasks.store_attachment.delay")
     def test_store_attachment_all_files_fail_returns_400(
-        self, mock_store_attachment, flask_app, mock_mongo_db
+        self, mock_store_attachment, flask_app
     ):
         from application.api.user.attachments.routes import StoreAttachment
 
@@ -691,7 +720,7 @@ class TestStoreAttachmentAdditional:
 
     @patch("application.api.user.tasks.store_attachment.delay")
     def test_store_attachment_outer_exception(
-        self, mock_store_attachment, flask_app, mock_mongo_db
+        self, mock_store_attachment, flask_app
     ):
         from application.api.user.attachments.routes import StoreAttachment
 
@@ -713,7 +742,7 @@ class TestStoreAttachmentAdditional:
                 response = resource.post()
                 assert _get_response_status(response) == 400
 
-    def test_store_attachment_empty_filename_files(self, flask_app, mock_mongo_db):
+    def test_store_attachment_empty_filename_files(self, flask_app):
         from application.api.user.attachments.routes import StoreAttachment
 
         app = Flask(__name__)
@@ -731,7 +760,7 @@ class TestStoreAttachmentAdditional:
 
     @patch("application.api.user.tasks.store_attachment.delay")
     def test_store_attachment_via_api_key_auth(
-        self, mock_store_attachment, flask_app, mock_mongo_db
+        self, mock_store_attachment, flask_app
     ):
         from application.api.user.attachments.routes import StoreAttachment
 
@@ -739,15 +768,11 @@ class TestStoreAttachmentAdditional:
         mock_storage = MagicMock()
         mock_storage.save_file.return_value = {"storage_type": "local"}
         mock_store_attachment.return_value = SimpleNamespace(id="task-api")
-        mock_agents = MagicMock()
-        mock_agents.find_one.return_value = {
-            "key": "valid_key",
-            "user": "apikey_user",
-        }
+        p1, p2 = _patch_agents_repo(
+            {"key": "valid_key", "user_id": "apikey_user"}
+        )
 
-        with patch("application.api.user.base.storage", mock_storage), patch(
-            "application.api.user.base.agents_collection", mock_agents
-        ):
+        with patch("application.api.user.base.storage", mock_storage), p1, p2:
             with app.test_request_context(
                 "/api/store_attachment",
                 method="POST",
@@ -776,7 +801,7 @@ class TestSpeechToTextAdditional:
         return_value=False,
     )
     def test_stt_rejects_unsupported_mimetype(
-        self, mock_mimetype_check, flask_app, mock_mongo_db
+        self, mock_mimetype_check, flask_app
     ):
         from application.api.user.attachments.routes import SpeechToText
 
@@ -796,7 +821,7 @@ class TestSpeechToTextAdditional:
 
     @patch("application.stt.upload_limits.settings")
     def test_stt_rejects_oversized_audio(
-        self, mock_limit_settings, flask_app, mock_mongo_db
+        self, mock_limit_settings, flask_app
     ):
         from application.api.user.attachments.routes import SpeechToText
 
@@ -820,7 +845,7 @@ class TestSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.STTCreator.create_stt")
     def test_stt_transcription_error_returns_400(
-        self, mock_create_stt, flask_app, mock_mongo_db
+        self, mock_create_stt, flask_app
     ):
         from application.api.user.attachments.routes import SpeechToText
 
@@ -847,7 +872,7 @@ class TestSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.STTCreator.create_stt")
     def test_stt_uses_language_form_param(
-        self, mock_create_stt, flask_app, mock_mongo_db
+        self, mock_create_stt, flask_app
     ):
         from application.api.user.attachments.routes import SpeechToText
 
@@ -883,7 +908,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_start_returns_401_no_auth(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import LiveSpeechToTextStart
 
@@ -903,7 +928,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_start_returns_503_when_redis_unavailable(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import LiveSpeechToTextStart
 
@@ -923,7 +948,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_returns_401_no_auth(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import LiveSpeechToTextChunk
 
@@ -948,7 +973,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_returns_503_no_redis(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import LiveSpeechToTextChunk
 
@@ -973,7 +998,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_missing_session_id(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import LiveSpeechToTextChunk
 
@@ -999,7 +1024,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_forbidden_different_user(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import (
             LiveSpeechToTextChunk,
@@ -1038,7 +1063,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_missing_chunk_index(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import (
             LiveSpeechToTextChunk,
@@ -1078,7 +1103,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_invalid_chunk_index(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import (
             LiveSpeechToTextChunk,
@@ -1118,7 +1143,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_missing_file(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import (
             LiveSpeechToTextChunk,
@@ -1157,7 +1182,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_unsupported_extension(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import (
             LiveSpeechToTextChunk,
@@ -1198,7 +1223,7 @@ class TestLiveSpeechToTextAdditional:
     @patch("application.api.user.attachments.routes.STTCreator.create_stt")
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_transcription_error(
-        self, mock_get_redis, mock_create_stt, flask_app, mock_mongo_db
+        self, mock_get_redis, mock_create_stt, flask_app
     ):
         from application.api.user.attachments.routes import (
             LiveSpeechToTextChunk,
@@ -1247,7 +1272,7 @@ class TestLiveSpeechToTextAdditional:
     @patch("application.api.user.attachments.routes.STTCreator.create_stt")
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_chunk_detects_language(
-        self, mock_get_redis, mock_create_stt, mock_settings, flask_app, mock_mongo_db
+        self, mock_get_redis, mock_create_stt, mock_settings, flask_app
     ):
         from application.api.user.attachments.routes import (
             LiveSpeechToTextChunk,
@@ -1301,7 +1326,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_finish_returns_401_no_auth(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import LiveSpeechToTextFinish
 
@@ -1321,7 +1346,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_finish_returns_503_no_redis(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import LiveSpeechToTextFinish
 
@@ -1341,7 +1366,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_finish_missing_session_id(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import LiveSpeechToTextFinish
 
@@ -1362,7 +1387,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_finish_session_not_found(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import LiveSpeechToTextFinish
 
@@ -1382,7 +1407,7 @@ class TestLiveSpeechToTextAdditional:
 
     @patch("application.api.user.attachments.routes.get_redis_instance")
     def test_live_stt_finish_forbidden_different_user(
-        self, mock_get_redis, flask_app, mock_mongo_db
+        self, mock_get_redis, flask_app
     ):
         from application.api.user.attachments.routes import (
             LiveSpeechToTextFinish,
@@ -1419,7 +1444,7 @@ class TestLiveSpeechToTextAdditional:
 class TestServeImage:
     """Tests for ServeImage endpoint."""
 
-    def test_serve_image_success(self, flask_app, mock_mongo_db):
+    def test_serve_image_success(self, flask_app):
         from application.api.user.attachments.routes import ServeImage
 
         app = Flask(__name__)
@@ -1437,7 +1462,7 @@ class TestServeImage:
                 assert _get_response_status(response) == 200
                 assert response.headers.get("Content-Type") == "image/png"
 
-    def test_serve_image_jpg_content_type(self, flask_app, mock_mongo_db):
+    def test_serve_image_jpg_content_type(self, flask_app):
         from application.api.user.attachments.routes import ServeImage
 
         app = Flask(__name__)
@@ -1455,7 +1480,7 @@ class TestServeImage:
                 assert _get_response_status(response) == 200
                 assert response.headers.get("Content-Type") == "image/jpeg"
 
-    def test_serve_image_not_found(self, flask_app, mock_mongo_db):
+    def test_serve_image_not_found(self, flask_app):
         from application.api.user.attachments.routes import ServeImage
 
         app = Flask(__name__)
@@ -1471,7 +1496,7 @@ class TestServeImage:
                 response = resource.get("missing/image.png")
                 assert _get_response_status(response) == 404
 
-    def test_serve_image_generic_error(self, flask_app, mock_mongo_db):
+    def test_serve_image_generic_error(self, flask_app):
         from application.api.user.attachments.routes import ServeImage
 
         app = Flask(__name__)
@@ -1493,7 +1518,7 @@ class TestTextToSpeech:
     """Tests for TextToSpeech endpoint."""
 
     @patch("application.api.user.attachments.routes.TTSCreator.create_tts")
-    def test_tts_success(self, mock_create_tts, flask_app, mock_mongo_db):
+    def test_tts_success(self, mock_create_tts, flask_app):
         from application.api.user.attachments.routes import TextToSpeech
 
         app = Flask(__name__)
@@ -1515,7 +1540,7 @@ class TestTextToSpeech:
             assert payload["lang"] == "en"
 
     @patch("application.api.user.attachments.routes.TTSCreator.create_tts")
-    def test_tts_error_returns_400(self, mock_create_tts, flask_app, mock_mongo_db):
+    def test_tts_error_returns_400(self, mock_create_tts, flask_app):
         from application.api.user.attachments.routes import TextToSpeech
 
         app = Flask(__name__)