diff --git a/application/agents/tools/mcp_tool.py b/application/agents/tools/mcp_tool.py index 1ca68ead..275fe581 100644 --- a/application/agents/tools/mcp_tool.py +++ b/application/agents/tools/mcp_tool.py @@ -20,10 +20,11 @@ from pydantic import AnyHttpUrl, ValidationError from redis import Redis from application.agents.tools.base import Tool -from application.api.user.tasks import mcp_oauth_status_task, mcp_oauth_task +from application.api.user.tasks import mcp_oauth_task from application.cache import get_redis_instance from application.core.settings import settings from application.core.url_validation import SSRFError, validate_url +from application.events.keys import stream_key from application.security.encryption import decrypt_credentials logger = logging.getLogger(__name__) @@ -76,6 +77,12 @@ class MCPTool(Tool): self.oauth_task_id = config.get("oauth_task_id", None) self.oauth_client_name = config.get("oauth_client_name", "DocsGPT-MCP") self.redirect_uri = self._resolve_redirect_uri(config.get("redirect_uri")) + # Pulled out of ``config`` (rather than left in ``self.config``) + # because it is a callable supplied by the OAuth worker — not + # something the rest of the tool plumbing should marshal or + # serialize. ``DocsGPTOAuth`` invokes it from ``redirect_handler`` + # so the SSE envelope can carry ``authorization_url``. + self.oauth_redirect_publish = config.pop("oauth_redirect_publish", None) self.available_tools = [] self._cache_key = self._generate_cache_key() @@ -167,6 +174,7 @@ class MCPTool(Tool): redirect_uri=self.redirect_uri, task_id=self.oauth_task_id, user_id=self.user_id, + redirect_publish=self.oauth_redirect_publish, ) elif self.auth_type == "bearer": token = self.auth_credentials.get( @@ -679,12 +687,17 @@ class DocsGPTOAuth(OAuthClientProvider): user_id=None, additional_client_metadata: dict[str, Any] | None = None, skip_redirect_validation: bool = False, + redirect_publish=None, ): self.redirect_uri = redirect_uri self.redis_client = redis_client self.redis_prefix = redis_prefix self.task_id = task_id self.user_id = user_id + # Worker-supplied callback. Invoked from ``redirect_handler`` + # once the authorization URL is known so the SSE envelope can + # carry it. ``None`` for any non-worker entrypoint. + self.redirect_publish = redirect_publish parsed_url = urlparse(mcp_url) self.server_base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" @@ -744,17 +757,19 @@ class DocsGPTOAuth(OAuthClientProvider): self.redis_client.setex(key, 600, auth_url) logger.info("Stored auth_url in Redis: %s", key) - if self.task_id: - status_key = f"mcp_oauth_status:{self.task_id}" - status_data = { - "status": "requires_redirect", - "message": "Authorization required", - "authorization_url": self.auth_url, - "state": self.extracted_state, - "requires_oauth": True, - "task_id": self.task_id, - } - self.redis_client.setex(status_key, 600, json.dumps(status_data)) + if self.redirect_publish is not None: + # Best-effort: a publish failure must not abort the OAuth + # handshake — the user can still authorize via the popup + # opened from the legacy polling fallback if the SSE + # envelope is lost. + try: + self.redirect_publish(auth_url) + except Exception: + logger.warning( + "redirect_publish callback raised for task_id=%s", + self.task_id, + exc_info=True, + ) async def callback_handler(self) -> tuple[str, str | None]: """Wait for auth code from Redis using the state value.""" @@ -764,17 +779,6 @@ class DocsGPTOAuth(OAuthClientProvider): max_wait_time = 300 code_key = f"{self.redis_prefix}code:{self.extracted_state}" - if self.task_id: - status_key = f"mcp_oauth_status:{self.task_id}" - status_data = { - "status": "awaiting_callback", - "message": "Waiting for authorization...", - "authorization_url": self.auth_url, - "state": self.extracted_state, - "requires_oauth": True, - "task_id": self.task_id, - } - self.redis_client.setex(status_key, 600, json.dumps(status_data)) start_time = time.time() while time.time() - start_time < max_wait_time: code_data = self.redis_client.get(code_key) @@ -789,14 +793,6 @@ class DocsGPTOAuth(OAuthClientProvider): self.redis_client.delete( f"{self.redis_prefix}state:{self.extracted_state}" ) - - if self.task_id: - status_data = { - "status": "callback_received", - "message": "Completing authentication...", - "task_id": self.task_id, - } - self.redis_client.setex(status_key, 600, json.dumps(status_data)) return code, returned_state error_key = f"{self.redis_prefix}error:{self.extracted_state}" error_data = self.redis_client.get(error_key) @@ -1038,8 +1034,73 @@ class MCPOAuthManager: logger.error("Error handling OAuth callback: %s", e) return False - def get_oauth_status(self, task_id: str) -> Dict[str, Any]: - """Get current status of OAuth flow using provided task_id.""" + def get_oauth_status(self, task_id: str, user_id: str) -> Dict[str, Any]: + """Return the latest OAuth status for ``task_id`` from the user's SSE journal. + + Mirrors the legacy polling contract: ``status`` derived from the + ``mcp.oauth.*`` event-type suffix, with payload fields surfaced + (e.g. ``tools``/``tools_count`` on ``completed``). + """ if not task_id: return {"status": "not_started", "message": "OAuth flow not started"} - return mcp_oauth_status_task(task_id) + if not user_id: + return {"status": "not_found", "message": "User not provided"} + if self.redis_client is None: + return {"status": "not_found", "message": "Redis unavailable"} + + try: + # OAuth flows are short-lived but a concurrent source + # ingest can flood the user channel between the OAuth + # popup completing and the user clicking Save, pushing the + # completion envelope outside the read window. Bound the + # scan by the configured stream cap so we cover the full + # journal — XADD MAXLEN keeps that bounded too. + scan_count = max(settings.EVENTS_STREAM_MAXLEN, 200) + entries = self.redis_client.xrevrange( + stream_key(user_id), count=scan_count + ) + except Exception: + logger.exception( + "xrevrange failed for oauth status: user_id=%s task_id=%s", + user_id, + task_id, + ) + return {"status": "not_found", "message": "Status unavailable"} + + for _entry_id, fields in entries: + if not isinstance(fields, dict): + continue + # decode_responses=False ⇒ bytes keys; the string-key fallback + # covers a future flip of that default without a forced refactor. + event_raw = fields.get(b"event") + if event_raw is None: + event_raw = fields.get("event") + if event_raw is None: + continue + if isinstance(event_raw, bytes): + try: + event_raw = event_raw.decode("utf-8") + except Exception: + continue + try: + envelope = json.loads(event_raw) + except Exception: + continue + if not isinstance(envelope, dict): + continue + event_type = envelope.get("type", "") + if not isinstance(event_type, str) or not event_type.startswith( + "mcp.oauth." + ): + continue + scope = envelope.get("scope") or {} + if scope.get("kind") != "mcp_oauth" or scope.get("id") != task_id: + continue + payload = envelope.get("payload") or {} + return { + "status": event_type[len("mcp.oauth."):], + "task_id": task_id, + **payload, + } + + return {"status": "not_found", "message": "Status not found"} diff --git a/application/alembic/versions/0001_initial.py b/application/alembic/versions/0001_initial.py index a1e2e2b4..f69c53aa 100644 --- a/application/alembic/versions/0001_initial.py +++ b/application/alembic/versions/0001_initial.py @@ -1,4 +1,4 @@ -"""0001 initial schema — consolidated Phase-1..3 baseline. +"""0001 initial schema — consolidated baseline for user-data tables. Revision ID: 0001_initial Revises: diff --git a/application/alembic/versions/0007_message_events.py b/application/alembic/versions/0007_message_events.py new file mode 100644 index 00000000..dfea0b56 --- /dev/null +++ b/application/alembic/versions/0007_message_events.py @@ -0,0 +1,40 @@ +"""0007 message_events — durable journal of chat-stream events. + +Snapshot half of the chat-stream snapshot+tail pattern. Composite PK +``(message_id, sequence_no)``, ``created_at`` indexed for retention +sweeps, ``ON DELETE CASCADE`` from ``conversation_messages``. + +Revision ID: 0007_message_events +Revises: 0006_idempotency_lease +""" + +from typing import Sequence, Union + +from alembic import op + + +revision: str = "0007_message_events" +down_revision: Union[str, None] = "0006_idempotency_lease" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + """ + CREATE TABLE message_events ( + message_id UUID NOT NULL REFERENCES conversation_messages(id) ON DELETE CASCADE, + sequence_no INTEGER NOT NULL, + event_type TEXT NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (message_id, sequence_no) + ); + CREATE INDEX message_events_created_at_idx ON message_events(created_at); + """ + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS message_events_created_at_idx;") + op.execute("DROP TABLE IF EXISTS message_events;") diff --git a/application/api/answer/routes/base.py b/application/api/answer/routes/base.py index 6f9669b3..dfe885ff 100644 --- a/application/api/answer/routes/base.py +++ b/application/api/answer/routes/base.py @@ -23,9 +23,16 @@ from application.core.settings import settings from application.error import sanitize_api_error from application.llm.llm_creator import LLMCreator from application.storage.db.repositories.agents import AgentsRepository +from application.storage.db.repositories.conversations import MessageUpdateOutcome from application.storage.db.repositories.token_usage import TokenUsageRepository from application.storage.db.repositories.user_logs import UserLogsRepository from application.storage.db.session import db_readonly, db_session +from application.events.publisher import publish_user_event +from application.streaming.event_replay import format_sse_event +from application.streaming.message_journal import ( + BatchedJournalWriter, + record_event, +) from application.utils import check_required_fields logger = logging.getLogger(__name__) @@ -277,6 +284,17 @@ class BaseAnswerResource: "update_message_status streaming failed for %s", reserved_message_id, ) + # Seed last_heartbeat_at so watchdog doesn't fall back to `timestamp` + # (creation time) before the first STREAM_HEARTBEAT_INTERVAL tick. + try: + self.conversation_service.heartbeat_message( + reserved_message_id, + ) + except Exception: + logger.exception( + "initial heartbeat seed failed for %s", + reserved_message_id, + ) streaming_marked = True last_heartbeat_at = time.monotonic() @@ -303,13 +321,73 @@ class BaseAnswerResource: try: agent.tool_executor.message_id = reserved_message_id except Exception: - pass + logger.debug( + "Could not set tool_executor.message_id; tool-call correlation will be missing for message_id=%s", + reserved_message_id, + ) + + # Per-stream monotonic SSE event id. Allocated by ``_emit`` and + # threaded through both the wire format (``id: \\n``) and + # the journal write so a reconnecting client can ``Last-Event- + # ID`` past anything they already saw. Continuations resume + # against the original ``reserved_message_id`` — seed the + # allocator from the journal's high-water mark so we don't + # collide on the duplicate-PK and silently lose every emit + # past the resume point. + sequence_no = -1 + if _continuation and reserved_message_id: + try: + from application.storage.db.repositories.message_events import ( + MessageEventsRepository, + ) + + with db_readonly() as conn: + latest = MessageEventsRepository(conn).latest_sequence_no( + reserved_message_id + ) + if latest is not None: + sequence_no = latest + except Exception: + logger.exception( + "Continuation seq seed lookup failed for message_id=%s; " + "falling back to seq=-1 (duplicate-PK collisions will " + "be swallowed)", + reserved_message_id, + ) + + # One batched journal writer per stream. + journal_writer: Optional[BatchedJournalWriter] = ( + BatchedJournalWriter(reserved_message_id) + if reserved_message_id + else None + ) + + def _emit(payload: dict) -> str: + """Format-and-journal one SSE event. + + With a reserved ``message_id``, buffers into the journal and + emits ``id: ``-tagged SSE frames; otherwise falls back to + legacy ``data: ...\\n\\n`` framing. + """ + nonlocal sequence_no + if not reserved_message_id or journal_writer is None: + return f"data: {json.dumps(payload)}\n\n" + sequence_no += 1 + seq = sequence_no + event_type = ( + payload.get("type", "data") + if isinstance(payload, dict) + else "data" + ) + normalised = payload if isinstance(payload, dict) else {"value": payload} + journal_writer.record(seq, event_type, normalised) + return format_sse_event(normalised, seq) try: # Surface the placeholder id before any LLM tokens so a # mid-handshake disconnect still has a row to tail-poll. if reserved_message_id: - early_event = json.dumps( + yield _emit( { "type": "message_id", "message_id": reserved_message_id, @@ -319,7 +397,6 @@ class BaseAnswerResource: "request_id": request_id, } ) - yield f"data: {early_event}\n\n" if _continuation: gen_iter = agent.gen_continuation( @@ -345,8 +422,9 @@ class BaseAnswerResource: schema_info = line.get("schema") structured_chunks.append(line["answer"]) else: - data = json.dumps({"type": "answer", "answer": line["answer"]}) - yield f"data: {data}\n\n" + yield _emit( + {"type": "answer", "answer": line["answer"]} + ) elif "sources" in line: _mark_streaming_once() truncated_sources = [] @@ -359,43 +437,40 @@ class BaseAnswerResource: ) truncated_sources.append(truncated_source) if truncated_sources: - data = json.dumps( + yield _emit( {"type": "source", "source": truncated_sources} ) - yield f"data: {data}\n\n" elif "tool_calls" in line: tool_calls = line["tool_calls"] - data = json.dumps({"type": "tool_calls", "tool_calls": tool_calls}) - yield f"data: {data}\n\n" + yield _emit({"type": "tool_calls", "tool_calls": tool_calls}) elif "thought" in line: thought += line["thought"] - data = json.dumps({"type": "thought", "thought": line["thought"]}) - yield f"data: {data}\n\n" + yield _emit({"type": "thought", "thought": line["thought"]}) elif "type" in line: if line.get("type") == "tool_calls_pending": # Save continuation state and end the stream paused = True - data = json.dumps(line) - yield f"data: {data}\n\n" + yield _emit(line) elif line.get("type") == "error": - sanitized_error = { - "type": "error", - "error": sanitize_api_error(line.get("error", "An error occurred")) - } - data = json.dumps(sanitized_error) - yield f"data: {data}\n\n" + yield _emit( + { + "type": "error", + "error": sanitize_api_error( + line.get("error", "An error occurred") + ), + } + ) else: - data = json.dumps(line) - yield f"data: {data}\n\n" + yield _emit(line) if is_structured and structured_chunks: - structured_data = { - "type": "structured_answer", - "answer": response_full, - "structured": True, - "schema": schema_info, - } - data = json.dumps(structured_data) - yield f"data: {data}\n\n" + yield _emit( + { + "type": "structured_answer", + "answer": response_full, + "structured": True, + "schema": schema_info, + } + ) # ---- Paused: save continuation state and end stream early ---- if paused: @@ -452,6 +527,7 @@ class BaseAnswerResource: exc_info=True, ) + state_saved = False if conversation_id: try: cont_service = ContinuationService() @@ -485,18 +561,65 @@ class BaseAnswerResource: agent.tool_executor, "client_tools", None ), ) + state_saved = True except Exception as e: logger.error( f"Failed to save continuation state: {str(e)}", exc_info=True, ) - id_data = {"type": "id", "id": str(conversation_id)} - data = json.dumps(id_data) - yield f"data: {data}\n\n" + # Notify the user out-of-band so they can navigate + # back to the conversation and decide on the + # pending tool calls. Gated on ``state_saved``: a + # missing pending_tool_state row would 404 the + # resume endpoint, so an unfulfillable notification + # is worse than no notification. + user_id_for_event = ( + decoded_token.get("sub") if decoded_token else None + ) + if state_saved and user_id_for_event and conversation_id: + pending_calls = continuation.get( + "pending_tool_calls", [] + ) if continuation else [] + # Trim each pending tool call to its identifying + # metadata so a tool with a multi-MB argument + # doesn't blow out the per-event payload size + # cap. The resume page fetches full args from + # ``pending_tool_state`` regardless. + pending_summaries = [ + { + k: tc.get(k) + for k in ( + "call_id", + "tool_name", + "action_name", + "name", + ) + if isinstance(tc, dict) and tc.get(k) is not None + } + for tc in (pending_calls or []) + if isinstance(tc, dict) + ] + publish_user_event( + user_id_for_event, + "tool.approval.required", + { + "conversation_id": str(conversation_id), + "message_id": reserved_message_id, + "pending_tool_calls": pending_summaries, + }, + scope={ + "kind": "conversation", + "id": str(conversation_id), + }, + ) - data = json.dumps({"type": "end"}) - yield f"data: {data}\n\n" + yield _emit({"type": "id", "id": str(conversation_id)}) + yield _emit({"type": "end"}) + # Drain the terminal ``end`` so a reconnecting client + # sees it on snapshot — same reason as the main exit. + if journal_writer is not None: + journal_writer.close() return if isNoneDoc: @@ -603,9 +726,7 @@ class BaseAnswerResource: f"completion: {e}", exc_info=True, ) - id_data = {"type": "id", "id": str(conversation_id)} - data = json.dumps(id_data) - yield f"data: {data}\n\n" + yield _emit({"type": "id", "id": str(conversation_id)}) tool_calls_for_logging = self._prepare_tool_calls_for_logging( getattr(agent, "tool_calls", tool_calls) or tool_calls @@ -646,12 +767,33 @@ class BaseAnswerResource: exc_info=True, ) - data = json.dumps({"type": "end"}) - yield f"data: {data}\n\n" + yield _emit({"type": "end"}) + # Drain the journal buffer so the terminal ``end`` event is + # visible to any reconnecting client. Without this the + # client could snapshot up to the last flush boundary and + # then live-tail waiting for an ``end`` that's still + # sitting in memory. + if journal_writer is not None: + journal_writer.close() except GeneratorExit: logger.info(f"Stream aborted by client for question: {question[:50]}... ") + # Drain any buffered events before the terminal one-shot + # ``record_event`` below — keeps the journal's seq order + # contiguous (buffered events ... terminal event). ``close`` + # is idempotent; pairing it with ``flush`` matches the + # normal-exit and error branches so any future ``record()`` + # past this point would log instead of silently buffering. + if journal_writer is not None: + journal_writer.flush() + journal_writer.close() # Save partial response + # Whether the DB row was flipped to ``complete`` during this + # abort handler. Drives the choice of terminal journal event + # below: journal ``end`` only when the row actually matches, + # else journal ``error`` so a reconnecting client sees a + # failed terminal state instead of a blank "success". + finalized_complete = False if should_save_conversation and response_full: try: if isNoneDoc: @@ -686,7 +828,7 @@ class BaseAnswerResource: ) llm._token_usage_source = "title" if reserved_message_id is not None: - self.conversation_service.finalize_message( + outcome = self.conversation_service.finalize_message( reserved_message_id, response_full, thought=thought, @@ -705,6 +847,15 @@ class BaseAnswerResource: ), }, ) + # ``ALREADY_COMPLETE`` means the normal-path + # finalize at line 632 won the race: the DB row + # is already at ``complete`` and the reconnect + # journal should reflect that with ``end``, + # not a spurious ``error``. + finalized_complete = outcome in ( + MessageUpdateOutcome.UPDATED, + MessageUpdateOutcome.ALREADY_COMPLETE, + ) else: self.conversation_service.save_conversation( conversation_id, @@ -724,6 +875,9 @@ class BaseAnswerResource: attachment_ids=attachment_ids, metadata=query_metadata if query_metadata else None, ) + # No journal row to gate, but flag the save as + # successful for symmetry with the WAL path. + finalized_complete = True compression_meta = getattr(agent, "compression_metadata", None) compression_saved = getattr(agent, "compression_saved", False) if conversation_id and compression_meta and not compression_saved: @@ -747,6 +901,63 @@ class BaseAnswerResource: logger.error( f"Error saving partial response: {str(e)}", exc_info=True ) + # Journal a terminal event so reconnecting clients stop tailing; + # ``end`` only when the row is ``complete``, else ``error``. + if reserved_message_id is not None: + try: + sequence_no += 1 + if finalized_complete: + # Match the wire shape ``_emit({"type": "end"})`` + # uses on the normal path — the replay terminal + # check at ``event_replay._payload_is_terminal`` + # reads ``payload.type``, and the frontend parses + # the same key off ``data:``. + record_event( + reserved_message_id, + sequence_no, + "end", + {"type": "end"}, + ) + else: + # Nothing was persisted under the complete status + # — mark the row failed so the reconciler doesn't + # need to sweep it, and journal an ``error`` so a + # reconnecting client surfaces the same failure + # the UI would show on a live error. + try: + self.conversation_service.finalize_message( + reserved_message_id, + response_full or TERMINATED_RESPONSE_PLACEHOLDER, + thought=thought, + sources=source_log_docs, + tool_calls=tool_calls, + model_id=model_id or self.default_model_id, + metadata=query_metadata if query_metadata else None, + status="failed", + error=ConnectionError( + "client disconnected before response was persisted" + ), + ) + except Exception as fin_err: + logger.error( + f"Failed to mark aborted message failed: {fin_err}", + exc_info=True, + ) + record_event( + reserved_message_id, + sequence_no, + "error", + { + "type": "error", + "error": "Stream aborted before any response was produced.", + "code": "client_disconnect", + }, + ) + except Exception as journal_err: + logger.error( + f"Failed to journal terminal event on abort: {journal_err}", + exc_info=True, + ) raise except Exception as e: logger.error(f"Error in stream: {str(e)}", exc_info=True) @@ -768,13 +979,16 @@ class BaseAnswerResource: f"Failed to finalize errored message: {fin_err}", exc_info=True, ) - data = json.dumps( + yield _emit( { "type": "error", "error": "Please try again later. We apologize for any inconvenience.", } ) - yield f"data: {data}\n\n" + # Drain the terminal ``error`` event we just yielded so a + # reconnecting client sees it on snapshot. + if journal_writer is not None: + journal_writer.close() return def process_response_stream(self, stream) -> Dict[str, Any]: @@ -796,8 +1010,22 @@ class BaseAnswerResource: for line in stream: try: - event_data = line.replace("data: ", "").strip() + # Each chunk may carry an ``id: `` header before + # the ``data:`` line. Pull just the ``data:`` body so + # the JSON decode doesn't choke on the SSE framing. + event_data = "" + for raw in line.split("\n"): + if raw.startswith("data:"): + event_data = raw[len("data:") :].lstrip() + break + if not event_data: + continue event = json.loads(event_data) + # The ``message_id`` event is informational for the + # streaming consumer and has no synchronous-API field; + # skip it so the type-switch below doesn't KeyError. + if event.get("type") == "message_id": + continue if event["type"] == "id": conversation_id = event["id"] diff --git a/application/api/answer/routes/messages.py b/application/api/answer/routes/messages.py new file mode 100644 index 00000000..b0f4e159 --- /dev/null +++ b/application/api/answer/routes/messages.py @@ -0,0 +1,135 @@ +"""GET /api/messages//events — chat-stream reconnect endpoint. + +Authenticates the caller, verifies ``message_id`` belongs to the user, +then hands off to ``build_message_event_stream`` for snapshot+tail. +""" + +from __future__ import annotations + +import logging +import re +from typing import Iterator, Optional + +from flask import Blueprint, Response, jsonify, make_response, request, stream_with_context +from sqlalchemy import text + +from application.core.settings import settings +from application.storage.db.session import db_readonly +from application.streaming.event_replay import ( + DEFAULT_KEEPALIVE_SECONDS, + DEFAULT_POLL_TIMEOUT_SECONDS, + build_message_event_stream, +) + +logger = logging.getLogger(__name__) + +messages_bp = Blueprint("message_stream", __name__) + +# A message_id is the canonical UUID hex format. Reject anything else +# before the SQL layer so a malformed cookie can't surface as a 500. +_MESSAGE_ID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) +# ``sequence_no`` is a non-negative decimal integer. Anything else is +# corrupt client state — fall through to a fresh-replay cursor and let +# the snapshot reader catch the client up. +_SEQUENCE_NO_RE = re.compile(r"^\d+$") + + +def _normalise_last_event_id(raw: Optional[str]) -> Optional[int]: + if raw is None: + return None + raw = raw.strip() + if not raw or not _SEQUENCE_NO_RE.match(raw): + return None + return int(raw) + + +def _user_owns_message(message_id: str, user_id: str) -> bool: + """Return True iff ``message_id`` belongs to ``user_id``.""" + try: + with db_readonly() as conn: + row = conn.execute( + text( + """ + SELECT 1 FROM conversation_messages + WHERE id = CAST(:id AS uuid) + AND user_id = :u + LIMIT 1 + """ + ), + {"id": message_id, "u": user_id}, + ).first() + return row is not None + except Exception: + logger.exception( + "Ownership lookup failed for message_id=%s user_id=%s", + message_id, + user_id, + ) + return False + + +@messages_bp.route("/api/messages//events", methods=["GET"]) +def stream_message_events(message_id: str) -> Response: + decoded = getattr(request, "decoded_token", None) + user_id = decoded.get("sub") if isinstance(decoded, dict) else None + if not user_id: + return make_response( + jsonify({"success": False, "message": "Authentication required"}), + 401, + ) + + if not _MESSAGE_ID_RE.match(message_id): + return make_response( + jsonify({"success": False, "message": "Invalid message id"}), + 400, + ) + + if not _user_owns_message(message_id, user_id): + # Don't disclose whether the row exists — a malicious caller + # gets the same 404 whether the id is bogus, taken by another + # user, or simply gone. + return make_response( + jsonify({"success": False, "message": "Not found"}), + 404, + ) + + raw_cursor = request.headers.get("Last-Event-ID") or request.args.get( + "last_event_id" + ) + last_event_id = _normalise_last_event_id(raw_cursor) + keepalive_seconds = float( + getattr(settings, "SSE_KEEPALIVE_SECONDS", DEFAULT_KEEPALIVE_SECONDS) + ) + + @stream_with_context + def generate() -> Iterator[str]: + try: + yield from build_message_event_stream( + message_id, + last_event_id=last_event_id, + keepalive_seconds=keepalive_seconds, + poll_timeout_seconds=DEFAULT_POLL_TIMEOUT_SECONDS, + ) + except GeneratorExit: + return + except Exception: + logger.exception( + "Reconnect stream crashed for message_id=%s user_id=%s", + message_id, + user_id, + ) + + response = Response(generate(), mimetype="text/event-stream") + response.headers["Cache-Control"] = "no-store" + response.headers["X-Accel-Buffering"] = "no" + response.headers["Connection"] = "keep-alive" + logger.info( + "message.event.connect message_id=%s user_id=%s last_event_id=%s", + message_id, + user_id, + last_event_id if last_event_id is not None else "-", + ) + return response diff --git a/application/api/answer/services/conversation_service.py b/application/api/answer/services/conversation_service.py index 28b66980..68d317cc 100644 --- a/application/api/answer/services/conversation_service.py +++ b/application/api/answer/services/conversation_service.py @@ -15,7 +15,10 @@ from sqlalchemy import text as sql_text from application.core.settings import settings from application.storage.db.base_repository import looks_like_uuid from application.storage.db.repositories.agents import AgentsRepository -from application.storage.db.repositories.conversations import ConversationsRepository +from application.storage.db.repositories.conversations import ( + ConversationsRepository, + MessageUpdateOutcome, +) from application.storage.db.session import db_readonly, db_session @@ -305,10 +308,17 @@ class ConversationService: status: str = "complete", error: Optional[BaseException] = None, title_inputs: Optional[Dict[str, Any]] = None, - ) -> bool: - """Commit the response and tool_call confirms in one transaction.""" + ) -> MessageUpdateOutcome: + """Commit the response and tool_call confirms in one transaction. + + The outcome propagates directly from ``update_message_by_id`` so + callers (notably the SSE abort handler) can tell a fresh + finalize from "the row was already terminal" — the latter must + still be treated as success when the prior state was + ``complete``. + """ if not message_id: - return False + return MessageUpdateOutcome.INVALID sources = sources or [] for source in sources: if "text" in source and isinstance(source["text"], str): @@ -336,16 +346,16 @@ class ConversationService: # retracting a row the reconciler already escalated. with db_session() as conn: repo = ConversationsRepository(conn) - ok = repo.update_message_by_id( + outcome = repo.update_message_by_id( message_id, update_fields, only_if_non_terminal=True, ) - if not ok: + if outcome is not MessageUpdateOutcome.UPDATED: logger.warning( f"finalize_message: no row updated for message_id={message_id} " - f"(possibly already terminal — reconciler may have escalated)" + f"(outcome={outcome.value} — possibly already terminal)" ) - return False + return outcome repo.confirm_executed_tool_calls(message_id) # Outside the txn — title-gen is a multi-second LLM round trip. @@ -358,7 +368,7 @@ class ConversationService: f"finalize_message title generation failed: {e}", exc_info=True, ) - return True + return MessageUpdateOutcome.UPDATED def _maybe_generate_title( self, diff --git a/application/api/events/__init__.py b/application/api/events/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/application/api/events/routes.py b/application/api/events/routes.py new file mode 100644 index 00000000..039259f1 --- /dev/null +++ b/application/api/events/routes.py @@ -0,0 +1,504 @@ +"""GET /api/events — user-scoped Server-Sent Events endpoint. + +Subscribe-then-snapshot pattern: subscribe to ``user:{user_id}`` +pub/sub, snapshot the Redis Streams backlog past ``Last-Event-ID`` +inside the SUBSCRIBE-ack callback, flush snapshot, then tail live +events (dedup'd by stream id). See ``docs/runbooks/sse-notifications.md``. +""" + +from __future__ import annotations + +import json +import logging +import re +import time +from typing import Iterator, Optional + +from flask import Blueprint, Response, jsonify, make_response, request, stream_with_context + +from application.cache import get_redis_instance +from application.core.settings import settings +from application.events.keys import ( + connection_counter_key, + replay_budget_key, + stream_id_compare, + stream_key, + topic_name, +) +from application.streaming.broadcast_channel import Topic + +logger = logging.getLogger(__name__) + +events = Blueprint("event_stream", __name__) + +SUBSCRIBE_POLL_INTERVAL_SECONDS = 1.0 + +# WHATWG SSE treats CRLF, CR, and LF equivalently as line terminators. +_SSE_LINE_SPLIT = re.compile(r"\r\n|\r|\n") + +# Redis Streams ids are ``ms`` or ``ms-seq`` where both halves are decimal. +# Anything else is a corrupted client cookie / IndexedDB residue and must +# not be passed to XRANGE — Redis would reject it and our truncation gate +# would silently fail. +_STREAM_ID_RE = re.compile(r"^\d+(-\d+)?$") + +# Only emitted at most once per process so a misconfigured deployment +# doesn't drown the logs. +_local_user_warned = False + + +def _format_sse(data: str, *, event_id: Optional[str] = None) -> str: + """Encode a payload as one SSE message terminated by a blank line. + + Splits on any line-terminator variant (``\\r\\n``, ``\\r``, ``\\n``) + so a stray CR in upstream content can't smuggle a premature line + boundary into the wire format. + """ + lines: list[str] = [] + if event_id: + lines.append(f"id: {event_id}") + for line in _SSE_LINE_SPLIT.split(data): + lines.append(f"data: {line}") + return "\n".join(lines) + "\n\n" + + +def _decode(value) -> Optional[str]: + if value is None: + return None + if isinstance(value, (bytes, bytearray)): + try: + return value.decode("utf-8") + except Exception: + return None + return str(value) + + +def _oldest_retained_id(redis_client, user_id: str) -> Optional[str]: + """Return the id of the oldest entry still in the stream, or ``None``. + + Used to detect ``Last-Event-ID`` having slid off the back of the + MAXLEN'd window. + """ + try: + info = redis_client.xinfo_stream(stream_key(user_id)) + except Exception: + return None + if not isinstance(info, dict): + return None + # redis-py 7.4 returns str-keyed dicts here; the bytes-key probe is + # defence in depth in case ``decode_responses`` is ever flipped. + first_entry = info.get("first-entry") or info.get(b"first-entry") + if not first_entry: + return None + # XINFO STREAM returns first-entry as [id, [field, value, ...]] + try: + return _decode(first_entry[0]) + except Exception: + return None + + +def _allow_replay( + redis_client, user_id: str, last_event_id: Optional[str] +) -> bool: + """Per-user sliding-window snapshot-replay budget. + + Fails open on Redis errors or when the budget is disabled. Empty-backlog + no-cursor connects skip INCR so dev double-mounts don't trip 429. + """ + budget = int(settings.EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW) + if budget <= 0: + return True + if redis_client is None: + return True + + # Cheap pre-check: only INCR when we might actually replay. XLEN + # is one Redis op; the alternative (INCR every connect) is two + # ops AND wrongly counts no-op probes. The check is conservative: + # if ``last_event_id`` is set we always INCR, even if the cursor + # has already overtaken the latest entry — that case is rare and + # short-lived, and probing further would mean a redundant XRANGE. + if last_event_id is None: + try: + if int(redis_client.xlen(stream_key(user_id))) == 0: + return True + except Exception: + # XLEN probe failed; fall through to the INCR path so a + # transient Redis hiccup can't bypass the budget. + logger.debug( + "XLEN probe failed for replay budget check user=%s; " + "proceeding to INCR", + user_id, + ) + + window = max(1, int(settings.EVENTS_REPLAY_BUDGET_WINDOW_SECONDS)) + key = replay_budget_key(user_id) + try: + used = int(redis_client.incr(key)) + # Always (re)seed the TTL. Gating on ``used == 1`` would wedge + # the counter forever if INCR succeeds but EXPIRE raises on + # the seeding call. EXPIRE on an existing key resets the TTL + # to ``window`` — within ±1s of the per-window budget semantic. + redis_client.expire(key, window) + except Exception: + logger.debug( + "replay budget probe failed for user=%s; failing open", + user_id, + ) + return True + return used <= budget + + +def _normalize_last_event_id(raw: Optional[str]) -> Optional[str]: + """Validate the ``Last-Event-ID`` header / query param. + + Returns the value unchanged when it parses as a Redis Streams id, + otherwise ``None`` — callers treat ``None`` as "client has nothing" + and replay from the start of the retained window. Invalid ids would + otherwise pass straight to XRANGE and surface as a quiet replay + failure plus broken truncation detection. + """ + if raw is None: + return None + raw = raw.strip() + if not raw or not _STREAM_ID_RE.match(raw): + return None + return raw + + +def _replay_backlog( + redis_client, user_id: str, last_event_id: Optional[str], max_count: int +) -> Iterator[tuple[str, str]]: + """Yield ``(entry_id, sse_line)`` for backlog entries past ``last_event_id``. + + Capped at ``max_count`` rows; clients catch up across reconnects. + Parse failures are skipped; the Streams id is injected into the + envelope so replay matches live-tail shape. + """ + # Exclusive start: '(' skips the already-delivered entry. + start = f"({last_event_id}" if last_event_id else "-" + try: + entries = redis_client.xrange( + stream_key(user_id), min=start, max="+", count=max_count + ) + except Exception as exc: + logger.warning( + "xrange replay failed for user=%s last_id=%s err=%s", + user_id, + last_event_id or "-", + exc, + ) + return + + for entry_id, fields in entries: + entry_id_str = _decode(entry_id) + if not entry_id_str: + continue + # decode_responses=False on the cache client ⇒ field keys/values + # are bytes. The string-key fallback covers a future flip of that + # default without a forced refactor here. + raw_event = None + if isinstance(fields, dict): + raw_event = fields.get(b"event") + if raw_event is None: + raw_event = fields.get("event") + event_str = _decode(raw_event) + if not event_str: + continue + try: + envelope = json.loads(event_str) + if isinstance(envelope, dict): + envelope["id"] = entry_id_str + event_str = json.dumps(envelope) + except Exception: + logger.debug( + "Replay envelope parse failed for entry %s; passing through raw", + entry_id_str, + ) + yield entry_id_str, _format_sse(event_str, event_id=entry_id_str) + + +def _truncation_notice_line(oldest_id: str) -> str: + """SSE event the frontend can react to with a full-state refetch.""" + return _format_sse( + json.dumps( + { + "type": "backlog.truncated", + "payload": {"oldest_retained_id": oldest_id}, + } + ) + ) + + +@events.route("/api/events", methods=["GET"]) +def stream_events() -> Response: + decoded = getattr(request, "decoded_token", None) + user_id = decoded.get("sub") if isinstance(decoded, dict) else None + if not user_id: + return make_response( + jsonify({"success": False, "message": "Authentication required"}), + 401, + ) + + # In dev deployments without AUTH_TYPE configured, every request + # resolves to user_id="local" and shares one stream. Surface this so + # an accidentally-multi-user dev box doesn't silently cross-stream. + global _local_user_warned + if user_id == "local" and not _local_user_warned: + logger.warning( + "SSE serving user_id='local' (AUTH_TYPE not set). " + "All clients on this deployment will share one event stream." + ) + _local_user_warned = True + + raw_last_event_id = request.headers.get("Last-Event-ID") or request.args.get( + "last_event_id" + ) + last_event_id = _normalize_last_event_id(raw_last_event_id) + last_event_id_invalid = raw_last_event_id is not None and last_event_id is None + + keepalive_seconds = float(settings.SSE_KEEPALIVE_SECONDS) + push_enabled = settings.ENABLE_SSE_PUSH + cap = int(settings.SSE_MAX_CONCURRENT_PER_USER) + + redis_client = get_redis_instance() + counter_key = connection_counter_key(user_id) + counted = False + + if push_enabled and redis_client is not None and cap > 0: + try: + current = int(redis_client.incr(counter_key)) + counted = True + except Exception: + current = 0 + logger.debug( + "SSE connection counter INCR failed for user=%s", user_id + ) + if counted: + # 1h safety TTL — orphaned counts from hard crashes self-heal. + # EXPIRE failure must NOT clobber ``current`` and bypass the cap. + try: + redis_client.expire(counter_key, 3600) + except Exception: + logger.debug( + "SSE connection counter EXPIRE failed for user=%s", user_id + ) + if current > cap: + try: + redis_client.decr(counter_key) + except Exception: + logger.debug( + "SSE connection counter DECR failed for user=%s", + user_id, + ) + return make_response( + jsonify( + { + "success": False, + "message": "Too many concurrent SSE connections", + } + ), + 429, + ) + + # Replay budget is checked here, before the generator opens the + # stream, so a denial can surface as HTTP 429 instead of a silent + # snapshot skip. The earlier in-generator skip lost events between + # the client's cursor and the first live-tailed entry: the live + # tail still carried ``id:`` headers, the frontend advanced + # ``lastEventId`` to one of those ids, and the events in between + # were never reachable on the next reconnect. 429 keeps the + # cursor pinned and lets the frontend back off until the window + # slides (eventStreamClient.ts treats 429 as escalated backoff). + if push_enabled and redis_client is not None and not _allow_replay( + redis_client, user_id, last_event_id + ): + if counted: + try: + redis_client.decr(counter_key) + except Exception: + logger.debug( + "SSE connection counter DECR failed for user=%s", + user_id, + ) + return make_response( + jsonify( + { + "success": False, + "message": "Replay budget exhausted", + } + ), + 429, + ) + + @stream_with_context + def generate() -> Iterator[str]: + connect_ts = time.monotonic() + replayed_count = 0 + try: + # First frame primes intermediaries (Cloudflare, nginx) so they + # don't sit on a buffer waiting for body bytes. + yield ": connected\n\n" + + if not push_enabled: + yield ": push_disabled\n\n" + return + + replay_lines: list[str] = [] + max_replayed_id: Optional[str] = None + replay_done = False + + # If the client sent a malformed Last-Event-ID, surface the + # truncation notice synchronously *before* the subscribe + # loop. Buffering it into ``replay_lines`` would lose it + # when ``Topic.subscribe`` returns immediately (Redis down) + # — the loop body never runs, and the flush at line ~335 + # never fires. + if last_event_id_invalid: + yield _truncation_notice_line("") + replayed_count += 1 + + def _on_subscribe_callback() -> None: + # Runs synchronously inside Topic.subscribe after the + # SUBSCRIBE is acked. By doing XRANGE here, any publisher + # firing between SUBSCRIBE-send and SUBSCRIBE-ack has its + # XADD captured by XRANGE *and* its PUBLISH buffered at + # the connection layer until we read it — closing the + # replay/subscribe race the design doc warns about. + # + # Truncation contract: ``backlog.truncated`` is emitted + # ONLY when the client's ``Last-Event-ID`` has slid off + # the MAXLEN'd window — that's the case where the + # journal is genuinely gone past the cursor and the + # frontend should clear its slice cursor and refetch + # state. Cap-hit skips the snapshot silently: the + # cursor advances via the per-entry ``id:`` headers + # and the frontend's slice keeps the latest id so the + # next reconnect resumes from there. Budget-exhausted + # never reaches this callback — the route 429s before + # opening the stream, keeping the cursor pinned. + # Conflating these with stale-cursor truncation would + # tell the client to clear its cursor and re-receive + # the same oldest-N entries on every reconnect — + # locking the user out of entries past N. + nonlocal max_replayed_id, replay_done + try: + if redis_client is None: + return + oldest = _oldest_retained_id(redis_client, user_id) + if ( + last_event_id + and oldest + and stream_id_compare(last_event_id, oldest) < 0 + ): + # The Last-Event-ID has slid off the MAXLEN window. + # Tell the client so it can fetch full state. + replay_lines.append(_truncation_notice_line(oldest)) + replay_cap = int(settings.EVENTS_REPLAY_MAX_PER_REQUEST) + for entry_id, sse_line in _replay_backlog( + redis_client, user_id, last_event_id, replay_cap + ): + replay_lines.append(sse_line) + max_replayed_id = entry_id + finally: + # Always flip the flag — even on partial-replay failure + # the outer loop must reach the flush step so we don't + # silently strand whatever entries did land. + replay_done = True + + topic = Topic(topic_name(user_id)) + last_keepalive = time.monotonic() + for payload in topic.subscribe( + on_subscribe=_on_subscribe_callback, + poll_timeout=SUBSCRIBE_POLL_INTERVAL_SECONDS, + ): + # Flush snapshot on the first iteration after the SUBSCRIBE + # callback ran. This runs at most once per connection. + if replay_done and replay_lines: + for line in replay_lines: + yield line + replayed_count += 1 + replay_lines.clear() + + now = time.monotonic() + if payload is None: + if now - last_keepalive >= keepalive_seconds: + yield ": keepalive\n\n" + last_keepalive = now + continue + + event_str = _decode(payload) or "" + event_id: Optional[str] = None + try: + envelope = json.loads(event_str) + if isinstance(envelope, dict): + candidate = envelope.get("id") + # Only trust ids that look like real Redis Streams + # ids (``ms`` or ``ms-seq``). A malformed or + # adversarial publisher could otherwise pin + # dedupe forever — a lex-greater bogus id would + # make every legitimate later id compare ``<=`` + # and get dropped silently. + if isinstance(candidate, str) and _STREAM_ID_RE.match( + candidate + ): + event_id = candidate + except Exception: + pass + + # Dedupe: if this id was already covered by replay, drop it. + if ( + event_id is not None + and max_replayed_id is not None + and stream_id_compare(event_id, max_replayed_id) <= 0 + ): + continue + + yield _format_sse(event_str, event_id=event_id) + last_keepalive = now + + # Topic.subscribe exited before the first yield (transient + # Redis hiccup between SUBSCRIBE-ack and the first poll, or + # an immediate Redis-down return). The callback may already + # have populated the snapshot — flush it so the client gets + # the backlog instead of a silent drop. Safe no-op when the + # in-loop flush ran (it clear()'d the buffer) and when the + # callback never fired (replay_done stays False). + if replay_done and replay_lines: + for line in replay_lines: + yield line + replayed_count += 1 + replay_lines.clear() + except GeneratorExit: + return + except Exception: + logger.exception( + "SSE event-stream generator crashed for user=%s", user_id + ) + finally: + duration_s = time.monotonic() - connect_ts + logger.info( + "event.disconnect user=%s duration_s=%.1f replayed=%d", + user_id, + duration_s, + replayed_count, + ) + if counted and redis_client is not None: + try: + redis_client.decr(counter_key) + except Exception: + logger.debug( + "SSE connection counter DECR failed for user=%s on disconnect", + user_id, + ) + + response = Response(generate(), mimetype="text/event-stream") + response.headers["Cache-Control"] = "no-store" + response.headers["X-Accel-Buffering"] = "no" + response.headers["Connection"] = "keep-alive" + logger.info( + "event.connect user=%s last_event_id=%s%s", + user_id, + last_event_id or "-", + " (rejected_invalid)" if last_event_id_invalid else "", + ) + return response diff --git a/application/api/user/attachments/routes.py b/application/api/user/attachments/routes.py index 0ea0cbe5..fabecd74 100644 --- a/application/api/user/attachments/routes.py +++ b/application/api/user/attachments/routes.py @@ -214,6 +214,10 @@ class StoreAttachment(Resource): { "success": True, "task_id": tasks[0]["task_id"], + # Surface the attachment_id so the frontend + # can correlate ``attachment.*`` SSE events + # to this row and skip the polling fallback. + "attachment_id": tasks[0]["attachment_id"], "message": "File uploaded successfully. Processing started.", } ), diff --git a/application/api/user/conversations/routes.py b/application/api/user/conversations/routes.py index 3fbd241c..d5dc0266 100644 --- a/application/api/user/conversations/routes.py +++ b/application/api/user/conversations/routes.py @@ -7,9 +7,13 @@ from flask_restx import fields, Namespace, Resource from sqlalchemy import text as sql_text from application.api import api +from application.api.answer.services.conversation_service import ( + TERMINATED_RESPONSE_PLACEHOLDER, +) from application.storage.db.base_repository import looks_like_uuid, row_to_dict from application.storage.db.repositories.attachments import AttachmentsRepository from application.storage.db.repositories.conversations import ConversationsRepository +from application.storage.db.repositories.message_events import MessageEventsRepository from application.storage.db.session import db_readonly, db_session from application.utils import check_required_fields @@ -346,6 +350,25 @@ class GetMessageTail(Resource): if row is None: return make_response(jsonify({"status": "not found"}), 404) msg = row_to_dict(row) + # Mid-stream the row's response is the placeholder; rebuild + # the live partial from the journal so /tail mirrors SSE. + status = msg.get("status") + response = msg.get("response") + thought = msg.get("thought") + sources = msg.get("sources") or [] + tool_calls = msg.get("tool_calls") or [] + if status in ("pending", "streaming") and ( + response == TERMINATED_RESPONSE_PLACEHOLDER + ): + partial = MessageEventsRepository(conn).reconstruct_partial( + message_id + ) + response = partial["response"] + thought = partial["thought"] or thought + if partial["sources"]: + sources = partial["sources"] + if partial["tool_calls"]: + tool_calls = partial["tool_calls"] except Exception as err: current_app.logger.error( f"Error tailing message {message_id}: {err}", exc_info=True @@ -356,11 +379,11 @@ class GetMessageTail(Resource): jsonify( { "message_id": str(msg["id"]), - "status": msg.get("status"), - "response": msg.get("response"), - "thought": msg.get("thought"), - "sources": msg.get("sources") or [], - "tool_calls": msg.get("tool_calls") or [], + "status": status, + "response": response, + "thought": thought, + "sources": sources, + "tool_calls": tool_calls, "request_id": msg.get("request_id"), "last_heartbeat_at": metadata.get("last_heartbeat_at"), "error": metadata.get("error"), diff --git a/application/api/user/sources/upload.py b/application/api/user/sources/upload.py index a7affbe5..9292406b 100644 --- a/application/api/user/sources/upload.py +++ b/application/api/user/sources/upload.py @@ -13,6 +13,7 @@ from sqlalchemy import text as sql_text from application.api import api from application.api.user.tasks import ingest, ingest_connector_task, ingest_remote from application.core.settings import settings +from application.storage.db.source_ids import derive_source_id as _derive_source_id from application.parser.connectors.connector_creator import ConnectorCreator from application.parser.file.constants import SUPPORTED_SOURCE_EXTENSIONS from application.storage.db.repositories.idempotency import IdempotencyRepository @@ -69,7 +70,13 @@ def _claim_task_or_get_cached(key, task_name): Pre-generates the celery task_id so a losing writer sees the same id immediately. Returns ``(task_id, cached_response)``; non-None - cached means the caller should return without enqueuing. + cached means the caller should return without enqueuing. The + cached payload mirrors the fresh-request response shape (including + ``source_id``) so the frontend can correlate SSE ingest events to + the cached upload task without an extra round-trip — but only when + the cached row actually exists; the "deduplicated" sentinel + deliberately omits ``source_id`` so the frontend doesn't bind to a + phantom source. """ predetermined_id = str(uuid.uuid4()) with db_session() as conn: @@ -81,10 +88,16 @@ def _claim_task_or_get_cached(key, task_name): with db_readonly() as conn: existing = IdempotencyRepository(conn).get_task(key) cached_id = existing.get("task_id") if existing else None - return None, { + payload: dict = { "success": True, "task_id": cached_id or "deduplicated", } + # Only surface ``source_id`` when there's a real winner whose worker + # is publishing SSE events tagged with that id. The "deduplicated" + # branch means the lock row vanished — we have nothing to correlate. + if cached_id is not None: + payload["source_id"] = str(_derive_source_id(key)) + return None, payload def _release_claim(key): @@ -236,6 +249,15 @@ class UploadFile(Resource): file_path = f"{base_path}/{safe_file}" with open(temp_file_path, "rb") as f: storage.save_file(f, file_path) + # Mint the source UUID up here so the HTTP response and the + # worker's SSE envelopes share one id. With an idempotency + # key we reuse the deterministic uuid5 (retried task lands on + # the same source row); without a key we fall back to uuid4. + # The worker is told to use this id verbatim — see + # ``ingest_worker(source_id=...)``. + source_uuid = ( + _derive_source_id(scoped_key) if scoped_key else uuid.uuid4() + ) ingest_kwargs = dict( args=( settings.UPLOAD_FOLDER, @@ -249,6 +271,7 @@ class UploadFile(Resource): "file_name_map": file_name_map, # Scoped so the worker dedup row matches the HTTP claim. "idempotency_key": scoped_key or idempotency_key, + "source_id": str(source_uuid), }, ) if predetermined_task_id is not None: @@ -273,7 +296,15 @@ class UploadFile(Resource): return make_response(jsonify({"success": False}), 400) # Predetermined id matches the dedup-claim row; loser GET sees same. response_task_id = predetermined_task_id or task.id - response_payload = {"success": True, "task_id": response_task_id} + # ``source_uuid`` was minted above and passed to the worker as + # ``source_id``; the worker uses it verbatim for every SSE event, + # so the frontend can correlate inbound ``source.ingest.*`` to + # this upload regardless of whether an idempotency key was set. + response_payload: dict = { + "success": True, + "task_id": response_task_id, + "source_id": str(source_uuid), + } return make_response(jsonify(response_payload), 200) @@ -326,6 +357,18 @@ class UploadRemote(Resource): ) if cached is not None: return make_response(jsonify(cached), 200) + # Mint the source UUID up here so the HTTP response and the + # worker's SSE envelopes share one id. Same pattern as + # ``UploadFile.post``: with an idempotency key we reuse the + # deterministic uuid5 (retried task lands on the same source + # row); without a key we fall back to uuid4. The worker is told + # to use this id verbatim — see ``remote_worker`` and + # ``ingest_connector``. Without this the no-key path would mint + # a random uuid4 inside the worker that the frontend has no way + # to correlate SSE events to. + source_uuid = ( + _derive_source_id(scoped_key) if scoped_key else uuid.uuid4() + ) try: config = json.loads(data["data"]) source_data = None @@ -382,13 +425,23 @@ class UploadRemote(Resource): "recursive": config.get("recursive", False), "retriever": config.get("retriever", "classic"), "idempotency_key": scoped_key or idempotency_key, + "source_id": str(source_uuid), }, } if predetermined_task_id is not None: connector_kwargs["task_id"] = predetermined_task_id task = ingest_connector_task.apply_async(**connector_kwargs) response_task_id = predetermined_task_id or task.id - response_payload = {"success": True, "task_id": response_task_id} + # ``source_uuid`` was minted above and passed to the + # worker as ``source_id``; the worker uses it verbatim + # for every SSE event, so the frontend can correlate + # inbound ``source.ingest.*`` regardless of whether an + # idempotency key was set. + response_payload = { + "success": True, + "task_id": response_task_id, + "source_id": str(source_uuid), + } return make_response(jsonify(response_payload), 200) remote_kwargs = { "kwargs": { @@ -397,6 +450,7 @@ class UploadRemote(Resource): "user": user, "loader": data["source"], "idempotency_key": scoped_key or idempotency_key, + "source_id": str(source_uuid), }, } if predetermined_task_id is not None: @@ -410,7 +464,11 @@ class UploadRemote(Resource): _release_claim(scoped_key) return make_response(jsonify({"success": False}), 400) response_task_id = predetermined_task_id or task.id - response_payload = {"success": True, "task_id": response_task_id} + response_payload = { + "success": True, + "task_id": response_task_id, + "source_id": str(source_uuid), + } return make_response(jsonify(response_payload), 200) @@ -553,6 +611,19 @@ class ManageSourceFiles(Resource): scoped_key, "reingest_source_task", ) if cached is not None: + # Frontend keys reingest polling on + # ``reingest_task_id``; the shared cache helper + # writes ``task_id``. Alias here so a dedup + # response doesn't silently break FileTree's + # poller. Override ``source_id`` too — the + # helper derives it from the scoped key, which + # is correct for upload but wrong for reingest + # (the worker publishes events scoped to the + # actual source row id). + cached_task_id = cached.pop("task_id", None) + if cached_task_id is not None: + cached["reingest_task_id"] = cached_task_id + cached["source_id"] = resolved_source_id return make_response(jsonify(cached), 200) added_files = [] @@ -608,6 +679,12 @@ class ManageSourceFiles(Resource): "added_files": added_files, "parent_dir": parent_dir, "reingest_task_id": task.id, + # ``source_id`` lets the frontend correlate + # inbound ``source.ingest.*`` SSE events + # (emitted by ``reingest_source_worker``) + # back to the reingest task — matches the + # upload route's source-id contract. + "source_id": resolved_source_id, } ), 200, @@ -659,6 +736,15 @@ class ManageSourceFiles(Resource): scoped_key, "reingest_source_task", ) if cached is not None: + cached_task_id = cached.pop("task_id", None) + if cached_task_id is not None: + cached["reingest_task_id"] = cached_task_id + # Override the helper's synthetic source_id (uuid5 + # of the scoped key) with the real source row id + # — the reingest worker publishes SSE events + # scoped to ``resolved_source_id`` and FileTree + # correlates on it. + cached["source_id"] = resolved_source_id return make_response(jsonify(cached), 200) # Remove files from storage and directory structure @@ -704,6 +790,7 @@ class ManageSourceFiles(Resource): "message": f"Removed {len(removed_files)} files", "removed_files": removed_files, "reingest_task_id": task.id, + "source_id": resolved_source_id, } ), 200, @@ -762,6 +849,14 @@ class ManageSourceFiles(Resource): scoped_key, "reingest_source_task", ) if cached is not None: + cached_task_id = cached.pop("task_id", None) + if cached_task_id is not None: + cached["reingest_task_id"] = cached_task_id + # Same source_id override as the ``remove`` / + # ``add`` cached branches — the helper's synthetic + # id doesn't match what reingest_source_worker + # tags its SSE events with. + cached["source_id"] = resolved_source_id return make_response(jsonify(cached), 200) success = storage.remove_directory(full_directory_path) @@ -825,6 +920,7 @@ class ManageSourceFiles(Resource): "message": f"Successfully removed directory: {directory_path}", "removed_directory": directory_path, "reingest_task_id": task.id, + "source_id": resolved_source_id, } ), 200, diff --git a/application/api/user/tasks.py b/application/api/user/tasks.py index d5a15a25..1fc7d091 100644 --- a/application/api/user/tasks.py +++ b/application/api/user/tasks.py @@ -7,7 +7,6 @@ from application.worker import ( attachment_worker, ingest_worker, mcp_oauth, - mcp_oauth_status, remote_worker, sync, sync_worker, @@ -40,6 +39,7 @@ def ingest( filename, file_name_map=None, idempotency_key=None, + source_id=None, ): resp = ingest_worker( self, @@ -51,16 +51,21 @@ def ingest( user, file_name_map=file_name_map, idempotency_key=idempotency_key, + source_id=source_id, ) return resp @celery.task(**DURABLE_TASK) @with_idempotency(task_name="ingest_remote") -def ingest_remote(self, source_data, job_name, user, loader, idempotency_key=None): +def ingest_remote( + self, source_data, job_name, user, loader, + idempotency_key=None, source_id=None, +): resp = remote_worker( self, source_data, job_name, user, loader, idempotency_key=idempotency_key, + source_id=source_id, ) return resp @@ -138,6 +143,7 @@ def ingest_connector_task( doc_id=None, sync_frequency="never", idempotency_key=None, + source_id=None, ): from application.worker import ingest_connector @@ -155,6 +161,7 @@ def ingest_connector_task( doc_id=doc_id, sync_frequency=sync_frequency, idempotency_key=idempotency_key, + source_id=source_id, ) return resp @@ -197,6 +204,15 @@ def setup_periodic_tasks(sender, **kwargs): version_check_task.s(), name="version-check", ) + # Bound ``message_events`` growth — every streamed SSE chunk writes + # one row, so retained chats accumulate hundreds of rows per + # message. Reconnect-replay is only meaningful for streams the user + # could plausibly still be waiting on, so 14 days is generous. + sender.add_periodic_task( + timedelta(hours=24), + cleanup_message_events.s(), + name="cleanup-message-events", + ) @celery.task(bind=True) @@ -205,12 +221,6 @@ def mcp_oauth_task(self, config, user): return resp -@celery.task(bind=True) -def mcp_oauth_status_task(self, task_id): - resp = mcp_oauth_status(self, task_id) - return resp - - @celery.task(bind=True, acks_late=False) def cleanup_pending_tool_state(self): """Revert stale ``resuming`` rows, then delete TTL-expired rows.""" @@ -265,6 +275,32 @@ def reconciliation_task(self): return run_reconciliation() +@celery.task(bind=True, acks_late=False) +def cleanup_message_events(self): + """Delete ``message_events`` rows older than the retention window. + + Streamed answer responses write one journal row per SSE yield, + so unbounded growth would dominate Postgres for any retained- + conversations deployment. The reconnect-replay path only needs + rows for in-flight streams; 14 days covers paused/tool-action + flows comfortably. + """ + from application.core.settings import settings + if not settings.POSTGRES_URI: + return {"deleted": 0, "skipped": "POSTGRES_URI not set"} + + from application.storage.db.engine import get_engine + from application.storage.db.repositories.message_events import ( + MessageEventsRepository, + ) + + ttl_days = settings.MESSAGE_EVENTS_RETENTION_DAYS + engine = get_engine() + with engine.begin() as conn: + deleted = MessageEventsRepository(conn).cleanup_older_than(ttl_days) + return {"deleted": deleted, "ttl_days": ttl_days} + + @celery.task(bind=True, acks_late=False) def version_check_task(self): """Periodic anonymous version check. diff --git a/application/api/user/tools/mcp.py b/application/api/user/tools/mcp.py index a15500f8..fab063fb 100644 --- a/application/api/user/tools/mcp.py +++ b/application/api/user/tools/mcp.py @@ -1,6 +1,5 @@ """Tool management MCP server integration.""" -import json from urllib.parse import urlencode, urlparse from flask import current_app, jsonify, make_response, redirect, request @@ -226,7 +225,9 @@ class MCPServerSave(Resource): ) redis_client = get_redis_instance() manager = MCPOAuthManager(redis_client) - result = manager.get_oauth_status(config["oauth_task_id"]) + result = manager.get_oauth_status( + config["oauth_task_id"], user + ) if not result.get("status") == "completed": return make_response( jsonify( @@ -438,56 +439,6 @@ class MCPOAuthCallback(Resource): ) -@tools_mcp_ns.route("/mcp_server/oauth_status/") -class MCPOAuthStatus(Resource): - def get(self, task_id): - try: - redis_client = get_redis_instance() - status_key = f"mcp_oauth_status:{task_id}" - status_data = redis_client.get(status_key) - - if status_data: - status = json.loads(status_data) - if "tools" in status and isinstance(status["tools"], list): - status["tools"] = [ - { - "name": t.get("name", "unknown"), - "description": t.get("description", ""), - } - for t in status["tools"] - ] - return make_response( - jsonify({"success": True, "task_id": task_id, **status}) - ) - else: - return make_response( - jsonify( - { - "success": True, - "task_id": task_id, - "status": "pending", - "message": "Waiting for OAuth to start...", - } - ), - 200, - ) - except Exception as e: - current_app.logger.error( - f"Error getting OAuth status for task {task_id}: {str(e)}", - exc_info=True, - ) - return make_response( - jsonify( - { - "success": False, - "error": "Failed to get OAuth status", - "task_id": task_id, - } - ), - 500, - ) - - @tools_mcp_ns.route("/mcp_server/auth_status") class MCPAuthStatus(Resource): @api.doc( diff --git a/application/api/v1/routes.py b/application/api/v1/routes.py index cfc9e784..0c862b17 100644 --- a/application/api/v1/routes.py +++ b/application/api/v1/routes.py @@ -222,13 +222,26 @@ def _stream_response( for line in internal_stream: if not line.strip(): continue - # Parse the internal SSE event - event_str = line.replace("data: ", "").strip() + # ``complete_stream`` prefixes each frame with ``id: \n`` + # before the ``data:`` line. Extract just the data line so JSON + # decode doesn't choke on the SSE framing. + event_str = "" + for raw in line.split("\n"): + if raw.startswith("data:"): + event_str = raw[len("data:") :].lstrip() + break + if not event_str: + continue try: event_data = json.loads(event_str) except (json.JSONDecodeError, TypeError): continue + # Skip the informational ``message_id`` event — it has no v1 / + # OpenAI-compatible analog. + if event_data.get("type") == "message_id": + continue + # Update completion_id when we get the conversation id if event_data.get("type") == "id": conv_id = event_data.get("id", "") diff --git a/application/app.py b/application/app.py index b9bfe574..23a59099 100644 --- a/application/app.py +++ b/application/app.py @@ -16,6 +16,8 @@ setup_logging() from application.api import api # noqa: E402 from application.api.answer import answer # noqa: E402 +from application.api.answer.routes.messages import messages_bp # noqa: E402 +from application.api.events.routes import events # noqa: E402 from application.api.internal.routes import internal # noqa: E402 from application.api.user.routes import user # noqa: E402 from application.api.connector.routes import connector # noqa: E402 @@ -49,6 +51,8 @@ ensure_database_ready( app = Flask(__name__) app.register_blueprint(user) app.register_blueprint(answer) +app.register_blueprint(events) +app.register_blueprint(messages_bp) app.register_blueprint(internal) app.register_blueprint(connector) app.register_blueprint(v1_bp) diff --git a/application/cache.py b/application/cache.py index 740c13c4..4e2b6719 100644 --- a/application/cache.py +++ b/application/cache.py @@ -29,8 +29,17 @@ def get_redis_instance(): with _instance_lock: if _redis_instance is None and not _redis_creation_failed: try: + # ``health_check_interval`` makes redis-py ping the + # connection every N seconds when otherwise idle. + # Without it, a half-open TCP (NAT silently dropped + # state, ELB idle-close) can hang the SSE generator + # in ``pubsub.get_message`` past its keepalive + # cadence — the kernel never surfaces the dead + # socket because no payload is in flight. _redis_instance = redis.Redis.from_url( - settings.CACHE_REDIS_URL, socket_connect_timeout=2 + settings.CACHE_REDIS_URL, + socket_connect_timeout=2, + health_check_interval=10, ) except ValueError as e: logger.error(f"Invalid Redis URL: {e}") diff --git a/application/core/settings.py b/application/core/settings.py index 16a860d8..dfb4cc66 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -188,6 +188,42 @@ class Settings(BaseSettings): COMPRESSION_PROMPT_VERSION: str = "v1.0" # Track prompt iterations COMPRESSION_MAX_HISTORY_POINTS: int = 3 # Keep only last N compression points to prevent DB bloat + # Internal SSE push channel (notifications + durable replay journal) + # Master switch — when False, /api/events emits a "push_disabled" comment + # and returns; clients fall back to polling. Publisher becomes a no-op. + ENABLE_SSE_PUSH: bool = True + # Per-user durable backlog cap (~entries). At typical event rates this + # gives ~24h of replay; tune up for verbose feeds, down for memory. + EVENTS_STREAM_MAXLEN: int = 1000 + # SSE keepalive comment cadence. Must sit under Cloudflare's 100s idle + # close and iOS Safari's ~60s — 15s gives generous headroom. + SSE_KEEPALIVE_SECONDS: int = 15 + # Cap on simultaneous SSE connections per user. Each connection holds + # one WSGI thread (32 per gunicorn worker) and one Redis pub/sub + # connection. 8 covers normal multi-tab use without letting one user + # starve the pool. Set to 0 to disable the cap. + SSE_MAX_CONCURRENT_PER_USER: int = 8 + # Per-request cap on the number of backlog entries XRANGE returns + # for ``/api/events`` snapshots. Bounds the bytes a single replay + # can move from Redis to the wire — a malicious client looping + # ``Last-Event-ID=`` reconnects can only enumerate this + # many entries per round-trip. Combined with the per-user + # connection cap above and the windowed budget below, total + # enumeration throughput is bounded. + EVENTS_REPLAY_MAX_PER_REQUEST: int = 200 + # Sliding-window cap on snapshot replays per user. Once the budget + # is exhausted the route returns HTTP 429 with the cursor pinned; + # the client backs off and retries after the window rolls over. + EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW: int = 30 + EVENTS_REPLAY_BUDGET_WINDOW_SECONDS: int = 60 + + # Retention for the ``message_events`` journal. The ``cleanup_message_events`` + # beat task deletes rows older than this. Reconnect-replay only + # needs the journal for streams a client could still be tailing, + # so 14 days is a generous default that covers paused/tool-action + # flows without unbounded table growth. + MESSAGE_EVENTS_RETENTION_DAYS: int = 14 + @field_validator("POSTGRES_URI", mode="before") @classmethod def _normalize_postgres_uri_validator(cls, v): diff --git a/application/events/__init__.py b/application/events/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/application/events/keys.py b/application/events/keys.py new file mode 100644 index 00000000..d72777eb --- /dev/null +++ b/application/events/keys.py @@ -0,0 +1,52 @@ +"""Stream/topic key derivations shared by publisher and SSE consumer. + +Single source of truth for the per-user Redis Streams key and pub/sub +topic name. Both must agree exactly — a typo here splits the +publisher's writes from the consumer's reads. +""" + +from __future__ import annotations + + +def stream_key(user_id: str) -> str: + """Redis Streams key holding the durable backlog for ``user_id``.""" + return f"user:{user_id}:stream" + + +def topic_name(user_id: str) -> str: + """Redis pub/sub channel used for live fan-out to ``user_id``.""" + return f"user:{user_id}" + + +def connection_counter_key(user_id: str) -> str: + """Redis counter tracking active SSE connections for ``user_id``.""" + return f"user:{user_id}:sse_count" + + +def replay_budget_key(user_id: str) -> str: + """Redis counter tracking snapshot replays for ``user_id`` in the + rolling rate-limit window.""" + return f"user:{user_id}:replay_count" + + +def stream_id_compare(a: str, b: str) -> int: + """Compare two Redis Streams ids. Returns -1, 0, 1 like ``cmp``. + + Stream ids are ``ms-seq`` strings; comparing as strings would be wrong + once ``ms`` straddles digit-count boundaries. We parse and compare + as ``(int, int)`` tuples. + + Raises ``ValueError`` on malformed input. Callers must pre-validate + against ``_STREAM_ID_RE`` (or equivalent) — a lex fallback here let + a malformed id compare lex-greater than a real one and silently pin + dedup forever. + """ + a_ms, _, a_seq = a.partition("-") + b_ms, _, b_seq = b.partition("-") + a_tuple = (int(a_ms), int(a_seq) if a_seq else 0) + b_tuple = (int(b_ms), int(b_seq) if b_seq else 0) + if a_tuple < b_tuple: + return -1 + if a_tuple > b_tuple: + return 1 + return 0 diff --git a/application/events/publisher.py b/application/events/publisher.py new file mode 100644 index 00000000..df806751 --- /dev/null +++ b/application/events/publisher.py @@ -0,0 +1,144 @@ +"""User-scoped event publisher: durable backlog + live fan-out. + +Each ``publish_user_event`` call writes twice: + +1. ``XADD user:{user_id}:stream MAXLEN ~ * event `` — the + durable backlog used by SSE reconnect (``Last-Event-ID``) and stream + replay. Bounded by ``EVENTS_STREAM_MAXLEN`` (~24h at typical event + rates) so the per-user footprint stays predictable. +2. ``PUBLISH user:{user_id} `` — live fan-out to every + currently connected SSE generator for the user, across instances. + +Together they give a snapshot-plus-tail story: a reconnecting client +reads ``XRANGE`` from its last seen id and then transitions onto the +live pub/sub. The Redis Streams entry id (e.g. ``1735682400000-0``) is +the canonical, monotonically increasing event id and is what +``Last-Event-ID`` carries. + +Failures are logged and swallowed: the caller is typically a Celery +task whose primary work has already succeeded, and a notification +delivery miss should not surface as a task failure. +""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from typing import Any, Optional + +from application.cache import get_redis_instance +from application.core.settings import settings +from application.events.keys import stream_key, topic_name +from application.streaming.broadcast_channel import Topic + +logger = logging.getLogger(__name__) + + +def _iso_now() -> str: + """ISO 8601 UTC with millisecond precision and Z suffix.""" + return ( + datetime.now(timezone.utc) + .isoformat(timespec="milliseconds") + .replace("+00:00", "Z") + ) + + +def publish_user_event( + user_id: str, + event_type: str, + payload: dict[str, Any], + *, + scope: Optional[dict[str, Any]] = None, +) -> Optional[str]: + """Publish a user-scoped event; return the Redis Streams id or ``None``. + + Fire-and-forget: never raises. ``None`` means the event reached + neither the journal nor live subscribers (see runbook for causes). + """ + if not user_id or not event_type: + logger.warning( + "publish_user_event called without user_id or event_type " + "(user_id=%r, event_type=%r)", + user_id, + event_type, + ) + return None + if not settings.ENABLE_SSE_PUSH: + return None + + envelope_partial: dict[str, Any] = { + "type": event_type, + "ts": _iso_now(), + "user_id": user_id, + "topic": topic_name(user_id), + "scope": scope or {}, + "payload": payload, + } + + try: + envelope_partial_json = json.dumps(envelope_partial) + except (TypeError, ValueError) as exc: + logger.warning( + "publish_user_event payload not JSON-serializable: " + "user=%s type=%s err=%s", + user_id, + event_type, + exc, + ) + return None + + redis = get_redis_instance() + if redis is None: + logger.debug("Redis unavailable; skipping publish_user_event") + return None + + maxlen = settings.EVENTS_STREAM_MAXLEN + stream_id: Optional[str] = None + try: + # Auto-id ('*') gives a monotonic ms-seq id that doubles as the + # SSE event id. ``approximate=True`` lets Redis trim in chunks + # for performance; the cap is treated as ~MAXLEN, never <. + result = redis.xadd( + stream_key(user_id), + {"event": envelope_partial_json}, + maxlen=maxlen, + approximate=True, + ) + stream_id = ( + result.decode("utf-8") + if isinstance(result, (bytes, bytearray)) + else str(result) + ) + except Exception: + logger.exception( + "xadd failed for user=%s event_type=%s", user_id, event_type + ) + + # If the durable journal write failed there is no canonical id to + # ship — publishing the envelope live would put an id-less record + # on the wire that bypasses the SSE route's dedup floor and breaks + # ``Last-Event-ID`` semantics for any reconnect. Best-effort + # delivery means dropping consistently, not delivering inconsistent + # state. + if stream_id is None: + return None + + envelope = dict(envelope_partial) + envelope["id"] = stream_id + + try: + Topic(topic_name(user_id)).publish(json.dumps(envelope)) + except Exception: + logger.exception( + "publish failed for user=%s event_type=%s", user_id, event_type + ) + + logger.debug( + "event.published topic=%s type=%s id=%s", + topic_name(user_id), + event_type, + stream_id, + ) + + return stream_id diff --git a/application/parser/embedding_pipeline.py b/application/parser/embedding_pipeline.py index d41d2044..2955eb73 100755 --- a/application/parser/embedding_pipeline.py +++ b/application/parser/embedding_pipeline.py @@ -4,6 +4,7 @@ from typing import Any, List, Optional from retry import retry from tqdm import tqdm from application.core.settings import settings +from application.events.publisher import publish_user_event from application.storage.db.repositories.ingest_chunk_progress import ( IngestChunkProgressRepository, ) @@ -152,6 +153,7 @@ def embed_and_store_documents( task_status: Any, *, attempt_id: Optional[str] = None, + user_id: Optional[str] = None, ) -> None: """Embeds documents and stores them in a vector store. @@ -170,6 +172,10 @@ def embed_and_store_documents( attempt_id: Stable id of the current task invocation, typically ``self.request.id`` from the Celery task body. ``None`` is treated as a fresh attempt every time. + user_id: When provided, per-percent SSE progress events are + published to ``user:{user_id}`` for the in-app upload toast. + ``None`` is the safe default — workers without a user + context (e.g. background syncs) skip the publish. Returns: None @@ -249,6 +255,8 @@ def embed_and_store_documents( # Process and embed documents chunk_error: Exception | None = None failed_idx: int | None = None + last_published_pct = -1 + source_id_str = str(source_id) for idx in tqdm( range(loop_start, total_docs), desc="Embedding 🦖", @@ -262,6 +270,24 @@ def embed_and_store_documents( progress = int(((idx + 1) / total_docs) * 100) task_status.update_state(state="PROGRESS", meta={"current": progress}) + # SSE push for sub-second upload-toast updates. Throttled to one + # event per percent so a 10k-chunk ingest emits ~100 events, + # not 10k. The Celery update_state above stays the source of + # truth for the polling-fallback path. + if user_id and progress > last_published_pct: + publish_user_event( + user_id, + "source.ingest.progress", + { + "current": progress, + "total": total_docs, + "embedded_chunks": idx + 1, + "stage": "embedding", + }, + scope={"kind": "source", "id": source_id_str}, + ) + last_published_pct = progress + # Add document to vector store add_text_to_store_with_retry(store, doc, source_id) _record_progress(source_id, last_index=idx, embedded_chunks=idx + 1) diff --git a/application/storage/db/models.py b/application/storage/db/models.py index c6667d02..2e840634 100644 --- a/application/storage/db/models.py +++ b/application/storage/db/models.py @@ -34,7 +34,7 @@ from sqlalchemy.dialects.postgresql import ARRAY, CITEXT, JSONB, UUID metadata = MetaData() -# --- Phase 1, Tier 1 -------------------------------------------------------- +# --- Users, prompts, tools, logs -------------------------------------------- users_table = Table( "users", @@ -138,7 +138,7 @@ app_metadata_table = Table( ) -# --- Phase 2, Tier 2 -------------------------------------------------------- +# --- Agents, sources, attachments, artifacts -------------------------------- agent_folders_table = Table( "agent_folders", @@ -307,7 +307,7 @@ connector_sessions_table = Table( ) -# --- Phase 3, Tier 3 -------------------------------------------------------- +# --- Conversations, messages, workflows ------------------------------------- conversations_table = Table( "conversations", @@ -363,6 +363,36 @@ conversation_messages_table = Table( UniqueConstraint("conversation_id", "position", name="conversation_messages_conv_pos_uidx"), ) +# Per-yield journal of chat-stream events, used by the snapshot+tail +# reconnect: the route's GET reconnect endpoint reads +# ``WHERE message_id = ? AND sequence_no > ?`` from this table before +# tailing the live ``channel:{message_id}`` pub/sub. See +# ``application/streaming/event_replay.py`` and migration 0007. +message_events_table = Table( + "message_events", + metadata, + # PK is the composite ``(message_id, sequence_no)`` — it doubles as + # the snapshot read index (covering range scan on + # ``WHERE message_id = ? AND sequence_no > ?``). + Column( + "message_id", + UUID(as_uuid=True), + ForeignKey("conversation_messages.id", ondelete="CASCADE"), + primary_key=True, + nullable=False, + ), + # Strictly monotonic per ``message_id``. Allocated by the route as it + # yields, so the writer is single-threaded for the lifetime of one + # stream — no contention, no SERIAL needed. + Column("sequence_no", Integer, primary_key=True, nullable=False), + Column("event_type", Text, nullable=False), + Column("payload", JSONB, nullable=False, server_default="{}"), + Column( + "created_at", DateTime(timezone=True), nullable=False, server_default=func.now() + ), +) + + shared_conversations_table = Table( "shared_conversations", metadata, @@ -403,7 +433,7 @@ pending_tool_state_table = Table( ) -# --- Tier 1 durability foundation (migration 0004) -------------------------- +# --- Durability foundation (idempotency / journals, migration 0004) --------- # CHECK constraints (status enums) and partial indexes are intentionally # omitted from these declarations — the DB is the authority. Repositories # use raw ``text(...)`` SQL against these tables, not the Core objects. diff --git a/application/storage/db/repositories/conversations.py b/application/storage/db/repositories/conversations.py index a21acbd9..b1d3890d 100644 --- a/application/storage/db/repositories/conversations.py +++ b/application/storage/db/repositories/conversations.py @@ -15,6 +15,7 @@ Covers every operation the legacy Mongo code performs on from __future__ import annotations import json +from enum import Enum from typing import Optional from sqlalchemy import Connection, text @@ -25,6 +26,22 @@ from application.storage.db.models import conversations_table, conversation_mess from application.storage.db.serialization import PGNativeJSONEncoder +class MessageUpdateOutcome(str, Enum): + """Discriminated result of ``update_message_by_id``. + + Distinguishes the row-actually-updated case from the row-already-at- + the-requested-terminal-state case so an abort handler can journal + ``end`` instead of ``error`` when the normal-path finalize already + flipped the row to ``complete``. + """ + + UPDATED = "updated" + ALREADY_COMPLETE = "already_complete" + ALREADY_FAILED = "already_failed" + NOT_FOUND = "not_found" + INVALID = "invalid" + + def _message_row_to_dict(row) -> dict: """Like ``row_to_dict`` but renames the DB column ``message_metadata`` back to the public API key ``metadata`` so callers keep the Mongo-era @@ -58,8 +75,8 @@ class ConversationsRepository: - Already-UUID-shaped → returned as-is. - Otherwise treated as a Mongo ObjectId and looked up via ``agents.legacy_mongo_id``. Returns ``None`` if no PG row - exists yet (e.g. the agent was created before Phase 1 - backfill). + exists yet (e.g. the agent was created before the backfill + ran). """ if not agent_id_raw: return None @@ -697,7 +714,7 @@ class ConversationsRepository: def update_message_by_id( self, message_id: str, fields: dict, *, only_if_non_terminal: bool = False, - ) -> bool: + ) -> MessageUpdateOutcome: """Update specific fields on a message identified by its UUID. ``metadata`` is merged into the existing JSONB rather than @@ -705,9 +722,13 @@ class ConversationsRepository: a successful late finalize. When ``only_if_non_terminal`` is True, the update is gated so a late finalize cannot retract a reconciler-set ``failed`` (or a prior ``complete``). + + The return value discriminates "I updated the row" from "the + row was already at a terminal state" so the abort handler can + journal ``end`` when the normal-path finalize already ran. """ if not looks_like_uuid(message_id): - return False + return MessageUpdateOutcome.INVALID allowed = { "prompt", "response", "thought", "sources", "tool_calls", "attachments", "model_id", "metadata", "timestamp", "status", @@ -715,7 +736,7 @@ class ConversationsRepository: } filtered = {k: v for k, v in fields.items() if k in allowed} if not filtered: - return False + return MessageUpdateOutcome.INVALID api_to_col = {"metadata": "message_metadata"} @@ -752,15 +773,44 @@ class ConversationsRepository: params[col] = val set_parts.append("updated_at = now()") - where_clauses = ["id = CAST(:id AS uuid)"] + update_where = ["id = CAST(:id AS uuid)"] if only_if_non_terminal: - where_clauses.append("status NOT IN ('complete', 'failed')") + update_where.append("status NOT IN ('complete', 'failed')") + # Single-statement attempt + prior-status probe. Both CTEs see + # the same MVCC snapshot, so ``prior.status`` reflects the row + # state before the UPDATE — exactly what we need to tell + # ``ALREADY_COMPLETE`` apart from ``ALREADY_FAILED`` apart from + # ``NOT_FOUND`` without a follow-up SELECT. sql = ( - f"UPDATE conversation_messages SET {', '.join(set_parts)} " - f"WHERE {' AND '.join(where_clauses)}" + "WITH attempted AS (" + f" UPDATE conversation_messages SET {', '.join(set_parts)} " + f" WHERE {' AND '.join(update_where)} " + " RETURNING 1 AS updated" + "), " + "prior AS (" + " SELECT status FROM conversation_messages " + " WHERE id = CAST(:id AS uuid)" + ") " + "SELECT (SELECT updated FROM attempted) AS updated, " + " (SELECT status FROM prior) AS prior_status" ) - result = self._conn.execute(text(sql), params) - return result.rowcount > 0 + row = self._conn.execute(text(sql), params).fetchone() + if row is None: + return MessageUpdateOutcome.NOT_FOUND + updated, prior_status = row[0], row[1] + if updated: + return MessageUpdateOutcome.UPDATED + if prior_status is None: + return MessageUpdateOutcome.NOT_FOUND + if prior_status == "complete": + return MessageUpdateOutcome.ALREADY_COMPLETE + if prior_status == "failed": + return MessageUpdateOutcome.ALREADY_FAILED + # ``only_if_non_terminal=False`` always updates an existing row, + # so reaching here means the gate excluded it for some status + # the terminal set doesn't cover — treat as "not found" rather + # than inventing a new variant. + return MessageUpdateOutcome.NOT_FOUND def update_message_status( self, message_id: str, status: str, diff --git a/application/storage/db/repositories/message_events.py b/application/storage/db/repositories/message_events.py new file mode 100644 index 00000000..553efd01 --- /dev/null +++ b/application/storage/db/repositories/message_events.py @@ -0,0 +1,248 @@ +"""Repository for ``message_events`` — the chat-stream snapshot journal. + +``record`` / ``bulk_record`` write per-yield events; ``read_after`` +replays rows past a cursor for reconnect snapshots. Composite PK +``(message_id, sequence_no)`` raises ``IntegrityError`` on duplicates. +Callers must use short-lived per-call transactions — long-lived +transactions hide writes from reconnecting clients on a separate +connection and turn one bad row into ``InFailedSqlTransaction``. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any, Optional + +from sqlalchemy import Connection, text + +from application.storage.db.base_repository import row_to_dict + +logger = logging.getLogger(__name__) + + +class MessageEventsRepository: + """Read/write helpers for ``message_events``.""" + + def __init__(self, conn: Connection) -> None: + self._conn = conn + + def record( + self, + message_id: str, + sequence_no: int, + event_type: str, + payload: Optional[Any] = None, + ) -> None: + """Append a single event to the journal. + + At this raw repo layer ``payload`` is preserved as-is when not + ``None`` (lists, scalars, and dicts all round-trip via JSONB); + ``None`` substitutes an empty object so the column's NOT NULL + invariant holds. The streaming-route wrapper + ``application/streaming/message_journal.py::record_event`` + tightens this contract to dicts only — the live and replay + paths reconstruct non-dict payloads differently, so the wrapper + rejects them at the gate. Direct callers of this repo method + (cleanup tasks, tests, future ad-hoc consumers) keep the wider + JSONB-compatible surface. + + Raises ``sqlalchemy.exc.IntegrityError`` on duplicate + ``(message_id, sequence_no)`` and ``DataError`` on a malformed + ``message_id`` UUID. Both abort the surrounding transaction — + callers must run inside a short-lived per-event session + (see module docstring). + """ + if not event_type: + raise ValueError("event_type must be a non-empty string") + materialised_payload = payload if payload is not None else {} + self._conn.execute( + text( + """ + INSERT INTO message_events ( + message_id, sequence_no, event_type, payload + ) VALUES ( + CAST(:message_id AS uuid), :sequence_no, :event_type, + CAST(:payload AS jsonb) + ) + """ + ), + { + "message_id": str(message_id), + "sequence_no": int(sequence_no), + "event_type": event_type, + "payload": json.dumps(materialised_payload), + }, + ) + + def bulk_record( + self, + message_id: str, + events: list[tuple[int, str, dict]], + ) -> None: + """Append multiple events for ``message_id`` in one INSERT. + + ``events`` is a list of ``(sequence_no, event_type, payload)`` + tuples. SQLAlchemy ``executemany`` issues one bulk INSERT; + Postgres treats the whole batch as one statement, so an + IntegrityError on any row aborts the entire batch. + + Caller contract: on IntegrityError, do NOT retry this method + with the same batch — fall back to per-row ``record()`` calls + (each in its own short-lived session) so a single colliding + seq doesn't drop the rest of the batch. ``BatchedJournalWriter`` + in ``application/streaming/message_journal.py`` is the canonical + consumer. + """ + if not events: + return + params = [ + { + "message_id": str(message_id), + "sequence_no": int(seq), + "event_type": event_type, + "payload": json.dumps(payload if payload is not None else {}), + } + for seq, event_type, payload in events + ] + self._conn.execute( + text( + """ + INSERT INTO message_events ( + message_id, sequence_no, event_type, payload + ) VALUES ( + CAST(:message_id AS uuid), :sequence_no, :event_type, + CAST(:payload AS jsonb) + ) + """ + ), + params, + ) + + def read_after( + self, + message_id: str, + last_sequence_no: Optional[int] = None, + ) -> list[dict]: + """Return events with ``sequence_no > last_sequence_no``. + + ``last_sequence_no=None`` returns the full backlog. Rows are + returned in ascending ``sequence_no`` order. The composite PK + is the snapshot read index for this scan — Postgres typically + picks an in-order index range scan, though for highly mixed + data the planner may pick a bitmap+sort. Either way the result + is sorted on ``sequence_no``. + + Returns a ``list`` (not a generator) so the underlying + ``Result`` is fully drained before the caller can issue + another query on the same connection. + """ + cursor = -1 if last_sequence_no is None else int(last_sequence_no) + rows = self._conn.execute( + text( + """ + SELECT message_id, sequence_no, event_type, payload, created_at + FROM message_events + WHERE message_id = CAST(:message_id AS uuid) + AND sequence_no > :cursor + ORDER BY sequence_no ASC + """ + ), + {"message_id": str(message_id), "cursor": cursor}, + ).fetchall() + return [row_to_dict(row) for row in rows] + + def cleanup_older_than(self, ttl_days: int) -> int: + """Delete journal rows older than ``ttl_days``. Returns row count. + + Reconnect-replay is meaningful only for streams the client + could plausibly still be waiting on, so old rows are dead + weight. The ``message_events_created_at_idx`` btree makes the + range delete a cheap index scan even on large tables. + """ + if ttl_days <= 0: + raise ValueError("ttl_days must be positive") + result = self._conn.execute( + text( + """ + DELETE FROM message_events + WHERE created_at < now() - make_interval(days => :ttl_days) + """ + ), + {"ttl_days": int(ttl_days)}, + ) + return int(result.rowcount or 0) + + def reconstruct_partial(self, message_id: str) -> dict: + """Rebuild partial response/thought/sources/tool_calls from journal events. + + ``answer``/``thought`` chunks concat in seq order; ``source``/ + ``tool_calls`` carry the full list at emit time (last-wins). + """ + rows = self._conn.execute( + text( + """ + SELECT sequence_no, event_type, payload + FROM message_events + WHERE message_id = CAST(:message_id AS uuid) + ORDER BY sequence_no ASC + """ + ), + {"message_id": str(message_id)}, + ).fetchall() + + response_parts: list[str] = [] + thought_parts: list[str] = [] + sources: list = [] + tool_calls: list = [] + + for row in rows: + payload = row.payload + if not isinstance(payload, dict): + continue + etype = row.event_type + if etype == "answer": + chunk = payload.get("answer") + if isinstance(chunk, str): + response_parts.append(chunk) + elif etype == "thought": + chunk = payload.get("thought") + if isinstance(chunk, str): + thought_parts.append(chunk) + elif etype == "source": + src = payload.get("source") + if isinstance(src, list): + sources = src + elif etype == "tool_calls": + tcs = payload.get("tool_calls") + if isinstance(tcs, list): + tool_calls = tcs + + return { + "response": "".join(response_parts), + "thought": "".join(thought_parts), + "sources": sources, + "tool_calls": tool_calls, + } + + def latest_sequence_no(self, message_id: str) -> Optional[int]: + """Largest ``sequence_no`` recorded for ``message_id``, or ``None``. + + Used by the route to seed the per-stream allocator on retry / + process restart so a re-run continues numbering instead of + trampling earlier entries with duplicate sequence_no. + """ + # ``MAX`` always returns one row — NULL when the journal is + # empty — so we test the value, not the row presence. + row = self._conn.execute( + text( + """ + SELECT MAX(sequence_no) AS s + FROM message_events + WHERE message_id = CAST(:message_id AS uuid) + """ + ), + {"message_id": str(message_id)}, + ).first() + value = row[0] if row is not None else None + return int(value) if value is not None else None diff --git a/application/storage/db/source_ids.py b/application/storage/db/source_ids.py new file mode 100644 index 00000000..dc0a8ba8 --- /dev/null +++ b/application/storage/db/source_ids.py @@ -0,0 +1,23 @@ +"""Deterministic source-id derivation for idempotent ingest. + +DO NOT CHANGE the pinned UUID namespace — it backs cross-deploy +idempotency keys. +""" + +from __future__ import annotations + +import uuid + +# DO NOT CHANGE. See module docstring. +DOCSGPT_INGEST_NAMESPACE = uuid.UUID("fa25d5d1-398b-46df-ac89-8d1c360b9bea") + + +def derive_source_id(idempotency_key) -> uuid.UUID: + """``uuid5(NS, key)`` when a key is supplied; ``uuid4()`` otherwise. + + A non-string / empty key falls back to ``uuid4()`` so the caller + always gets a fresh id rather than a TypeError mid-route. + """ + if isinstance(idempotency_key, str) and idempotency_key: + return uuid.uuid5(DOCSGPT_INGEST_NAMESPACE, idempotency_key) + return uuid.uuid4() diff --git a/application/streaming/__init__.py b/application/streaming/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/application/streaming/broadcast_channel.py b/application/streaming/broadcast_channel.py new file mode 100644 index 00000000..07861d5d --- /dev/null +++ b/application/streaming/broadcast_channel.py @@ -0,0 +1,126 @@ +"""Redis pub/sub Topic abstraction for SSE fan-out. + +A Topic is a named channel for one-shot live event delivery. Canonical uses: + +- ``user:{user_id}`` for per-user notifications +- ``channel:{message_id}`` for per-chat-message streams + +Subscription is race-free via ``on_subscribe``: the callback fires only +after Redis acknowledges ``SUBSCRIBE``, so a publisher dispatched inside +the callback cannot lose its first event to a not-yet-registered +subscriber. + +The subscribe iterator yields ``None`` on poll timeout so the caller can +emit SSE keepalive comments without spawning a separate timer thread. +""" + +from __future__ import annotations + +import logging +from typing import Callable, Iterator, Optional + +from application.cache import get_redis_instance + +logger = logging.getLogger(__name__) + + +class Topic: + """A pub/sub channel identified by a string name.""" + + def __init__(self, name: str) -> None: + self.name = name + + def publish(self, payload: str | bytes) -> int: + """Fan out a payload to currently subscribed clients. + + Returns the number Redis reports as receiving the message (limited + to subscribers connected to *this* Redis instance), or 0 if Redis + is unavailable. Never raises. + """ + redis = get_redis_instance() + if redis is None: + logger.debug("Redis unavailable; dropping publish to %s", self.name) + return 0 + try: + return int(redis.publish(self.name, payload)) + except Exception: + logger.exception("Topic.publish failed for %s", self.name) + return 0 + + def subscribe( + self, + on_subscribe: Optional[Callable[[], None]] = None, + poll_timeout: float = 1.0, + ) -> Iterator[Optional[bytes]]: + """Subscribe to the topic; yield raw payloads or ``None`` on tick. + + Yields ``None`` every ``poll_timeout`` seconds while idle so the + caller can emit keepalive frames or check cancellation. Yields + ``bytes`` for each delivered message. + + ``on_subscribe`` runs synchronously after Redis acknowledges the + SUBSCRIBE — use it to seed any state (e.g. read backlog) that + must be ordered after the subscriber is live but before the + first pub/sub message is processed. + + If Redis is unavailable, returns immediately without yielding. + Cleanly unsubscribes on ``GeneratorExit`` (client disconnect). + """ + redis = get_redis_instance() + if redis is None: + logger.debug("Redis unavailable; subscribe to %s yielded nothing", self.name) + return + pubsub = None + on_subscribe_fired = False + try: + pubsub = redis.pubsub() + try: + pubsub.subscribe(self.name) + except Exception: + # Subscribe failure (transient Redis hiccup, conn reset, etc.) + # is treated like "Redis unavailable": yield nothing, let the + # caller fall back to its own resilience strategy. The finally + # block will still tear down the pubsub object cleanly. + logger.exception("pubsub.subscribe failed for %s", self.name) + return + while True: + try: + msg = pubsub.get_message(timeout=poll_timeout) + except Exception: + logger.exception("pubsub.get_message failed for %s", self.name) + return + if msg is None: + yield None + continue + msg_type = msg.get("type") + if msg_type == "subscribe": + if not on_subscribe_fired and on_subscribe is not None: + try: + on_subscribe() + except Exception: + logger.exception( + "on_subscribe callback failed for %s", self.name + ) + on_subscribe_fired = True + continue + if msg_type != "message": + continue + data = msg.get("data") + if data is None: + continue + yield data if isinstance(data, bytes) else str(data).encode("utf-8") + finally: + if pubsub is not None: + if on_subscribe_fired: + try: + pubsub.unsubscribe(self.name) + except Exception: + logger.debug( + "pubsub unsubscribe error for %s", + self.name, + exc_info=True, + ) + try: + pubsub.close() + except Exception: + logger.debug("pubsub close error for %s", self.name, exc_info=True) diff --git a/application/streaming/event_replay.py b/application/streaming/event_replay.py new file mode 100644 index 00000000..d2bbc402 --- /dev/null +++ b/application/streaming/event_replay.py @@ -0,0 +1,434 @@ +"""Snapshot+tail iterator for chat-stream reconnect-after-disconnect. + +Subscribe to ``channel:{message_id}``, snapshot ``message_events`` +rows past ``last_event_id`` inside the SUBSCRIBE-ack callback, flush +snapshot, then tail live pub/sub (dedup'd by ``sequence_no``). See +``docs/runbooks/sse-notifications.md``. +""" + +from __future__ import annotations + +import json +import logging +import re +import time +from typing import Iterator, Optional + +from sqlalchemy import text as sql_text + +from application.storage.db.repositories.message_events import ( + MessageEventsRepository, +) +from application.storage.db.session import db_readonly +from application.streaming.broadcast_channel import Topic +from application.streaming.keys import message_topic_name + +logger = logging.getLogger(__name__) + +DEFAULT_KEEPALIVE_SECONDS = 15.0 +DEFAULT_POLL_TIMEOUT_SECONDS = 1.0 +# When the live tail has no events and no terminal in snapshot, fall +# back to checking ``conversation_messages`` directly. If the row has +# already gone terminal (worker journaled ``end``/``error`` to the DB +# but the matching pub/sub publish was lost, or the row was finalized +# without a journal write at all) we surface a terminal event so the +# client doesn't hang on keepalives. If the row is still non-terminal +# but the producer heartbeat is older than ``PRODUCER_IDLE_SECONDS`` +# the producer is presumed dead (worker crash / recycle between chunks +# and finalize) and we emit a terminal ``error`` so the UI can recover. +DEFAULT_WATCHDOG_INTERVAL_SECONDS = 5.0 +# 1.5× the route's 60s heartbeat interval — long enough that a normal +# heartbeat skew doesn't false-positive, short enough that a stuck +# stream surfaces before the 5-minute reconciler sweep escalates. +DEFAULT_PRODUCER_IDLE_SECONDS = 90.0 + +# WHATWG SSE accepts CRLF, CR, LF — split on any of them so a stray CR +# can't smuggle a record boundary into the wire format. +_SSE_LINE_SPLIT_PATTERN = re.compile(r"\r\n|\r|\n") + +# Event types that mark the end of a chat answer. After delivering one +# we close the reconnect stream — keeping the connection open past a +# terminal event would leak both the client's reconnect promise and +# the server's WSGI thread waiting on keepalives that the user no +# longer cares about. The agent loop emits ``end`` for normal / +# tool-paused completion and ``error`` for the catch-all failure path +# (which doesn't get a trailing ``end``). +_TERMINAL_EVENT_TYPES = frozenset({"end", "error"}) + + +def _payload_is_terminal( + payload: object, event_type: Optional[str] = None +) -> bool: + """True if ``payload['type']`` or ``event_type`` is a terminal sentinel.""" + if isinstance(payload, dict) and payload.get("type") in _TERMINAL_EVENT_TYPES: + return True + return event_type in _TERMINAL_EVENT_TYPES + + +def format_sse_event(payload: dict, sequence_no: int) -> str: + """Encode a journal event as one ``id:``/``data:`` SSE record. + + The body is the payload's JSON serialisation. ``complete_stream`` + payloads are flat JSON dicts with no embedded newlines, so a + single ``data:`` line is sufficient — but we still split on any + line terminator in case a future caller passes a multi-line string. + """ + body = json.dumps(payload) + lines = [f"id: {sequence_no}"] + for line in _SSE_LINE_SPLIT_PATTERN.split(body): + lines.append(f"data: {line}") + return "\n".join(lines) + "\n\n" + + +def _check_producer_liveness( + message_id: str, idle_seconds: float +) -> Optional[dict]: + """Inspect ``conversation_messages`` and return a terminal SSE + payload when the producer is no longer alive, else ``None``. + + Three terminal cases collapse into a single DB round-trip: + + - ``status='complete'`` — the live finalize ran but its journal + terminal write didn't reach us (or never happened). Synthesise + ``end`` so the client closes cleanly on the row's user-visible + state. + - ``status='failed'`` — same, but for the failure path. Carry the + stashed ``error`` from ``message_metadata`` so the UI shows the + real reason. + - non-terminal status and ``last_heartbeat_at`` (or ``timestamp``) + older than ``idle_seconds`` — the producing worker is gone. + Synthesise ``error`` so the client doesn't hang on keepalives + until the proxy idle-timeout kicks in. + """ + try: + with db_readonly() as conn: + row = conn.execute( + sql_text( + """ + SELECT + status, + message_metadata->>'error' AS err, + GREATEST( + timestamp, + COALESCE( + (message_metadata->>'last_heartbeat_at') + ::timestamptz, + timestamp + ) + ) < now() - make_interval(secs => :idle_secs) + AS is_stale + FROM conversation_messages + WHERE id = CAST(:id AS uuid) + """ + ), + {"id": message_id, "idle_secs": float(idle_seconds)}, + ).first() + except Exception: + logger.exception( + "Watchdog liveness check failed for message_id=%s", message_id + ) + return None + + if row is None: + # Row deleted out from under us — treat as terminal so the + # client doesn't keep tailing a message that no longer exists. + return { + "type": "error", + "error": "Message no longer exists; please refresh.", + "code": "message_missing", + "message_id": message_id, + } + + status, err, is_stale = row[0], row[1], bool(row[2]) + if status == "complete": + return {"type": "end"} + if status == "failed": + return { + "type": "error", + "error": err or "Stream failed; please try again.", + "code": "producer_failed", + "message_id": message_id, + } + if is_stale: + return { + "type": "error", + "error": ( + "Stream producer is no longer responding; please try again." + ), + "code": "producer_stale", + "message_id": message_id, + } + return None + + +def build_message_event_stream( + message_id: str, + last_event_id: Optional[int] = None, + *, + keepalive_seconds: float = DEFAULT_KEEPALIVE_SECONDS, + poll_timeout_seconds: float = DEFAULT_POLL_TIMEOUT_SECONDS, + watchdog_interval_seconds: float = DEFAULT_WATCHDOG_INTERVAL_SECONDS, + producer_idle_seconds: float = DEFAULT_PRODUCER_IDLE_SECONDS, +) -> Iterator[str]: + """Yield SSE-formatted lines for one ``message_id`` reconnect stream. + + First frame is ``: connected``; subsequent frames are snapshot rows, + live-tail events, or ``: keepalive`` comments. Runs until the client + disconnects. + """ + yield ": connected\n\n" + + # Replay buffer — populated inside ``_on_subscribe`` (or the + # Redis-unavailable fallback below), drained on the first iteration + # of the subscribe loop after the callback runs. + replay_buffer: list[str] = [] + # Dedup floor: seeded with the client's cursor so an empty snapshot + # still rejects re-published live events with seq <= last_event_id. + # Advanced by snapshot rows AND by yielded live events, so any + # republish past the snapshot ceiling is also dropped. + max_replayed_seq: Optional[int] = last_event_id + replay_done = False + replay_failed = False + # Set when a snapshot row carries a terminal ``end`` / ``error`` + # event. After flushing the buffer the generator returns; if we + # kept tailing we'd loop on keepalives forever for a stream that + # already finished. + terminal_in_snapshot = False + + def _read_snapshot_into_buffer() -> None: + nonlocal max_replayed_seq, replay_failed, terminal_in_snapshot + try: + with db_readonly() as conn: + rows = MessageEventsRepository(conn).read_after( + message_id, last_sequence_no=last_event_id + ) + for row in rows: + seq = int(row["sequence_no"]) + payload = row.get("payload") + if not isinstance(payload, dict): + # ``record_event`` rejects non-dict payloads at the + # write gate, so this can only be a legacy row from + # before that contract or a direct SQL insert. The + # original synthetic fallback (``{"type": event_type}``) + # used to ship a malformed envelope here — drop the + # row instead so a corrupt journal entry doesn't + # poison a reconnect. + logger.warning( + "Skipping non-dict payload from message_events: " + "message_id=%s seq=%s type=%s", + message_id, + seq, + row.get("event_type"), + ) + continue + replay_buffer.append(format_sse_event(payload, seq)) + if max_replayed_seq is None or seq > max_replayed_seq: + max_replayed_seq = seq + if _payload_is_terminal(payload, row.get("event_type")): + terminal_in_snapshot = True + except Exception: + logger.exception( + "Snapshot read failed for message_id=%s last_event_id=%s", + message_id, + last_event_id, + ) + replay_failed = True + + def _on_subscribe() -> None: + # SUBSCRIBE has been acked — Postgres reads from this point + # capture every row that's been committed. Pub/sub messages + # published after this point are queued at the connection level + # until the outer loop calls ``get_message`` again. + nonlocal replay_done + try: + _read_snapshot_into_buffer() + finally: + # Flip even on failure so the outer loop continues to live + # tail and the client doesn't hang waiting for a snapshot + # flush that will never come. + replay_done = True + + topic = Topic(message_topic_name(message_id)) + last_keepalive = time.monotonic() + # Rate-limit the watchdog's DB hit. ``-inf`` makes the first idle + # tick after replay_done fire immediately so a snapshot-already- + # terminal-in-DB case is surfaced before any keepalive cadence. + # Subsequent checks are gated by ``watchdog_interval_seconds``. + last_watchdog_check = float("-inf") + # Synthetic terminal events emitted by the watchdog use the same + # ``sequence_no=-1`` convention as the snapshot-failure path so the + # frontend's strict ``\d+`` cursor regex rejects them as a + # ``Last-Event-ID`` for any future reconnect. The chosen + # discriminator ensures a manual page refresh after a watchdog-fired + # error doesn't loop on the same synthetic id. + watchdog_synthetic_seq = -1 + + try: + for payload in topic.subscribe( + on_subscribe=_on_subscribe, + poll_timeout=poll_timeout_seconds, + ): + # Flush snapshot exactly once after the SUBSCRIBE callback + # has run and produced a buffer. + if replay_done and replay_buffer: + for line in replay_buffer: + yield line + replay_buffer.clear() + if terminal_in_snapshot: + # The original stream already finished; tailing + # would just emit keepalives forever and pin both a + # client reconnect promise and a server WSGI thread. + return + + if replay_failed: + # Snapshot read failed (DB blip / transient timeout). Emit a + # terminal ``error`` event and return — the client only + # reconnects after the original stream has already moved on, + # so without a snapshot there's nothing live left to tail and + # holding the connection open would just emit keepalives + # until the proxy idle-timeout fires. ``code`` preserves the + # snapshot-vs-agent-loop distinction so a future client can + # opt into a refetch instead of a hard failure. + yield format_sse_event( + { + "type": "error", + "error": "Stream replay failed; please refresh to load the latest state.", + "code": "snapshot_failed", + "message_id": message_id, + }, + sequence_no=-1, + ) + return + + now = time.monotonic() + if payload is None: + # Idle tick — check both keepalive and watchdog. The + # watchdog only kicks in once the snapshot half has been + # flushed (``replay_done``) so we don't race the + # snapshot read on the first iteration. + if ( + replay_done + and watchdog_interval_seconds >= 0 + and now - last_watchdog_check >= watchdog_interval_seconds + ): + last_watchdog_check = now + terminal_payload = _check_producer_liveness( + message_id, producer_idle_seconds + ) + if terminal_payload is not None: + yield format_sse_event( + terminal_payload, + sequence_no=watchdog_synthetic_seq, + ) + return + if now - last_keepalive >= keepalive_seconds: + yield ": keepalive\n\n" + last_keepalive = now + continue + + envelope = _decode_pubsub_message(payload) + if envelope is None: + continue + seq = envelope.get("sequence_no") + inner = envelope.get("payload") + if ( + not isinstance(seq, int) + or isinstance(seq, bool) + or not isinstance(inner, dict) + ): + continue + if max_replayed_seq is not None and seq <= max_replayed_seq: + # Snapshot already covered this id — drop the duplicate. + continue + yield format_sse_event(inner, seq) + # Advance the dedup floor on the live path too, so a stale + # republish of an already-yielded seq (process restart, retry + # tool, etc.) is dropped on a later iteration. + max_replayed_seq = seq + last_keepalive = now + if _payload_is_terminal(inner, envelope.get("event_type")): + # Live tail just delivered the terminal event — close + # out the reconnect stream so the client's drain + # promise resolves and the WSGI thread is freed. + return + + # Subscribe exited without ever yielding (Redis unavailable, + # ``pubsub.subscribe`` raised, or the inner loop died between + # SUBSCRIBE-ack and the first poll). The snapshot half is in + # Postgres and is still serviceable — read it directly so a + # Redis-only outage doesn't cost the client their reconnect + # backlog. Gate the read on ``replay_done`` rather than + # ``subscribe_started``: if ``_on_subscribe`` already populated + # the buffer, re-reading would append the same rows twice and + # double the answer chunks on the client (the per-message + # reconnect dispatcher does not dedup by ``id``). + if not replay_done: + _read_snapshot_into_buffer() + replay_done = True + for line in replay_buffer: + yield line + replay_buffer.clear() + if replay_failed: + # Mirror the live-tail branch: emit a terminal ``error`` so + # the frontend's existing end/error handling drives the UI + # to a failed state instead of relying on the proxy timeout. + yield format_sse_event( + { + "type": "error", + "error": "Stream replay failed; please refresh to load the latest state.", + "code": "snapshot_failed", + "message_id": message_id, + }, + sequence_no=-1, + ) + return + # Same close-on-terminal contract as the live-tail branch. + # Without it a Redis-down + already-completed-stream client + # would also hang on a never-ending generator. + if terminal_in_snapshot: + return + except GeneratorExit: + # Client disconnect — let the underlying ``Topic.subscribe`` + # ``finally`` block tear down its pubsub cleanly. + return + + +def _decode_pubsub_message(raw) -> Optional[dict]: + """Parse a ``Topic.publish`` payload to ``{sequence_no, payload, ...}``. + + Returns ``None`` for malformed messages (drop silently — the + journal is still authoritative on reconnect). + """ + try: + if isinstance(raw, (bytes, bytearray)): + text_value = raw.decode("utf-8") + else: + text_value = str(raw) + envelope = json.loads(text_value) + except Exception: + return None + if not isinstance(envelope, dict): + return None + return envelope + + +def encode_pubsub_message( + message_id: str, + sequence_no: int, + event_type: str, + payload: dict, +) -> str: + """Build the JSON envelope used for ``channel:{message_id}`` publishes. + + Kept here (not in ``message_journal.py``) so the encode/decode pair + stays in one file — replay's ``_decode_pubsub_message`` and the + journal's publish must agree on the shape exactly. + """ + return json.dumps( + { + "message_id": str(message_id), + "sequence_no": int(sequence_no), + "event_type": event_type, + "payload": payload, + } + ) diff --git a/application/streaming/keys.py b/application/streaming/keys.py new file mode 100644 index 00000000..f75b4fec --- /dev/null +++ b/application/streaming/keys.py @@ -0,0 +1,19 @@ +"""Per-chat-message stream key derivations. + +Single source of truth for the Redis pub/sub topic name and any +auxiliary keys that the chat-stream snapshot+tail reconnect path +shares between the writer (``complete_stream`` + journal) and the +reader (``/api/messages//events`` reconnect endpoint). +""" + +from __future__ import annotations + + +def message_topic_name(message_id: str) -> str: + """Redis pub/sub channel for live fan-out of one chat message. + + Subscribers tail this topic for every event that ``complete_stream`` + yielded after the SUBSCRIBE-ack arrived; older events are recovered + from the ``message_events`` snapshot half of the pattern. + """ + return f"channel:{message_id}" diff --git a/application/streaming/message_journal.py b/application/streaming/message_journal.py new file mode 100644 index 00000000..e49f4486 --- /dev/null +++ b/application/streaming/message_journal.py @@ -0,0 +1,400 @@ +"""Per-yield journal write for the chat-stream snapshot+tail pattern. + +``record_event`` inserts into ``message_events`` and publishes to +``channel:{message_id}``. Both are best-effort; the INSERT commits +before the publish so a fast reconnect sees the row. See +``docs/runbooks/sse-notifications.md``. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any, Optional + +from sqlalchemy.exc import IntegrityError + +from application.storage.db.repositories.message_events import ( + MessageEventsRepository, +) +from application.storage.db.session import db_readonly, db_session +from application.streaming.broadcast_channel import Topic +from application.streaming.event_replay import encode_pubsub_message +from application.streaming.keys import message_topic_name + +logger = logging.getLogger(__name__) + + +# Tunables for ``BatchedJournalWriter``. A streaming answer emits ~100s +# of ``answer`` chunks per response; without batching, that's one PG +# transaction per yield in the WSGI thread. With these defaults, ~10x +# fewer commits at the cost of a ≤100ms reconnect-visibility lag for +# any event still sitting in the buffer. +DEFAULT_BATCH_SIZE = 16 +DEFAULT_BATCH_INTERVAL_MS = 100 + + +def _strip_null_bytes(value: Any) -> Any: + """Recursively strip ``\\x00`` from string keys/values in ``value``. + + Postgres JSONB rejects the NUL escape; an LLM emitting a stray NUL + in a chunk would otherwise raise ``DataError`` at INSERT and the row + would be lost from the journal (live stream proceeds, reconnect + snapshot misses the chunk). Mirrors the strip already done in + ``parser/embedding_pipeline.py`` and + ``api/user/attachments/routes.py``. + """ + if isinstance(value, str): + return value.replace("\x00", "") if "\x00" in value else value + if isinstance(value, dict): + return { + (k.replace("\x00", "") if isinstance(k, str) and "\x00" in k else k): + _strip_null_bytes(v) + for k, v in value.items() + } + if isinstance(value, list): + return [_strip_null_bytes(item) for item in value] + if isinstance(value, tuple): + return tuple(_strip_null_bytes(item) for item in value) + return value + + +def record_event( + message_id: str, + sequence_no: int, + event_type: str, + payload: Optional[dict[str, Any]] = None, +) -> bool: + """Journal one SSE event and publish it live. Best-effort. + + ``payload`` must be a ``dict`` or ``None`` (non-dicts are dropped so + live and replay envelopes stay byte-identical). Returns ``True`` when + the journal INSERT committed. Never raises. + """ + if not message_id or not event_type: + logger.warning( + "record_event called without message_id/event_type " + "(message_id=%r, event_type=%r)", + message_id, + event_type, + ) + return False + + if payload is None: + materialised_payload: dict[str, Any] = {} + elif isinstance(payload, dict): + materialised_payload = _strip_null_bytes(payload) + else: + logger.warning( + "record_event called with non-dict payload " + "(message_id=%s seq=%s type=%s payload_type=%s) — dropping", + message_id, + sequence_no, + event_type, + type(payload).__name__, + ) + return False + + journal_committed = False + # The seq we actually managed to write. Diverges from + # ``sequence_no`` only on the IntegrityError-retry path below. + materialised_seq = sequence_no + try: + # Short-lived per-event transaction. Critical for visibility: + # the reconnect endpoint reads the journal from a separate + # connection and only sees committed rows. + with db_session() as conn: + MessageEventsRepository(conn).record( + message_id, sequence_no, event_type, materialised_payload + ) + journal_committed = True + except IntegrityError: + # Composite-PK collision on (message_id, sequence_no). Most + # likely cause is a stale ``latest_sequence_no`` seed on a + # continuation retry — the route read MAX(seq) from a separate + # connection before another writer committed past it. Look up + # the live latest and retry once with latest+1 so the event is + # not silently lost. Bounded to a single retry — if two + # writers keep racing in lockstep the route-level retry will + # converge them across attempts. + try: + with db_readonly() as conn: + latest = MessageEventsRepository(conn).latest_sequence_no( + message_id + ) + materialised_seq = (latest if latest is not None else -1) + 1 + with db_session() as conn: + MessageEventsRepository(conn).record( + message_id, + materialised_seq, + event_type, + materialised_payload, + ) + journal_committed = True + logger.info( + "record_event: collision at seq=%s recovered → wrote at " + "seq=%s message_id=%s type=%s", + sequence_no, + materialised_seq, + message_id, + event_type, + ) + except IntegrityError: + # Second collision under the same retry — give up and log. + # The route's nonlocal counter will continue at + # ``sequence_no+1`` on the next emit; the next call may + # land cleanly past the contended window. + logger.warning( + "record_event: IntegrityError persists after seq+1 retry; " + "dropping. message_id=%s original_seq=%s retry_seq=%s " + "type=%s", + message_id, + sequence_no, + materialised_seq, + event_type, + ) + except Exception: + logger.exception( + "record_event: retry path failed unexpectedly " + "(message_id=%s seq=%s type=%s)", + message_id, + sequence_no, + event_type, + ) + except Exception: + logger.exception( + "message_events INSERT failed: message_id=%s seq=%s type=%s", + message_id, + sequence_no, + event_type, + ) + + try: + # Publish using ``materialised_seq`` so the live pubsub frame + # matches the journal row that other clients will snapshot on + # reconnect. The original POST stream's SSE ``id:`` still + # carries the caller's ``sequence_no`` — a reconnect from that + # client will receive the same event at ``materialised_seq`` + # on the snapshot, which is a benign duplicate (the slice's + # ``max_replayed_seq`` advances past it). No-collision case: + # ``materialised_seq == sequence_no`` and this is identical to + # the prior behaviour. + wire = encode_pubsub_message( + message_id, materialised_seq, event_type, materialised_payload + ) + Topic(message_topic_name(message_id)).publish(wire) + except Exception: + logger.exception( + "channel:%s publish failed: seq=%s type=%s", + message_id, + materialised_seq, + event_type, + ) + + return journal_committed + + +class BatchedJournalWriter: + """Per-stream journal writer that batches PG INSERTs. + + One writer per ``message_id``; ``record()`` buffers events and flushes + on size/time/``close()`` triggers. Pubsub publishes fire only after the + INSERT commits. On ``IntegrityError`` falls back to per-row writes. + """ + + def __init__( + self, + message_id: str, + *, + batch_size: int = DEFAULT_BATCH_SIZE, + batch_interval_ms: int = DEFAULT_BATCH_INTERVAL_MS, + ) -> None: + self._message_id = message_id + self._batch_size = batch_size + self._batch_interval_ms = batch_interval_ms + self._buffer: list[tuple[int, str, dict[str, Any]]] = [] + self._last_flush_mono_ms = time.monotonic() * 1000.0 + self._closed = False + + def record( + self, + sequence_no: int, + event_type: str, + payload: Optional[dict[str, Any]] = None, + ) -> bool: + """Buffer one event; maybe flush. Publish happens after journal commit.""" + if self._closed: + logger.warning( + "BatchedJournalWriter.record after close: " + "message_id=%s seq=%s type=%s", + self._message_id, + sequence_no, + event_type, + ) + return False + if not event_type: + logger.warning( + "BatchedJournalWriter.record without event_type: " + "message_id=%s seq=%s", + self._message_id, + sequence_no, + ) + return False + if payload is None: + materialised: dict[str, Any] = {} + elif isinstance(payload, dict): + materialised = _strip_null_bytes(payload) + else: + # Same contract as ``record_event`` — non-dict payloads + # are rejected so the live and replay paths can't diverge + # on envelope reconstruction. + logger.warning( + "BatchedJournalWriter.record with non-dict payload: " + "message_id=%s seq=%s type=%s payload_type=%s — dropping", + self._message_id, + sequence_no, + event_type, + type(payload).__name__, + ) + return False + + self._buffer.append((sequence_no, event_type, materialised)) + + if self._should_flush(): + self.flush() + return True + + def _should_flush(self) -> bool: + if len(self._buffer) >= self._batch_size: + return True + elapsed_ms = (time.monotonic() * 1000.0) - self._last_flush_mono_ms + return elapsed_ms >= self._batch_interval_ms and len(self._buffer) > 0 + + def flush(self) -> None: + """Commit buffered events to PG. Best-effort. + + Tries one bulk INSERT first; on ``IntegrityError`` (composite + PK collision — typically a stale continuation seed) falls back + to per-row ``record_event`` so one bad seq doesn't drop the + rest of the batch. Always clears the buffer to bound memory, + even on failure — a journaled event missing from a snapshot + is degraded UX, but a runaway buffer is corruption. + """ + if not self._buffer: + self._last_flush_mono_ms = time.monotonic() * 1000.0 + return + + # Snapshot and clear before the I/O so a concurrent record() + # call would land in a fresh buffer rather than racing the + # flush. ``complete_stream`` is single-threaded per stream, so + # this is belt-and-suspenders for any future change. + pending = self._buffer + self._buffer = [] + self._last_flush_mono_ms = time.monotonic() * 1000.0 + + try: + with db_session() as conn: + MessageEventsRepository(conn).bulk_record( + self._message_id, pending + ) + except IntegrityError: + logger.info( + "BatchedJournalWriter: bulk INSERT collided for " + "message_id=%s n=%d; falling back to per-row writes", + self._message_id, + len(pending), + ) + self._flush_per_row(pending) + return + except Exception: + logger.exception( + "BatchedJournalWriter: bulk INSERT failed for " + "message_id=%s n=%d; events dropped from journal", + self._message_id, + len(pending), + ) + return + + # Bulk INSERT committed — publish each frame in order. Best-effort: + # one failed publish must not poison the rest of the batch. + for seq, event_type, payload in pending: + self._publish(seq, event_type, payload) + + def _flush_per_row( + self, pending: list[tuple[int, str, dict[str, Any]]] + ) -> None: + """Per-row fallback after a bulk collision. Publishes after each commit.""" + for seq, event_type, payload in pending: + committed_seq: Optional[int] = None + try: + with db_session() as conn: + MessageEventsRepository(conn).record( + self._message_id, seq, event_type, payload + ) + committed_seq = seq + except IntegrityError: + try: + with db_readonly() as conn: + latest = MessageEventsRepository( + conn + ).latest_sequence_no(self._message_id) + retry_seq = (latest if latest is not None else -1) + 1 + with db_session() as conn: + MessageEventsRepository(conn).record( + self._message_id, retry_seq, event_type, payload + ) + committed_seq = retry_seq + except IntegrityError: + logger.warning( + "BatchedJournalWriter: IntegrityError persists " + "after seq+1 retry; dropping. message_id=%s " + "original_seq=%s type=%s", + self._message_id, + seq, + event_type, + ) + except Exception: + logger.exception( + "BatchedJournalWriter: per-row retry failed " + "(message_id=%s seq=%s type=%s)", + self._message_id, + seq, + event_type, + ) + except Exception: + logger.exception( + "BatchedJournalWriter: per-row INSERT failed " + "(message_id=%s seq=%s type=%s)", + self._message_id, + seq, + event_type, + ) + + if committed_seq is not None: + self._publish(committed_seq, event_type, payload) + + def _publish( + self, sequence_no: int, event_type: str, payload: dict[str, Any] + ) -> None: + """Publish one frame to the per-message pubsub channel. Best-effort.""" + try: + wire = encode_pubsub_message( + self._message_id, sequence_no, event_type, payload + ) + Topic(message_topic_name(self._message_id)).publish(wire) + except Exception: + logger.exception( + "channel:%s publish failed: seq=%s type=%s", + self._message_id, + sequence_no, + event_type, + ) + + def close(self) -> None: + """Final flush. Idempotent — safe to call from multiple + finally clauses. + """ + if self._closed: + return + self.flush() + self._closed = True diff --git a/application/worker.py b/application/worker.py index f71df293..3d2e17c6 100755 --- a/application/worker.py +++ b/application/worker.py @@ -19,8 +19,8 @@ import requests from application.agents.agent_creator import AgentCreator from application.api.answer.services.stream_processor import get_prompt -from application.cache import get_redis_instance from application.core.settings import settings +from application.events.publisher import publish_user_event from application.parser.chunking import Chunker from application.parser.connectors.connector_creator import ConnectorCreator from application.parser.embedding_pipeline import ( @@ -52,18 +52,16 @@ MAX_TOKENS = 1250 RECURSION_DEPTH = 2 INGEST_HEARTBEAT_INTERVAL_SECONDS = 30 -# Stable namespace for deterministic source IDs derived from idempotency keys. -# Pinned literal — do not change. Re-rolling this would mint different -# source_ids for the same idempotency_keys across deploys, defeating the -# retry-resume contract. -DOCSGPT_INGEST_NAMESPACE = uuid.UUID("fa25d5d1-398b-46df-ac89-8d1c360b9bea") - - -def _derive_source_id(idempotency_key): - """``uuid5(NS, key)`` when a key is supplied; ``uuid4()`` otherwise.""" - if isinstance(idempotency_key, str) and idempotency_key: - return uuid.uuid5(DOCSGPT_INGEST_NAMESPACE, idempotency_key) - return uuid.uuid4() +# Re-exported here for backward-compatible imports +# (``from application.worker import _derive_source_id`` / +# ``DOCSGPT_INGEST_NAMESPACE``) from tests and any other in-tree callers. +# New code should import from ``application.storage.db.source_ids`` +# directly to avoid pulling this Celery worker module into the API +# process at import time. +from application.storage.db.source_ids import ( # noqa: E402, F401 + DOCSGPT_INGEST_NAMESPACE, + derive_source_id as _derive_source_id, +) def _ingest_heartbeat_loop(source_id, stop_event, interval=INGEST_HEARTBEAT_INTERVAL_SECONDS): @@ -510,6 +508,7 @@ def ingest_worker( retriever="classic", file_name_map=None, idempotency_key=None, + source_id=None, ): """ Ingest and process documents. @@ -527,6 +526,11 @@ def ingest_worker( idempotency_key (str|None): When provided, the ``source_id`` is derived deterministically from the key so a retried task reuses the same source row instead of duplicating it. + source_id (str|None): UUID minted by the HTTP route and returned in + its response. When supplied, the worker uses it verbatim so SSE + envelopes carry the same id the frontend already has — required + for non-idempotent uploads where the route can't predict + ``_derive_source_id(idempotency_key)``. Returns: dict: Information about the completed ingestion task, including input parameters and a "limited" flag. @@ -541,10 +545,41 @@ def ingest_worker( logging.info(f"Ingest path: {file_path}", extra={"user": user, "job": job_name}) - # Create temporary working directory + # Source id resolution order: + # 1. Caller-supplied ``source_id`` (HTTP route minted + returned to + # the frontend) — keeps the route response and the SSE event + # payloads in lockstep on the non-idempotent path. + # 2. Deterministic uuid5 from ``idempotency_key`` — retried tasks + # reuse the original source row instead of duplicating it. + # 3. Fresh uuid4 (caller has neither) — opaque, single-shot only. + if source_id: + source_uuid = uuid.UUID(source_id) + else: + source_uuid = _derive_source_id(idempotency_key) + source_id_for_events = str(source_uuid) + # Only emit ``queued`` on the original attempt. Celery retries re-run + # the body, and re-publishing here would oscillate the toast through + # ``queued`` again between ``failed`` and ``completed``. + if self.request.retries == 0: + publish_user_event( + user, + "source.ingest.queued", + { + "job_name": job_name, + "filename": filename, + "source_id": source_id_for_events, + "operation": "upload", + }, + scope={"kind": "source", "id": source_id_for_events}, + ) - with tempfile.TemporaryDirectory() as temp_dir: - try: + # Wrap the entire body in try/except so a failure between the + # ``queued`` publish above and the inner work (e.g. tempdir + # creation, OS-level resource exhaustion) still emits a terminal + # ``failed`` event rather than leaving the toast wedged on + # 'training' until the polling fallback rescues it 30s later. + try: + with tempfile.TemporaryDirectory() as temp_dir: os.makedirs(temp_dir, exist_ok=True) if storage.is_directory(file_path): @@ -633,23 +668,22 @@ def ingest_worker( docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs] - id = _derive_source_id(idempotency_key) - vector_store_path = os.path.join(temp_dir, "vector_store") os.makedirs(vector_store_path, exist_ok=True) - heartbeat_thread, heartbeat_stop = _start_ingest_heartbeat(id) + heartbeat_thread, heartbeat_stop = _start_ingest_heartbeat(source_uuid) try: embed_and_store_documents( - docs, vector_store_path, id, self, + docs, vector_store_path, source_uuid, self, attempt_id=getattr(self.request, "id", None), + user_id=user, ) finally: _stop_ingest_heartbeat(heartbeat_thread, heartbeat_stop) # Defense-in-depth: chunk-progress is the authoritative # record of how many chunks landed; mismatch raises so the # task fails loud rather than caching a partial index. - assert_index_complete(id) + assert_index_complete(source_uuid) tokens = count_tokens_docs(docs) @@ -664,7 +698,7 @@ def ingest_worker( "user": user, "tokens": tokens, "retriever": retriever, - "id": str(id), + "id": source_id_for_events, "type": "local", "file_path": file_path, "directory_structure": json.dumps(directory_structure), @@ -673,9 +707,36 @@ def ingest_worker( file_data["file_name_map"] = json.dumps(file_name_map) upload_index(vector_store_path, file_data) - except Exception as e: - logging.error(f"Error in ingest_worker: {e}", exc_info=True) - raise + publish_user_event( + user, + "source.ingest.completed", + { + "source_id": source_id_for_events, + "filename": filename, + "tokens": tokens, + "operation": "upload", + # Forward-looking contract: ``limited`` is always + # ``False`` today but is carried on the wire so a + # future token-cap detection path can flip it and + # the frontend slice / UploadToast already react. + "limited": False, + }, + scope={"kind": "source", "id": source_id_for_events}, + ) + except Exception as e: + logging.error(f"Error in ingest_worker: {e}", exc_info=True) + publish_user_event( + user, + "source.ingest.failed", + { + "source_id": source_id_for_events, + "filename": filename, + "operation": "upload", + "error": str(e)[:1024], + }, + scope={"kind": "source", "id": source_id_for_events}, + ) + raise return { "directory": directory, "formats": formats, @@ -699,7 +760,23 @@ def reingest_source_worker(self, source_id, user): Returns: dict: Information about the re-ingestion task + + Note: + Reingest does its own ``vector_store.add_chunk`` work rather + than going through ``embed_and_store_documents`` so it does + *not* emit per-percent SSE progress events — only ``queued``, + ``completed`` (carrying ``chunks_added`` / ``chunks_deleted``), + or ``failed``. v1 limitation; revisit if reingest gains a + progress-driven UI. """ + # Declared at the function scope so the outer except can include + # ``name`` in the failed event payload when the failure happens + # after the source lookup. Empty string until the lookup succeeds. + source_name = "" + # Tracks inner-block failures so a ``completed`` event reflects + # partial-success accurately rather than masking it. + inner_warnings: list[str] = [] + try: from application.vectorstore.vector_creator import VectorCreator @@ -713,6 +790,27 @@ def reingest_source_worker(self, source_id, user): if not source: raise ValueError(f"Source {source_id} not found or access denied") source_id = str(source["id"]) + source_name = source.get("name") or "" + + # Publish ``queued`` *after* canonicalising ``source_id`` so the + # event references the same id as the source row. Trade-off + # documented: a Celery-backend or PG-lookup hiccup before this + # publish means the toast may see only a ``failed`` event with + # no preceding ``queued`` — acceptable for v1 since both + # conditions also imply broader system trouble. Gate on first + # attempt only so Celery retries don't re-emit ``queued`` after + # a prior attempt already published ``failed``. + if self.request.retries == 0: + publish_user_event( + user, + "source.ingest.queued", + { + "source_id": source_id, + "name": source_name, + "operation": "reingest", + }, + scope={"kind": "source", "id": source_id}, + ) storage = StorageCreator.get_storage() source_file_path = source.get("file_path", "") @@ -810,6 +908,19 @@ def reingest_source_worker(self, source_id, user): try: if not added_files and not removed_files: logging.info("No changes detected.") + publish_user_event( + user, + "source.ingest.completed", + { + "source_id": source_id, + "name": source_name, + "operation": "reingest", + "no_changes": True, + "chunks_added": 0, + "chunks_deleted": 0, + }, + scope={"kind": "source", "id": source_id}, + ) return { "source_id": source_id, "user": user, @@ -861,6 +972,9 @@ def reingest_source_worker(self, source_id, user): f"Error during deletion of removed file chunks: {e}", exc_info=True, ) + inner_warnings.append( + f"deletion failed: {str(e)[:200]}" + ) # 2) Add chunks from new files added = 0 @@ -953,6 +1067,9 @@ def reingest_source_worker(self, source_id, user): logging.error( f"Error during ingestion of new files: {e}", exc_info=True ) + inner_warnings.append( + f"add failed: {str(e)[:200]}" + ) # 3) Update source directory structure timestamp try: @@ -981,6 +1098,25 @@ def reingest_source_worker(self, source_id, user): meta={"current": 100, "status": "Re-ingestion completed"}, ) + completed_payload: dict = { + "source_id": source_id, + "name": source_name, + "operation": "reingest", + "chunks_added": added, + "chunks_deleted": deleted, + "tokens": int(total_tokens) if "total_tokens" in locals() else 0, + } + if inner_warnings: + # Surface the per-block failures so the toast can warn + # rather than claim a clean success. + completed_payload["warnings"] = inner_warnings + publish_user_event( + user, + "source.ingest.completed", + completed_payload, + scope={"kind": "source", "id": source_id}, + ) + return { "source_id": source_id, "user": user, @@ -998,6 +1134,17 @@ def reingest_source_worker(self, source_id, user): except Exception as e: logging.error(f"Error in reingest_source_worker: {e}", exc_info=True) + publish_user_event( + user, + "source.ingest.failed", + { + "source_id": str(source_id), + "name": source_name, + "operation": "reingest", + "error": str(e)[:1024], + }, + scope={"kind": "source", "id": str(source_id)}, + ) raise @@ -1013,12 +1160,51 @@ def remote_worker( operation_mode="upload", doc_id=None, idempotency_key=None, + source_id=None, ): safe_user = safe_filename(user) full_path = os.path.join(directory, safe_user, uuid.uuid4().hex) os.makedirs(full_path, exist_ok=True) - self.update_state(state="PROGRESS", meta={"current": 1}) + + # Source id resolution order matches ``ingest_worker``: + # 1. ``operation_mode == "sync"`` reuses the existing source's ``doc_id``. + # 2. Caller-supplied ``source_id`` (the HTTP route minted it and + # already returned it to the frontend) — keeps the route + # response and the SSE event payloads in lockstep on the + # no-idempotency-key path. + # 3. Deterministic uuid5 from ``idempotency_key`` — retried tasks + # reuse the original source row instead of duplicating it. + # 4. Fresh uuid4 — opaque, single-shot only. + if operation_mode == "sync" and doc_id: + source_uuid = str(doc_id) + elif source_id: + source_uuid = uuid.UUID(source_id) + else: + source_uuid = _derive_source_id(idempotency_key) + source_id_for_events = str(source_uuid) + + # Emit the queued event before any work that could fail (including + # ``update_state``) so the toast UI always sees a queued envelope + # before any subsequent failed event. Gated on first attempt so + # Celery retries don't re-emit ``queued`` after a prior ``failed``. + if self.request.retries == 0: + publish_user_event( + user, + "source.ingest.queued", + { + "source_id": source_id_for_events, + "job_name": name_job, + "loader": loader, + "operation": operation_mode, + }, + scope={"kind": "source", "id": source_id_for_events}, + ) + + # Wrap ``update_state`` plus the entire body so any pre-loader + # failure (Celery backend down, OS resource issue) still emits a + # terminal ``failed`` event rather than wedging the toast. try: + self.update_state(state="PROGRESS", meta={"current": 1}) logging.info("Initializing remote loader with type: %s", loader) remote_loader = RemoteCreator.create_loader(loader) raw_docs = remote_loader.load_data(source_data) @@ -1105,22 +1291,22 @@ def remote_worker( ) if operation_mode == "upload": - id = _derive_source_id(idempotency_key) embed_and_store_documents( - docs, full_path, id, self, + docs, full_path, source_uuid, self, attempt_id=getattr(self.request, "id", None), + user_id=user, ) - assert_index_complete(id) + assert_index_complete(source_uuid) elif operation_mode == "sync": if not doc_id: logging.error("Invalid doc_id provided for sync operation: %s", doc_id) raise ValueError("doc_id must be provided for sync operation.") - id = str(doc_id) embed_and_store_documents( - docs, full_path, id, self, + docs, full_path, source_uuid, self, attempt_id=getattr(self.request, "id", None), + user_id=user, ) - assert_index_complete(id) + assert_index_complete(source_uuid) self.update_state(state="PROGRESS", meta={"current": 100}) # Serialize remote_data as JSON if it's a dict (for S3, Reddit, etc.) @@ -1132,7 +1318,7 @@ def remote_worker( "user": user, "tokens": tokens, "retriever": retriever, - "id": str(id), + "id": source_id_for_events, "type": loader, "remote_data": remote_data_serialized, "sync_frequency": sync_frequency, @@ -1146,23 +1332,49 @@ def remote_worker( try: with db_session() as conn: repo = SourcesRepository(conn) - src = repo.get_any(str(id), user) + src = repo.get_any(source_id_for_events, user) if src is not None: repo.update(str(src["id"]), user, {"date": last_sync_now}) except Exception as upd_err: logging.warning( - f"Failed to update last_sync for source {id}: {upd_err}" + f"Failed to update last_sync for source {source_id_for_events}: {upd_err}" ) upload_index(full_path, file_data) + publish_user_event( + user, + "source.ingest.completed", + { + "source_id": source_id_for_events, + "job_name": name_job, + "loader": loader, + "operation": operation_mode, + "tokens": tokens, + # Forward-looking contract: see ingest_worker. + "limited": False, + }, + scope={"kind": "source", "id": source_id_for_events}, + ) except Exception as e: logging.error("Error in remote_worker task: %s", str(e), exc_info=True) + publish_user_event( + user, + "source.ingest.failed", + { + "source_id": source_id_for_events, + "job_name": name_job, + "loader": loader, + "operation": operation_mode, + "error": str(e)[:1024], + }, + scope={"kind": "source", "id": source_id_for_events}, + ) raise finally: if os.path.exists(full_path): shutil.rmtree(full_path) logging.info("remote_worker task completed successfully") return { - "id": str(id), + "id": source_id_for_events, "urls": source_data, "name_job": name_job, "user": user, @@ -1245,6 +1457,13 @@ def attachment_worker(self, file_info, user): relative_path = file_info["path"] metadata = file_info.get("metadata", {}) + publish_user_event( + user, + "attachment.queued", + {"attachment_id": str(attachment_id), "filename": filename}, + scope={"kind": "attachment", "id": str(attachment_id)}, + ) + try: self.update_state(state="PROGRESS", meta={"current": 10}) storage = StorageCreator.get_storage() @@ -1252,6 +1471,17 @@ def attachment_worker(self, file_info, user): self.update_state( state="PROGRESS", meta={"current": 30, "status": "Processing content"} ) + publish_user_event( + user, + "attachment.progress", + { + "attachment_id": str(attachment_id), + "filename": filename, + "current": 30, + "stage": "processing", + }, + scope={"kind": "attachment", "id": str(attachment_id)}, + ) file_extractor = get_default_file_extractor( ocr_enabled=settings.DOCLING_OCR_ATTACHMENTS_ENABLED @@ -1284,6 +1514,17 @@ def attachment_worker(self, file_info, user): self.update_state( state="PROGRESS", meta={"current": 80, "status": "Storing in database"} ) + publish_user_event( + user, + "attachment.progress", + { + "attachment_id": str(attachment_id), + "filename": filename, + "current": 80, + "stage": "storing", + }, + scope={"kind": "attachment", "id": str(attachment_id)}, + ) mime_type = mimetypes.guess_type(filename)[0] or "application/octet-stream" @@ -1308,6 +1549,18 @@ def attachment_worker(self, file_info, user): self.update_state(state="PROGRESS", meta={"current": 100, "status": "Complete"}) + publish_user_event( + user, + "attachment.completed", + { + "attachment_id": str(attachment_id), + "filename": filename, + "token_count": token_count, + "mime_type": mime_type, + }, + scope={"kind": "attachment", "id": str(attachment_id)}, + ) + return { "filename": filename, "path": relative_path, @@ -1322,6 +1575,16 @@ def attachment_worker(self, file_info, user): extra={"user": user}, exc_info=True, ) + publish_user_event( + user, + "attachment.failed", + { + "attachment_id": str(attachment_id), + "filename": filename, + "error": str(e)[:1024], + }, + scope={"kind": "attachment", "id": str(attachment_id)}, + ) raise @@ -1385,6 +1648,7 @@ def ingest_connector( doc_id=None, sync_frequency: str = "never", idempotency_key=None, + source_id=None, ) -> Dict[str, Any]: """ Ingestion for internal knowledge bases (GoogleDrive, etc.). @@ -1403,14 +1667,50 @@ def ingest_connector( sync_frequency: How often to sync ("never", "daily", "weekly", "monthly") idempotency_key: When provided, the ``source_id`` is derived deterministically so a retried upload reuses the same source row. + source_id: When supplied, the worker uses it verbatim so SSE envelopes + carry the same id the HTTP route already returned to the frontend + — required for non-idempotent uploads where the route can't + predict ``_derive_source_id(idempotency_key)``. """ logging.info( f"Starting remote ingestion from {source_type} for user: {user}, job: {job_name}" ) + + # Source id resolution mirrors ``ingest_worker`` / ``remote_worker``: + # sync mode reuses ``doc_id``; otherwise the caller-supplied + # ``source_id`` (minted by the HTTP route and already echoed to the + # client) wins; fall back to ``_derive_source_id`` only when neither + # is supplied. Without rule (2) the no-idempotency-key path would + # mint a fresh uuid4 here that the frontend has no way to correlate + # SSE envelopes to. + if operation_mode == "sync" and doc_id: + source_uuid = str(doc_id) + elif source_id: + source_uuid = uuid.UUID(source_id) + else: + source_uuid = _derive_source_id(idempotency_key) + source_id_for_events = str(source_uuid) + + # First-attempt gate: Celery retries re-run the body, and a + # repeated ``queued`` here would oscillate the toast through + # ``queued`` again between ``failed`` and ``completed``. + if self.request.retries == 0: + publish_user_event( + user, + "source.ingest.queued", + { + "source_id": source_id_for_events, + "job_name": job_name, + "loader": source_type, + "operation": operation_mode, + }, + scope={"kind": "source", "id": source_id_for_events}, + ) + self.update_state(state="PROGRESS", meta={"current": 1}) - with tempfile.TemporaryDirectory() as temp_dir: - try: + try: + with tempfile.TemporaryDirectory() as temp_dir: # Step 1: Initialize the appropriate loader self.update_state( state="PROGRESS", @@ -1448,6 +1748,22 @@ def ingest_connector( "files_downloaded", 0 ): logging.warning(f"No files were downloaded from {source_type}") + # Connector returned no files — surface as a benign + # ``completed`` event with zero tokens so the toast + # closes out cleanly instead of waiting on polling. + publish_user_event( + user, + "source.ingest.completed", + { + "source_id": source_id_for_events, + "job_name": job_name, + "loader": source_type, + "operation": operation_mode, + "tokens": 0, + "no_changes": True, + }, + scope={"kind": "source", "id": source_id_for_events}, + ) # Create empty result directly instead of calling a separate method return { "name": job_name, @@ -1497,16 +1813,16 @@ def ingest_connector( docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs] - if operation_mode == "upload": - id = _derive_source_id(idempotency_key) - elif operation_mode == "sync": - if not doc_id: - logging.error( - "Invalid doc_id provided for sync operation: %s", doc_id - ) - raise ValueError("doc_id must be provided for sync operation.") - id = str(doc_id) - else: + # Validate operation_mode here too (the source_uuid path + # at the top of the function only branches on the + # sync+doc_id combination; surfacing the wrong-mode error + # this far in matches the legacy behaviour). + if operation_mode == "sync" and not doc_id: + logging.error( + "Invalid doc_id provided for sync operation: %s", doc_id + ) + raise ValueError("doc_id must be provided for sync operation.") + if operation_mode not in ("upload", "sync"): raise ValueError(f"Invalid operation_mode: {operation_mode}") vector_store_path = os.path.join(temp_dir, "vector_store") @@ -1516,10 +1832,11 @@ def ingest_connector( state="PROGRESS", meta={"current": 80, "status": "Storing documents"} ) embed_and_store_documents( - docs, vector_store_path, id, self, + docs, vector_store_path, source_uuid, self, attempt_id=getattr(self.request, "id", None), + user_id=user, ) - assert_index_complete(id) + assert_index_complete(source_uuid) tokens = count_tokens_docs(docs) @@ -1529,7 +1846,7 @@ def ingest_connector( "name": job_name, "tokens": tokens, "retriever": retriever, - "id": str(id), + "id": source_id_for_events, "type": "connector:file", "remote_data": json.dumps( {"provider": source_type, **api_source_config} @@ -1538,16 +1855,13 @@ def ingest_connector( "sync_frequency": sync_frequency, } - if operation_mode == "sync": - file_data["last_sync"] = datetime.datetime.now() - else: - file_data["last_sync"] = datetime.datetime.now() + file_data["last_sync"] = datetime.datetime.now() if operation_mode == "sync": try: with db_session() as conn: repo = SourcesRepository(conn) - src = repo.get_any(str(id), user) + src = repo.get_any(source_id_for_events, user) if src is not None: repo.update( str(src["id"]), user, @@ -1555,7 +1869,9 @@ def ingest_connector( ) except Exception as upd_err: logging.warning( - f"Failed to update last_sync for source {id}: {upd_err}" + "Failed to update last_sync for source %s: %s", + source_id_for_events, + upd_err, ) upload_index(vector_store_path, file_data) @@ -1567,45 +1883,104 @@ def ingest_connector( logging.info(f"Remote ingestion completed: {job_name}") + publish_user_event( + user, + "source.ingest.completed", + { + "source_id": source_id_for_events, + "job_name": job_name, + "loader": source_type, + "operation": operation_mode, + "tokens": tokens, + }, + scope={"kind": "source", "id": source_id_for_events}, + ) + return { "user": user, "name": job_name, "tokens": tokens, "type": source_type, - "id": str(id), + "id": source_id_for_events, "status": "complete", } - - except Exception as e: - logging.error(f"Error during remote ingestion: {e}", exc_info=True) - raise + except Exception as e: + logging.error(f"Error during remote ingestion: {e}", exc_info=True) + publish_user_event( + user, + "source.ingest.failed", + { + "source_id": source_id_for_events, + "job_name": job_name, + "loader": source_type, + "operation": operation_mode, + "error": str(e)[:1024], + }, + scope={"kind": "source", "id": source_id_for_events}, + ) + raise def mcp_oauth(self, config: Dict[str, Any], user_id: str = None) -> Dict[str, Any]: - """Worker to handle MCP OAuth flow asynchronously.""" + """Worker to handle MCP OAuth flow asynchronously. + + Publishes SSE events at each phase boundary so the frontend can + drive the OAuth popup directly from the push channel. The + ``mcp.oauth.awaiting_redirect`` envelope carries the + ``authorization_url`` once the upstream OAuth client surfaces it, + eliminating the prior polling-only path for that URL. + """ + + # Bind ``task_id`` and the publish helpers OUTSIDE the outer try so + # the ``except`` handler at the bottom can reach them even when an + # early statement raises. Without this, ``publish_oauth`` would + # UnboundLocalError on top of the original failure. + task_id = self.request.id if getattr(self, "request", None) else None + + def publish_oauth(event_type: str, payload: Dict[str, Any]) -> None: + # MCP OAuth can be invoked without a route-bound user_id by + # legacy paths. Skip the SSE publish in that case \u2014 the caller + # has no per-user channel to subscribe to, and the status is + # surfaced via the task's return value. + if not user_id or task_id is None: + return + publish_user_event( + user_id, + event_type, + {"task_id": task_id, **payload}, + scope={"kind": "mcp_oauth", "id": task_id}, + ) + + def publish_awaiting_redirect(authorization_url: str) -> None: + """Callback invoked by ``DocsGPTOAuth.redirect_handler`` once + the OAuth client has minted the authorization URL. + + Carrying the URL on the SSE envelope lets the frontend open the + popup directly from the event \u2014 the prior polling-only path + for the URL is gone. + """ + publish_oauth( + "mcp.oauth.awaiting_redirect", + { + "message": "Awaiting OAuth redirect...", + "authorization_url": authorization_url, + }, + ) try: import asyncio from application.agents.tools.mcp_tool import MCPTool - task_id = self.request.id - redis_client = get_redis_instance() - - def update_status(status_data: Dict[str, Any]): - status_key = f"mcp_oauth_status:{task_id}" - redis_client.setex(status_key, 600, json.dumps(status_data)) - - update_status( - { - "status": "in_progress", - "message": "Starting OAuth...", - "task_id": task_id, - } - ) + publish_oauth("mcp.oauth.in_progress", {"message": "Starting OAuth..."}) tool_config = config.copy() tool_config["oauth_task_id"] = task_id + # Inject the awaiting-redirect publish callback. ``MCPTool`` pops + # it out of the config and threads it into ``DocsGPTOAuth`` so + # the publish fires synchronously from inside + # ``redirect_handler`` \u2014 the only point where the URL is known. + tool_config["oauth_redirect_publish"] = publish_awaiting_redirect mcp_tool = MCPTool(tool_config, user_id) async def run_oauth_discovery(): @@ -1613,14 +1988,6 @@ def mcp_oauth(self, config: Dict[str, Any], user_id: str = None) -> Dict[str, An mcp_tool._setup_client() return await mcp_tool._execute_with_client("list_tools") - update_status( - { - "status": "awaiting_redirect", - "message": "Awaiting OAuth redirect...", - "task_id": task_id, - } - ) - loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -1628,49 +1995,21 @@ def mcp_oauth(self, config: Dict[str, Any], user_id: str = None) -> Dict[str, An loop.run_until_complete(run_oauth_discovery()) tools = mcp_tool.get_actions_metadata() - update_status( - { - "status": "completed", - "message": f"Connected \u2014 found {len(tools)} tool{'s' if len(tools) != 1 else ''}.", - "tools": tools, - "tools_count": len(tools), - "task_id": task_id, - } + publish_oauth( + "mcp.oauth.completed", + {"tools": tools, "tools_count": len(tools)}, ) return {"success": True, "tools": tools, "tools_count": len(tools)} except Exception as e: error_msg = f"OAuth failed: {str(e)}" logging.error("MCP OAuth discovery failed: %s", error_msg, exc_info=True) - update_status( - { - "status": "error", - "message": error_msg, - "task_id": task_id, - } - ) + publish_oauth("mcp.oauth.failed", {"error": error_msg[:1024]}) return {"success": False, "error": error_msg} finally: loop.close() except Exception as e: error_msg = f"OAuth init failed: {str(e)}" logging.error("MCP OAuth init failed: %s", error_msg, exc_info=True) - update_status( - { - "status": "error", - "message": error_msg, - "task_id": task_id, - } - ) + publish_oauth("mcp.oauth.failed", {"error": error_msg[:1024]}) return {"success": False, "error": error_msg} - - -def mcp_oauth_status(self, task_id: str) -> Dict[str, Any]: - """Check the status of an MCP OAuth flow.""" - redis_client = get_redis_instance() - status_key = f"mcp_oauth_status:{task_id}" - - status_data = redis_client.get(status_key) - if status_data: - return json.loads(status_data) - return {"status": "not_found", "message": "Status not found"} diff --git a/docs/runbooks/sse-notifications.md b/docs/runbooks/sse-notifications.md new file mode 100644 index 00000000..1dc1bc1b --- /dev/null +++ b/docs/runbooks/sse-notifications.md @@ -0,0 +1,385 @@ +# SSE Notifications Runbook + +> Operations guide for "user says they didn't get a notification" — and +> the related "the bell never lights up" / "my upload toast hangs" / +> "the chat answer doesn't reconnect" symptoms. + +The user-facing notifications channel is the SSE pipe at +`/api/events` plus per-message reconnects at +`/api/messages//events`. This document maps a user complaint to +the diagnostic that surfaces the cause. + +--- + +## TL;DR — first 60 seconds + +Run these three commands in parallel before anything else: + +```bash +# 1) Is Redis up and serving the pipe? Should print PONG instantly. +redis-cli -n 2 PING + +# 2) Anyone subscribed to the channel right now? Numbers per channel. +redis-cli -n 2 PUBSUB NUMSUB user: + +# 3) Is the user's backlog populated? Returns the count of journaled events. +redis-cli -n 2 XLEN user::stream +``` + +- `PING` failing → Redis is the problem. Skip to "Redis-down". +- `NUMSUB user:` returns 0 → no client connected. Skip to "Client never connects". +- `XLEN user::stream` returns 0 or low → publisher isn't writing. Skip to "Publisher silent". +- All three look healthy → the events are flowing on the wire; the issue is downstream of the slice (UI rendering, toast suppression, etc.). Skip to "Events flowing but UI silent". + +--- + +## Architecture cheat-sheet + +``` +Worker (publish_user_event) Frontend tab + │ ▲ + ▼ │ GET /api/events SSE + Redis Streams: XADD Flask route + user::stream ──────────────► replay_backlog (snapshot) + │ + + ▼ Topic.subscribe (live tail) + Redis pub/sub: PUBLISH │ + user: ────────────────────────────────┘ +``` + +**Source of truth:** +- Persistent journal: Redis Stream `user::stream`, capped at + `EVENTS_STREAM_MAXLEN` (default 1000) entries via `MAXLEN ~`. ~24h + at typical event rates. +- Live fan-out: Redis pub/sub channel `user:`. No durability; + subscribers must be attached at publish time. + +The chat-stream pipe is separate, parallel infrastructure: +- Journal: Postgres `message_events` table. +- Live fan-out: Redis pub/sub `channel:`. + +Same patterns, different durability layer. This doc covers both; +they share most diagnostic commands. + +--- + +## Symptom → diagnostic map + +### A. "I uploaded a source and the toast never appeared" + +User flow: chat → upload → expect toast. + +| Step | Command | Expect | +| ------------------------------------------------- | ------------------------------------------------------------- | ----------------------------------------------- | +| Worker received the task | `tail -f celery.log` filtered by user | `ingest_worker` start log line | +| Worker published the queued event | `redis-cli -n 2 XREVRANGE user::stream + - COUNT 5` | A `source.ingest.queued` entry within seconds | +| Frontend got it | DevTools → Network → `/api/events` → EventStream tab | `data: {"type":"source.ingest.queued",...}` | +| Slice updated | Redux DevTools → state.upload.tasks | Task with matching `sourceId`, `status:'training'` | + +If the worker's queued log line is there but the XADD didn't land → +look for a `publish_user_event payload not JSON-serializable` warning +in the worker log (the publisher swallows `TypeError`). + +If the XADD landed but the frontend never received it → check +`PUBSUB NUMSUB user:` while the user is on the page. If 0, the +SSE connection isn't subscribed; skip to "Client never connects". + +If the frontend received it but the toast didn't render → the +`uploadSlice` extraReducer requires `task.sourceId` to match the +event's `scope.id`. Check the upload route returned `source_id` in +its POST response (the upload, connector, and reingest paths all +include it). Idempotent / cached responses must also include +`source_id` (`_claim_task_or_get_cached`). + +### B. "The bell badge never goes up" + +There is no bell — the global notifications surface is per-event +toasts, not an aggregated counter. If the user is on an old build, +`Cmd-Shift-R` to bypass cache. The surfaces they're looking for are +`UploadToast` for source uploads and `ToolApprovalToast` for +tool-approval events. + +### C. "My chat answer froze mid-stream and never recovered" + +User flow: ask question → answer streaming → network blip → answer +stops; should reconnect. + +```bash +# Was the original message reserved in PG? +psql -c "SELECT id, status, prompt FROM conversation_messages \ + WHERE user_id = '' ORDER BY timestamp DESC LIMIT 5;" + +# Did the journal capture events past the user's last-seen seq? +psql -c "SELECT sequence_no, event_type FROM message_events \ + WHERE message_id = '' ORDER BY sequence_no;" + +# Is the live tail still producing? (subscribe and watch) +redis-cli -n 2 SUBSCRIBE channel: +``` + +The frontend should reconnect via `GET /api/messages//events` +when the original POST stream closes without a typed `end` or +`error` event. If it's not reconnecting, `console.warn('Stream +reconnect failed', ...)` will be in the browser console — the +reconnect HTTP errored. Common cases: + +- The user's JWT rotated mid-stream → 401 on the GET. Frontend + doesn't auto-refresh; the user reloads. +- The user is on a different host than the API and CORS is rejecting + the GET → check `application/asgi.py` allow-headers. + +### D. "The dev install never delivers any notifications at all" + +Default `AUTH_TYPE` unset means `decoded_token = {"sub": "local"}` +for every request. The SSE client connects without the +`Authorization` header in this case, and `user:local:stream` is +the shared channel everything goes to. If the user has multiple dev +machines pointing at the same Redis, they will see each other's +events. Confirm with: + +```bash +redis-cli -n 2 KEYS 'user:local:*' +``` + +If multiple deployments share the Redis, document that as a known +multi-user-on-local-channel limitation. Set `AUTH_TYPE=simple_jwt` +to scope per-user. + +### E. "The notifications channel was working, then suddenly stopped after the user reloaded the page" + +Likely path: `backlog.truncated` event fired, the slice cleared +`lastEventId` to null, the closure was carrying the same stale id and +re-tripped the same truncation on every reconnect. **Verify the user +is on a current build — `eventStreamClient.ts` must re-read +`lastEventId = opts.getLastEventId();` without a truthy guard so the +null clear propagates into the next reconnect.** + +### F. "I keep getting 429 on /api/events" + +The per-user concurrent-connection cap (`SSE_MAX_CONCURRENT_PER_USER`, +default 8) refused the connection. User has too many tabs open or a +runaway reconnect loop. `redis-cli -n 2 GET user::sse_count` +shows the live counter; the TTL is 1h from the last connection +attempt (rolling — every INCR re-seeds it), so the key only ages +out after the user stops reconnecting for a full hour. + +If the count is wedged high without explanation, the +counter-DECR-in-finally path didn't run (worker SIGKILL, OOM). Wait +for the TTL or `redis-cli -n 2 DEL user::sse_count` to reset. + +### G. "Replay snapshot stops at 200 events" + +The route caps each replay at `EVENTS_REPLAY_MAX_PER_REQUEST` +(default 200). The cap is intentionally **silent** — the route does NOT +emit a `backlog.truncated` notice for cap-hit. The 200 entries each +carry their own `id:` header, so the frontend's slice cursor +advances to the most-recent delivered id. Next reconnect sends +`last_event_id=` and the snapshot resumes from there. +A user that was 1000 entries behind catches up over ~5 reconnects. + +If the user reports getting HTTP 429 on `/api/events` despite being +well under `SSE_MAX_CONCURRENT_PER_USER`, they hit the windowed +replay budget (`EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW`, default +30 / `EVENTS_REPLAY_BUDGET_WINDOW_SECONDS` 60s). The route refuses +the connection so the slice cursor stays pinned at whatever value +it had; the frontend backs off and the next reconnect (after the +window rolls) gets the proper snapshot. Serving the live tail +without a snapshot used to be the behavior here, but that let the +client advance `lastEventId` past entries it never received, +permanently stranding the un-replayed window — so the route now +429s instead. `redis-cli -n 2 GET user::replay_count` shows the +current counter; TTL is the window size. + +`backlog.truncated` is emitted ONLY when the client's +`Last-Event-ID` has slid off the MAXLEN'd window — i.e. the journal +is genuinely gone past the cursor and the frontend should clear the +slice cursor and refetch state. Treating cap-hit or +budget-exhaustion the same way would lock the user into re-receiving +the oldest 200 entries on every reconnect (the cursor would clear, +the snapshot would re-serve from the start, the cap would re-trip). + +### H. "User says push notifications stopped after a deploy" + +- Pull `event.published topic=user: type=...` from the worker + logs to confirm the publisher is still firing. +- Pull `event.connect user=` from the API logs to confirm the + client is reconnecting. +- Check the gunicorn worker count and `WSGIMiddleware(workers=32)` — + if the deploy reduced worker count, the per-user cap is still 8 + but total concurrent SSE connections are bounded by `gunicorn + workers × 32`. A capacity miss looks like users randomly getting + 429'd. + +--- + +## Common failure modes + +### Redis-down + +Symptoms: `/api/events` returns 200 but emits only `: connected` +then the body closes. `XLEN` and `PUBLISH` both fail. The publisher's +`record_event` swallows the failure and returns False; the live tail +publish also drops on the floor. Frontend retries forever with +exponential backoff. + +Resolution: bring Redis back. The journal is gone (was in-memory +only — Streams persist within a single Redis instance, no replication +configured). New events flow as soon as Redis comes back. + +### `AUTH_TYPE` misconfigured = sub:"local" cross-stream + +Symptoms: every user shares `user:local:stream`. Any user sees +everyone else's notifications. + +Resolution: set `AUTH_TYPE=simple_jwt` (or `session_jwt`) in `.env`. +The events route logs a one-time WARNING per process when +`sub == "local"` is observed. A repeat WARNING after a restart +confirms the misconfiguration. + +### MAXLEN trimmed past Last-Event-ID + +Symptoms: client reconnects with `last_event_id=X`, snapshot returns +the entire MAXLEN'd backlog (because X is older than the oldest +retained entry). Old events appear duplicated. + +Detection: the route's `_oldest_retained_id` check emits +`backlog.truncated` when this case fires. Frontend's +`dispatchSSEEvent` clears `lastEventId` so the next reconnect starts +fresh. + +If the WARNING isn't firing but symptoms match: the user's client +may have a corrupt cached `lastEventId`. `localStorage` doesn't +store this state; check Redux state via DevTools. + +### Stale event-stream client + +Symptoms: events visible in `XRANGE` but the frontend slice doesn't +update. + +```bash +# Is the client subscribed? +redis-cli -n 2 PUBSUB NUMSUB user: + +# When did its connection start? +grep "event.connect user=" /var/log/docsgpt.log | tail -3 +``` + +If `NUMSUB` is 0 and no recent `event.connect`, the user's tab is +closed or the connection died and never reconnected. Push them to +reload. + +### Publisher silent + +Symptoms: worker is processing the task (Celery says SUCCESS), but +no XADD and no PUBLISH. User sees no events. + +```bash +# Was the publisher import error suppressed? +grep "publish_user_event" /var/log/celery.log | grep -i "warn\|error" | tail -20 + +# Is push disabled? +grep "ENABLE_SSE_PUSH" /var/log/docsgpt.log | tail -5 +``` + +`ENABLE_SSE_PUSH=False` in `.env` would silence the publisher +globally. Useful for incident response if a runaway publisher is +DoS'ing Redis; toggle off, fix root cause, toggle on. + +--- + +## Useful one-liners + +```bash +# Watch a user's live event stream in real time (all events, all types) +redis-cli -n 2 PSUBSCRIBE 'user:*' | grep "user:" + +# Last 10 events the user would see on reconnect +redis-cli -n 2 XREVRANGE user::stream + - COUNT 10 + +# Live count of subscribed clients per user +redis-cli -n 2 PUBSUB NUMSUB $(redis-cli -n 2 PUBSUB CHANNELS 'user:*') + +# Trim a runaway stream (CAREFUL — destroys backlog for all current +# subscribers; OK after explaining to the user) +redis-cli -n 2 XTRIM user::stream MAXLEN 0 + +# Clear a wedged concurrent-connection counter +redis-cli -n 2 DEL user::sse_count + +# Force-flip every client to re-snapshot (drop the stream key entirely +# — destroys the backlog; clients reconnect with their last id and +# get a backlog.truncated) +redis-cli -n 2 DEL user::stream +``` + +--- + +## Settings reference + +Everything in `application/core/settings.py`: + +| Setting | Default | Purpose | +| --------------------------------------------- | ------- | --------------------------------------------- | +| `ENABLE_SSE_PUSH` | `True` | Master switch. False = publisher no-ops, route serves "push_disabled" comment. | +| `EVENTS_STREAM_MAXLEN` | `1000` | Per-user backlog cap. Approximate via `XADD MAXLEN ~`. | +| `SSE_KEEPALIVE_SECONDS` | `15` | Comment-frame cadence. Must sit under reverse-proxy idle close. | +| `SSE_MAX_CONCURRENT_PER_USER` | `8` | Cap on simultaneous SSE connections per user. 0 = disabled. | +| `EVENTS_REPLAY_MAX_PER_REQUEST` | `200` | Hard cap on snapshot rows per request. | +| `EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW` | `30` | Per-user replays per window. 0 = disabled. | +| `EVENTS_REPLAY_BUDGET_WINDOW_SECONDS` | `60` | Window length. | +| `MESSAGE_EVENTS_RETENTION_DAYS` | `14` | Retention for the `message_events` journal; `cleanup_message_events` beat task deletes older rows. | + +--- + +## Known limitations + +### Each tab runs its own SSE connection + +There is no cross-tab dedup. Every tab open to the app holds its +own SSE connection and dispatches every received event into its +own Redux store, so a user with N tabs open will see N copies of +each toast. With `SSE_MAX_CONCURRENT_PER_USER=8` (the default) a +heavy multi-tab user can also hit the connection cap and start +seeing 429s. Cross-tab dedup via a `BroadcastChannel` ring + +`navigator.locks`-based leader election is tracked as future work. + +### `/c/` normalises to `/c/new` + +If a user navigates to a conversation id that isn't in their +loaded list, the conversation route rewrites the URL to `/c/new`. +`ToolApprovalToast`'s gate uses `useMatch('/c/:conversationId')`, +so for the brief window after the rewrite the toast may surface +for a conversation the user *thought* they were already viewing. +Pre-existing route behaviour; not a notifications regression. + +### Terminal events un-dismiss running uploads + +`frontend/src/upload/uploadSlice.ts` sets `dismissed: false` when +an upload reaches `completed` or `failed`. If the user dismissed a +running task and the terminal SSE arrives later, the toast pops +back. Intentional ("notify the user it's done"); revisit if the +re-surface UX is too aggressive for v2. + +### Werkzeug doesn't auto-reload route files + +The dev server (`flask run`) doesn't watch +`application/api/events/routes.py` for changes by default. +After editing the route, restart Flask manually — `--reload` +isn't on. (Production gunicorn reloads via deploy.) + +### MCP OAuth completion can fall outside the user stream's MAXLEN window + +`get_oauth_status` scans up to `EVENTS_STREAM_MAXLEN` (~1000) entries via `XREVRANGE`. If the user has a high-rate ingest running concurrent with the OAuth handshake, the `mcp.oauth.completed` envelope can be trimmed off the back before they click Save. Symptom: backend returns "OAuth failed or not completed" even though the popup completed successfully. + +Mitigation today: bump `EVENTS_STREAM_MAXLEN` per-deployment if your users routinely flood the channel during OAuth flows. A dedicated short-TTL Redis key for OAuth task results is tracked as a follow-up. + +### React StrictMode double-mounts SSE + +In dev, React 18 StrictMode mounts → unmounts → remounts every +component, briefly opening two SSE connections per tab before the +first is aborted. With `SSE_MAX_CONCURRENT_PER_USER=8` and 4–5 +tabs open concurrently you can transiently hit the cap and see +HTTP 429 on cold-load. The first connection's counter increment +fires before the AbortController-induced disconnect can decrement +it. Production (single mount, no StrictMode) is unaffected; raise +the cap in dev or accept transient 429s. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6757c022..8e008ef3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -58,6 +58,7 @@ "eslint-plugin-promise": "^6.6.0", "eslint-plugin-react": "^7.37.5", "eslint-plugin-unused-imports": "^4.1.4", + "happy-dom": "^17.6.3", "husky": "^9.1.7", "lint-staged": "^16.4.0", "postcss": "^8.5.12", @@ -67,7 +68,8 @@ "tw-animate-css": "^1.4.0", "typescript": "^6.0.3", "vite": "^8.0.10", - "vite-plugin-svgr": "^4.3.0" + "vite-plugin-svgr": "^4.3.0", + "vitest": "^3.2.4" } }, "node_modules/@alloc/quick-lru": { @@ -451,6 +453,448 @@ "tslib": "^2.4.0" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.9.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", @@ -2870,6 +3314,356 @@ } } }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.3.tgz", + "integrity": "sha512-x35CNW/ANXG3hE/EZpRU8MXX1JDN86hBb2wMGAtltkz7pc6cxgjpy1OMMfDosOQ+2hWqIkag/fGok1Yady9nGw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.3.tgz", + "integrity": "sha512-xw3xtkDApIOGayehp2+Rz4zimfkaX65r4t47iy+ymQB2G4iJCBBfj0ogVg5jpvjpn8UWn/+q9tprxleYeNp3Hw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.3.tgz", + "integrity": "sha512-vo6Y5Qfpx7/5EaamIwi0WqW2+zfiusVihKatLvtN1VFVy3D13uERk/6gZLU1UiHRL6fDXqj/ELIeVRGnvcTE1g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.3.tgz", + "integrity": "sha512-D+0QGcZhBzTN82weOnsSlY7V7+RMmPuF1CkbxyMAGE8+ZHeUjyb76ZiWmBlCu//AQQONvxcqRbwZTajZKqjuOw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.3.tgz", + "integrity": "sha512-6HnvHCT7fDyj6R0Ph7A6x8dQS/S38MClRWeDLqc0MdfWkxjiu1HSDYrdPhqSILzjTIC/pnXbbJbo+ft+gy/9hQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.3.tgz", + "integrity": "sha512-KHLgC3WKlUYW3ShFKnnosZDOJ0xjg9zp7au3sIm2bs/tGBeC2ipmvRh/N7JKi0t9Ue20C0dpEshi8WUubg+cnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.3.tgz", + "integrity": "sha512-DV6fJoxEYWJOvaZIsok7KrYl0tPvga5OZ2yvKHNNYyk/2roMLqQAbGhr78EQ5YhHpnhLKJD3S1WFusAkmUuV5g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.3.tgz", + "integrity": "sha512-mQKoJAzvuOs6F+TZybQO4GOTSMUu7v0WdxEk24krQ/uUxXoPTtHjuaUuPmFhtBcM4K0ons8nrE3JyhTuCFtT/w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.3.tgz", + "integrity": "sha512-Whjj2qoiJ6+OOJMGptTYazaJvjOJm+iKHpXQM1P3LzGjt7Ff++Tp7nH4N8J/BUA7R9IHfDyx4DJIflifwnbmIA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.3.tgz", + "integrity": "sha512-4YTNHKqGng5+yiZt3mg77nmyuCfmNfX4fPmyUapBcIk+BdwSwmCWGXOUxhXbBEkFHtoN5boLj/5NON+u5QC9tg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.3.tgz", + "integrity": "sha512-SU3kNlhkpI4UqlUc2VXPGK9o886ZsSeGfMAX2ba2b8DKmMXq4AL7KUrkSWVbb7koVqx41Yczx6dx5PNargIrEA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.3.tgz", + "integrity": "sha512-6lDLl5h4TXpB1mTf2rQWnAk/LcXrx9vBfu/DT5TIPhvMhRWaZ5MxkIc8u4lJAmBo6klTe1ywXIUHFjylW505sg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.3.tgz", + "integrity": "sha512-BMo8bOw8evlup/8G+cj5xWtPyp93xPdyoSN16Zy90Q2QZ0ZYRhCt6ZJSwbrRzG9HApFabjwj2p25TUPDWrhzqQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.3.tgz", + "integrity": "sha512-E0L8X1dZN1/Rph+5VPF6Xj2G7JJvMACVXtamTJIDrVI44Y3K+G8gQaMEAavbqCGTa16InptiVrX6eM6pmJ+7qA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.3.tgz", + "integrity": "sha512-oZJ/WHaVfHUiRAtmTAeo3DcevNsVvH8mbvodjZy7D5QKvCefO371SiKRpxoDcCxB3PTRTLayWBkvmDQKTcX/sw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.3.tgz", + "integrity": "sha512-Dhbyh7j9FybM3YaTgaHmVALwA8AkUwTPccyCQ79TG9AJUsMQqgN1DDEZNr4+QUfwiWvLDumW5vdwzoeUF+TNxQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.3.tgz", + "integrity": "sha512-cJd1X5XhHHlltkaypz1UcWLA8AcoIi1aWhsvaWDskD1oz2eKCypnqvTQ8ykMNI0RSmm7NkTdSqSSD7zM0xa6Ig==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.3.tgz", + "integrity": "sha512-DAZDBHQfG2oQuhY7mc6I3/qB4LU2fQCjRvxbDwd/Jdvb9fypP4IJ4qmtu6lNjes6B531AI8cg1aKC2di97bUxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.3.tgz", + "integrity": "sha512-cRxsE8c13mZOh3vP+wLDxpQBRrOHDIGOWyDL93Sy0Ga8y515fBcC2pjUfFwUe5T7tqvTvWbCpg1URM/AXdWIXA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.3.tgz", + "integrity": "sha512-QaWcIgRxqEdQdhJqW4DJctsH6HCmo5vHxY0krHSX4jMtOqfzC+dqDGuHM87bu4H8JBeibWx7jFz+h6/4C8wA5Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.3.tgz", + "integrity": "sha512-AaXwSvUi3QIPtroAUw1t5yHGIyqKEXwH54WUocFolZhpGDruJcs8c+xPNDRn4XiQsS7MEwnYsHW2l0MBLDMkWg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.3.tgz", + "integrity": "sha512-65LAKM/bAWDqKNEelHlcHvm2V+Vfb8C6INFxQXRHCvaVN1rJfwr4NvdP4FyzUaLqWfaCGaadf6UbTm8xJeYfEg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.3.tgz", + "integrity": "sha512-EEM2gyhBF5MFnI6vMKdX1LAosE627RGBzIoGMdLloPZkXrUN0Ckqgr2Qi8+J3zip/8NVVro3/FjB+tjhZUgUHA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.3.tgz", + "integrity": "sha512-E5Eb5H/DpxaoXH++Qkv28RcUJboMopmdDUALBczvHMf7hNIxaDZqwY5lK12UK1BHacSmvupoEWGu+n993Z0y1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.3.tgz", + "integrity": "sha512-hPt/bgL5cE+Qp+/TPHBqptcAgPzgj46mPcg/16zNUmbQk0j+mOEQV/+Lqu8QRtDV3Ek95Q6FeFITpuhl6OTsAA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@rtsao/scc": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@rtsao/scc/-/scc-1.1.0.tgz", @@ -3473,6 +4267,17 @@ "tslib": "^2.4.0" } }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, "node_modules/@types/d3": { "version": "7.4.3", "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", @@ -3735,6 +4540,13 @@ "@types/ms": "*" } }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -3817,6 +4629,7 @@ "version": "19.2.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", + "dev": true, "license": "MIT", "dependencies": { "csstype": "^3.2.2" @@ -3826,7 +4639,7 @@ "version": "19.2.3", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", - "devOptional": true, + "dev": true, "license": "MIT", "peerDependencies": { "@types/react": "^19.2.0" @@ -4162,6 +4975,131 @@ } } }, + "node_modules/@vitest/expect": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", + "integrity": "sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.4.tgz", + "integrity": "sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "3.2.4", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/mocker/node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/@vitest/pretty-format": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", + "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.4.tgz", + "integrity": "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "3.2.4", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.4.tgz", + "integrity": "sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", + "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", + "integrity": "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -4425,6 +5363,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/async-function": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-function/-/async-function-1.0.0.tgz", @@ -4544,6 +5492,16 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/call-bind": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", @@ -4648,6 +5606,23 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -4717,6 +5692,16 @@ "pnpm": ">=8" } }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, "node_modules/chevrotain": { "version": "12.0.0", "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-12.0.0.tgz", @@ -4965,6 +5950,7 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, "license": "MIT" }, "node_modules/cytoscape": { @@ -5565,6 +6551,16 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -5870,6 +6866,13 @@ "node": ">= 0.4" } }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, "node_modules/es-object-atoms": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", @@ -5930,6 +6933,48 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -6505,6 +7550,16 @@ "dev": true, "license": "MIT" }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/exsolve": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/exsolve/-/exsolve-1.0.7.tgz", @@ -6558,6 +7613,24 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -6889,6 +7962,20 @@ "integrity": "sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==", "license": "MIT" }, + "node_modules/happy-dom": { + "version": "17.6.3", + "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-17.6.3.tgz", + "integrity": "sha512-UVIHeVhxmxedbWPCfgS55Jg2rDfwf2BCKeylcPSqazLz5w3Kri7Q4xdBJubsr/+VUzFLh0VjIvh13RaDA2/Xug==", + "dev": true, + "license": "MIT", + "dependencies": { + "webidl-conversions": "^7.0.0", + "whatwg-mimetype": "^3.0.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/has-bigints": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.1.0.tgz", @@ -8440,6 +9527,13 @@ "loose-envify": "cli.js" } }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, "node_modules/lower-case": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", @@ -9893,6 +10987,16 @@ "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", "license": "MIT" }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -10828,6 +11932,51 @@ "dev": true, "license": "MIT" }, + "node_modules/rollup": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.3.tgz", + "integrity": "sha512-pAQK9HalE84QSm4Po3EmWIZPd3FnjkShVkiMlz1iligWYkWQ7wHYd1PF/T7QZ5TVSD6uSTon5gBVMSM4JfBV+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.60.3", + "@rollup/rollup-android-arm64": "4.60.3", + "@rollup/rollup-darwin-arm64": "4.60.3", + "@rollup/rollup-darwin-x64": "4.60.3", + "@rollup/rollup-freebsd-arm64": "4.60.3", + "@rollup/rollup-freebsd-x64": "4.60.3", + "@rollup/rollup-linux-arm-gnueabihf": "4.60.3", + "@rollup/rollup-linux-arm-musleabihf": "4.60.3", + "@rollup/rollup-linux-arm64-gnu": "4.60.3", + "@rollup/rollup-linux-arm64-musl": "4.60.3", + "@rollup/rollup-linux-loong64-gnu": "4.60.3", + "@rollup/rollup-linux-loong64-musl": "4.60.3", + "@rollup/rollup-linux-ppc64-gnu": "4.60.3", + "@rollup/rollup-linux-ppc64-musl": "4.60.3", + "@rollup/rollup-linux-riscv64-gnu": "4.60.3", + "@rollup/rollup-linux-riscv64-musl": "4.60.3", + "@rollup/rollup-linux-s390x-gnu": "4.60.3", + "@rollup/rollup-linux-x64-gnu": "4.60.3", + "@rollup/rollup-linux-x64-musl": "4.60.3", + "@rollup/rollup-openbsd-x64": "4.60.3", + "@rollup/rollup-openharmony-arm64": "4.60.3", + "@rollup/rollup-win32-arm64-msvc": "4.60.3", + "@rollup/rollup-win32-ia32-msvc": "4.60.3", + "@rollup/rollup-win32-x64-gnu": "4.60.3", + "@rollup/rollup-win32-x64-msvc": "4.60.3", + "fsevents": "~2.3.2" + } + }, "node_modules/roughjs": { "version": "4.6.6", "resolved": "https://registry.npmjs.org/roughjs/-/roughjs-4.6.6.tgz", @@ -11080,6 +12229,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, "node_modules/signal-exit": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", @@ -11154,6 +12310,20 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, "node_modules/stop-iteration-iterator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.1.0.tgz", @@ -11346,6 +12516,26 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strip-literal": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", + "integrity": "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tokens": "^9.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/strip-literal/node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true, + "license": "MIT" + }, "node_modules/style-to-js": { "version": "1.1.19", "resolved": "https://registry.npmjs.org/style-to-js/-/style-to-js-1.1.19.tgz", @@ -11450,6 +12640,13 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, "node_modules/tinyexec": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.4.tgz", @@ -11476,22 +12673,34 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, - "node_modules/tinyglobby/node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", "dev": true, "license": "MIT", "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.4.tgz", + "integrity": "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" } }, "node_modules/toggle-selection": { @@ -11702,7 +12911,7 @@ "version": "6.0.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -12078,6 +13287,104 @@ } } }, + "node_modules/vite-node": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", + "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.1", + "es-module-lexer": "^1.7.0", + "pathe": "^2.0.3", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vite-node/node_modules/vite": { + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.3.tgz", + "integrity": "sha512-/4XH147Ui7OGTjg3HbdWe5arnZQSbfuRzdr9Ec7TQi5I7R+ir0Rlc9GIvD4v0XZurELqA035KVXJXpR61xhiTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, "node_modules/vite-plugin-svgr": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/vite-plugin-svgr/-/vite-plugin-svgr-4.5.0.tgz", @@ -12093,6 +13400,161 @@ "vite": ">=2.6.0" } }, + "node_modules/vitest": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.4.tgz", + "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.4", + "@vitest/mocker": "3.2.4", + "@vitest/pretty-format": "^3.2.4", + "@vitest/runner": "3.2.4", + "@vitest/snapshot": "3.2.4", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.4", + "@vitest/ui": "3.2.4", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/vitest/node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/vitest/node_modules/vite": { + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.3.tgz", + "integrity": "sha512-/4XH147Ui7OGTjg3HbdWe5arnZQSbfuRzdr9Ec7TQi5I7R+ir0Rlc9GIvD4v0XZurELqA035KVXJXpR61xhiTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, "node_modules/void-elements": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/void-elements/-/void-elements-3.1.0.tgz", @@ -12161,6 +13623,26 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-mimetype": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -12266,6 +13748,23 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", diff --git a/frontend/package.json b/frontend/package.json index b374a4ec..e5396327 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -7,6 +7,8 @@ "dev": "vite", "build": "tsc && vite build", "preview": "vite preview", + "test": "vitest run", + "test:watch": "vitest", "lint": "eslint ./src --ext .jsx,.js,.ts,.tsx", "lint-fix": "eslint ./src --ext .jsx,.js,.ts,.tsx --fix", "format": "prettier ./src --write", @@ -69,6 +71,7 @@ "eslint-plugin-promise": "^6.6.0", "eslint-plugin-react": "^7.37.5", "eslint-plugin-unused-imports": "^4.1.4", + "happy-dom": "^17.6.3", "husky": "^9.1.7", "lint-staged": "^16.4.0", "postcss": "^8.5.12", @@ -78,6 +81,7 @@ "tw-animate-css": "^1.4.0", "typescript": "^6.0.3", "vite": "^8.0.10", - "vite-plugin-svgr": "^4.3.0" + "vite-plugin-svgr": "^4.3.0", + "vitest": "^3.2.4" } } diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index d6d6e332..382d6abb 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -10,6 +10,7 @@ import Spinner from './components/Spinner'; import UploadToast from './components/UploadToast'; import Conversation from './conversation/Conversation'; import { SharedConversation } from './conversation/SharedConversation'; +import { EventStreamProvider } from './events/EventStreamProvider'; import { useDarkTheme, useMediaQuery } from './hooks'; import useDataInitializer from './hooks/useDataInitializer'; import useTokenAuth from './hooks/useTokenAuth'; @@ -17,6 +18,7 @@ import Navigation from './Navigation'; import PageNotFound from './PageNotFound'; import Setting from './settings'; import Notification from './components/Notification'; +import ToolApprovalToast from './notifications/ToolApprovalToast'; function AuthWrapper({ children }: { children: React.ReactNode }) { const { isAuthLoading } = useTokenAuth(); @@ -29,7 +31,7 @@ function AuthWrapper({ children }: { children: React.ReactNode }) { ); } - return <>{children}; + return {children}; } function MainLayout() { @@ -50,6 +52,7 @@ function MainLayout() { + ); } diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index 07685d1f..f3d251e8 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -28,7 +28,6 @@ const endpoints = { UPDATE_PROMPT: '/api/update_prompt', SINGLE_PROMPT: (id: string) => `/api/get_single_prompt?id=${id}`, DELETE_PATH: (docPath: string) => `/api/delete_old?source_id=${docPath}`, - TASK_STATUS: (task_id: string) => `/api/task_status?task_id=${task_id}`, MESSAGE_ANALYTICS: '/api/get_message_analytics', TOKEN_ANALYTICS: '/api/get_token_analytics', FEEDBACK_ANALYTICS: '/api/get_feedback_analytics', @@ -73,8 +72,6 @@ const endpoints = { MANAGE_SOURCE_FILES: '/api/manage_source_files', MCP_TEST_CONNECTION: '/api/mcp_server/test', MCP_SAVE_SERVER: '/api/mcp_server/save', - MCP_OAUTH_STATUS: (task_id: string) => - `/api/mcp_server/oauth_status/${task_id}`, MCP_AUTH_STATUS: '/api/mcp_server/auth_status', AGENT_FOLDERS: '/api/agents/folders/', AGENT_FOLDER: (id: string) => `/api/agents/folders/${id}`, diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index 28488f63..70afb719 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -61,8 +61,6 @@ const userService = { apiClient.get(endpoints.USER.SINGLE_PROMPT(id), token), deletePath: (docPath: string, token: string | null): Promise => apiClient.get(endpoints.USER.DELETE_PATH(docPath), token), - getTaskStatus: (task_id: string, token: string | null): Promise => - throttledApiClient.get(endpoints.USER.TASK_STATUS(task_id), token), getMessageAnalytics: (data: any, token: string | null): Promise => apiClient.post(endpoints.USER.MESSAGE_ANALYTICS, data, token), getTokenAnalytics: (data: any, token: string | null): Promise => @@ -172,8 +170,6 @@ const userService = { apiClient.post(endpoints.USER.MCP_TEST_CONNECTION, data, token), saveMCPServer: (data: any, token: string | null): Promise => apiClient.post(endpoints.USER.MCP_SAVE_SERVER, data, token), - getMCPOAuthStatus: (task_id: string, token: string | null): Promise => - throttledApiClient.get(endpoints.USER.MCP_OAUTH_STATUS(task_id), token), getMCPAuthStatus: (token: string | null): Promise => throttledApiClient.get(endpoints.USER.MCP_AUTH_STATUS, token), syncConnector: ( diff --git a/frontend/src/components/ConnectorAuth.tsx b/frontend/src/components/ConnectorAuth.tsx index 951e71fb..5787ba29 100644 --- a/frontend/src/components/ConnectorAuth.tsx +++ b/frontend/src/components/ConnectorAuth.tsx @@ -1,4 +1,4 @@ -import React, { useRef } from 'react'; +import React, { useEffect, useRef } from 'react'; import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; @@ -32,13 +32,24 @@ const ConnectorAuth: React.FC = ({ const [isDarkTheme] = useDarkTheme(); const completedRef = useRef(false); const intervalRef = useRef(null); + const authWindowRef = useRef(null); + // Hold the exact listener identity so unmount cleanup removes the same fn. + const messageHandlerRef = useRef<((event: MessageEvent) => void) | null>( + null, + ); + // Tracks mount status so async ``fetch`` resolves after unmount don't + // call ``onSuccess`` / ``onError`` on a vanished parent. + const mountedRef = useRef(true); const cleanup = () => { if (intervalRef.current) { clearInterval(intervalRef.current); intervalRef.current = null; } - window.removeEventListener('message', handleAuthMessage as any); + if (messageHandlerRef.current) { + window.removeEventListener('message', messageHandlerRef.current as any); + messageHandlerRef.current = null; + } }; const handleAuthMessage = (event: MessageEvent) => { @@ -49,6 +60,7 @@ const ConnectorAuth: React.FC = ({ if (successGeneric || successProvider) { completedRef.current = true; cleanup(); + authWindowRef.current = null; onSuccess({ session_token: event.data.session_token, user_email: @@ -58,6 +70,7 @@ const ConnectorAuth: React.FC = ({ } else if (errorProvider) { completedRef.current = true; cleanup(); + authWindowRef.current = null; onError( event.data.error || t('modals.uploadDoc.connectors.auth.authFailed'), ); @@ -67,12 +80,20 @@ const ConnectorAuth: React.FC = ({ const handleAuth = async () => { try { completedRef.current = false; + // Close any popup left over from a previous click before wiping + // the ref — otherwise the old window keeps living with no + // interval watching it and no listener handling its messages. + if (authWindowRef.current && !authWindowRef.current.closed) { + authWindowRef.current.close(); + } + authWindowRef.current = null; cleanup(); const authResponse = await userService.getConnectorAuthUrl( provider, token, ); + if (!mountedRef.current) return; if (!authResponse.ok) { throw new Error( @@ -81,6 +102,7 @@ const ConnectorAuth: React.FC = ({ } const authData = await authResponse.json(); + if (!mountedRef.current) return; if (!authData.success || !authData.authorization_url) { throw new Error( authData.error || t('modals.uploadDoc.connectors.auth.authUrlFailed'), @@ -95,13 +117,23 @@ const ConnectorAuth: React.FC = ({ if (!authWindow) { throw new Error(t('modals.uploadDoc.connectors.auth.popupBlocked')); } + authWindowRef.current = authWindow; + messageHandlerRef.current = handleAuthMessage; window.addEventListener('message', handleAuthMessage as any); const checkClosed = window.setInterval(() => { if (authWindow.closed) { clearInterval(checkClosed); - window.removeEventListener('message', handleAuthMessage as any); + intervalRef.current = null; + if (messageHandlerRef.current) { + window.removeEventListener( + 'message', + messageHandlerRef.current as any, + ); + messageHandlerRef.current = null; + } + authWindowRef.current = null; if (!completedRef.current) { onError(t('modals.uploadDoc.connectors.auth.authCancelled')); } @@ -109,6 +141,7 @@ const ConnectorAuth: React.FC = ({ }, 1000); intervalRef.current = checkClosed; } catch (error) { + if (!mountedRef.current) return; onError( error instanceof Error ? error.message @@ -117,6 +150,18 @@ const ConnectorAuth: React.FC = ({ } }; + // Release interval, message listener, and popup on unmount only. + useEffect(() => { + return () => { + mountedRef.current = false; + cleanup(); + if (authWindowRef.current && !authWindowRef.current.closed) { + authWindowRef.current.close(); + } + authWindowRef.current = null; + }; + }, []); + return ( <> {errorMessage && ( diff --git a/frontend/src/components/ConnectorTree.tsx b/frontend/src/components/ConnectorTree.tsx index 81b0b282..f8354116 100644 --- a/frontend/src/components/ConnectorTree.tsx +++ b/frontend/src/components/ConnectorTree.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import { useSelector } from 'react-redux'; +import { useSelector, useStore } from 'react-redux'; import userService from '../api/services/userService'; import ArrowLeft from '../assets/arrow-left.svg'; @@ -14,6 +14,7 @@ import { useLoaderState, useOutsideAlerter } from '../hooks'; import ConfirmationModal from '../modals/ConfirmationModal'; import { ActiveState } from '../models/misc'; import { selectToken } from '../preferences/preferenceSlice'; +import type { RootState } from '../store'; import { formatBytes } from '../utils/stringUtils'; import Chunks from './Chunks'; import ContextMenu, { MenuOption } from './ContextMenu'; @@ -64,6 +65,7 @@ const ConnectorTree: React.FC = ({ useState(null); const [currentPath, setCurrentPath] = useState([]); const token = useSelector(selectToken); + const store = useStore(); const [activeMenuId, setActiveMenuId] = useState(null); const menuRefs = useRef<{ [key: string]: React.RefObject; @@ -81,6 +83,25 @@ const ConnectorTree: React.FC = ({ const [syncDone, setSyncDone] = useState(false); const [syncConfirmationModal, setSyncConfirmationModal] = useState('INACTIVE'); + const mountedRef = useRef(true); + const syncUnsubscribeRef = useRef<(() => void) | null>(null); + // Holds the 5-minute SSE-wait timer so the unmount cleanup can clear + // it — otherwise the timer fires up to 5 min after unmount and + // resolves an abandoned Promise. + const syncTimerRef = useRef(null); + + useEffect( + () => () => { + mountedRef.current = false; + syncUnsubscribeRef.current?.(); + syncUnsubscribeRef.current = null; + if (syncTimerRef.current !== null) { + window.clearTimeout(syncTimerRef.current); + syncTimerRef.current = null; + } + }, + [], + ); useOutsideAlerter( searchDropdownRef, @@ -116,67 +137,108 @@ const ConnectorTree: React.FC = ({ console.log('Sync started successfully:', data.task_id); setSyncProgress(10); - // Poll task status using userService - const maxAttempts = 30; - const pollInterval = 2000; + // The connector worker (``ingest_connector`` in + // ``application/worker.py``) publishes + // ``source.ingest.{queued,completed,failed}`` envelopes keyed on + // ``scope.id == docId`` (sync mode reuses the source uuid). Wait + // on the bounded ``notifications.recentEvents`` ring for a + // terminal envelope rather than polling ``/task_status``. + // Mirrors FileTree's slice-walking pattern, including the + // ``opStartedAt`` guard so a stale terminal event from a prior + // sync of this same source can't short-circuit the current op. + const opStartedAt = Date.now(); + + const terminalFromSse = (): 'completed' | 'failed' | null => { + const events = store.getState().notifications.recentEvents; + for (const event of events) { + if (event.scope?.id !== docId) continue; + const ts = event.ts ? Date.parse(event.ts) : NaN; + if (!Number.isFinite(ts) || ts < opStartedAt) continue; + if (event.type === 'source.ingest.completed') return 'completed'; + if (event.type === 'source.ingest.failed') return 'failed'; + } + return null; + }; + + const MAX_WAIT_MS = 5 * 60_000; + const terminal = await new Promise< + 'completed' | 'failed' | 'timeout' | 'unmounted' + >((resolve) => { + // Cover the race where the event landed between the POST + // returning and the subscribe call. + const initial = terminalFromSse(); + if (initial) { + resolve(initial); + return; + } + if (!mountedRef.current) { + resolve('unmounted'); + return; + } + let settled = false; + const finish = ( + value: 'completed' | 'failed' | 'timeout' | 'unmounted', + ) => { + if (settled) return; + settled = true; + if (syncTimerRef.current !== null) { + window.clearTimeout(syncTimerRef.current); + syncTimerRef.current = null; + } + if (syncUnsubscribeRef.current) { + syncUnsubscribeRef.current(); + syncUnsubscribeRef.current = null; + } + resolve(value); + }; + syncTimerRef.current = window.setTimeout( + () => finish('timeout'), + MAX_WAIT_MS, + ); + syncUnsubscribeRef.current = store.subscribe(() => { + if (!mountedRef.current) { + finish('unmounted'); + return; + } + const next = terminalFromSse(); + if (next) finish(next); + }); + }); + + if (terminal === 'timeout') { + console.error('Sync timed out waiting for SSE terminal'); + } else if (terminal === 'unmounted') { + return; + } + + if (terminal === 'completed') { + // The "no files downloaded" early-return path publishes + // ``completed`` with ``no_changes: true`` — treated as success + // here; refreshing the directory is cheap and idempotent. + setSyncProgress(100); + console.log('Sync completed successfully'); - for (let attempt = 0; attempt < maxAttempts; attempt++) { try { - const statusResponse = await userService.getTaskStatus( - data.task_id, + const refreshResponse = await userService.getDirectoryStructure( + docId, token, ); - const statusData = await statusResponse.json(); - - console.log( - `Task status (attempt ${attempt + 1}):`, - statusData.status, - ); - - if (statusData.status === 'SUCCESS') { - setSyncProgress(100); - console.log('Sync completed successfully'); - - // Refresh directory structure - try { - const refreshResponse = await userService.getDirectoryStructure( - docId, - token, - ); - const refreshData = await refreshResponse.json(); - if (refreshData && refreshData.directory_structure) { - setDirectoryStructure(refreshData.directory_structure); - setCurrentPath([]); - } - if (refreshData && refreshData.provider) { - setSourceProvider(refreshData.provider); - } - - setSyncDone(true); - setTimeout(() => setSyncDone(false), 5000); - } catch (err) { - console.error('Error refreshing directory structure:', err); - } - break; - } else if (statusData.status === 'FAILURE') { - console.error('Sync task failed:', statusData.result); - break; - } else if (statusData.status === 'PROGRESS') { - const progress = Number( - statusData.result && statusData.result.current != null - ? statusData.result.current - : statusData.meta && statusData.meta.current != null - ? statusData.meta.current - : 0, - ); - setSyncProgress(Math.max(10, progress)); + const refreshData = await refreshResponse.json(); + if (refreshData && refreshData.directory_structure) { + setDirectoryStructure(refreshData.directory_structure); + setCurrentPath([]); + } + if (refreshData && refreshData.provider) { + setSourceProvider(refreshData.provider); } - await new Promise((resolve) => setTimeout(resolve, pollInterval)); - } catch (error) { - console.error('Error polling task status:', error); - break; + setSyncDone(true); + setTimeout(() => setSyncDone(false), 5000); + } catch (err) { + console.error('Error refreshing directory structure:', err); } + } else if (terminal === 'failed') { + console.error('Sync task failed (per SSE)'); } } else { console.error('Sync failed:', data.error); diff --git a/frontend/src/components/FileTree.tsx b/frontend/src/components/FileTree.tsx index 503ab6fc..04626051 100644 --- a/frontend/src/components/FileTree.tsx +++ b/frontend/src/components/FileTree.tsx @@ -1,7 +1,8 @@ import React, { useState, useRef, useEffect } from 'react'; import { useTranslation } from 'react-i18next'; -import { useSelector } from 'react-redux'; +import { useSelector, useStore } from 'react-redux'; import { selectToken } from '../preferences/preferenceSlice'; +import type { RootState } from '../store'; import { formatBytes } from '../utils/stringUtils'; import Chunks from './Chunks'; import ContextMenu, { MenuOption } from './ContextMenu'; @@ -56,6 +57,7 @@ const FileTree: React.FC = ({ onBackToDocuments, }) => { const { t } = useTranslation(); + const store = useStore(); const [loading, setLoading] = useLoaderState(true, 500); const [error, setError] = useState(null); const [directoryStructure, setDirectoryStructure] = @@ -95,6 +97,25 @@ const FileTree: React.FC = ({ const opQueueRef = useRef([]); const processingRef = useRef(false); const [queueLength, setQueueLength] = useState(0); + const mountedRef = useRef(true); + const waitUnsubscribeRef = useRef<(() => void) | null>(null); + // Holds the 5-minute SSE-wait timer so the unmount cleanup can clear + // it — otherwise the timer fires up to 5 min after unmount and + // resolves an abandoned Promise. + const waitTimerRef = useRef(null); + + useEffect( + () => () => { + mountedRef.current = false; + waitUnsubscribeRef.current?.(); + waitUnsubscribeRef.current = null; + if (waitTimerRef.current !== null) { + window.clearTimeout(waitTimerRef.current); + waitTimerRef.current = null; + } + }, + [], + ); useOutsideAlerter( searchDropdownRef, @@ -313,47 +334,103 @@ const FileTree: React.FC = ({ } console.log('Reingest task started:', result.reingest_task_id); - const maxAttempts = 30; - const pollInterval = 2000; + // SSE is the sole driver here. The backend's + // ``reingest_source_worker`` publishes ``source.ingest.*`` + // keyed on the resolved ``source_id`` (the + // ``manage_source_files`` route returns it explicitly so we + // can match without consulting any slice). Subscribe to the + // store and resolve when a terminal event tagged with our + // source lands in ``notifications.recentEvents``. Re-checking + // on every dispatch (rather than polling on a timer) avoids + // races where a terminal could roll off the bounded ring + // before the next tick observes it in chatty sessions. + const reingestSourceId: string | undefined = result.source_id; + // Cutoff so we don't pick up terminal events from a *previous* + // reingest of the same source — the backend's + // ``source.ingest.*`` payload doesn't carry a Celery task id, + // so source_id alone is ambiguous when ops repeat. + const opStartedAt = Date.now(); + const MAX_WAIT_MS = 5 * 60_000; - for (let attempt = 0; attempt < maxAttempts; attempt++) { - try { - const statusResponse = await userService.getTaskStatus( - result.reingest_task_id, - token, - ); - const statusData = await statusResponse.json(); - - console.log( - `Task status (attempt ${attempt + 1}):`, - statusData.status, - ); - - if (statusData.status === 'SUCCESS') { - console.log('Task completed successfully'); - - const structureResponse = await userService.getDirectoryStructure( - docId, - token, - ); - const structureData = await structureResponse.json(); - - if (structureData && structureData.directory_structure) { - setDirectoryStructure(structureData.directory_structure); - currentOpRef.current = null; - return true; - } - break; - } else if (statusData.status === 'FAILURE') { - console.error('Task failed'); - break; - } - - await new Promise((resolve) => setTimeout(resolve, pollInterval)); - } catch (error) { - console.error('Error polling task status:', error); - break; + const terminalFromSse = (): 'completed' | 'failed' | null => { + if (!reingestSourceId) return null; + const events = store.getState().notifications.recentEvents; + for (const event of events) { + if (event.scope?.id !== reingestSourceId) continue; + const ts = event.ts ? Date.parse(event.ts) : NaN; + if (!Number.isFinite(ts) || ts < opStartedAt) continue; + if (event.type === 'source.ingest.completed') return 'completed'; + if (event.type === 'source.ingest.failed') return 'failed'; } + return null; + }; + + const refreshStructure = async (): Promise => { + const structureResponse = await userService.getDirectoryStructure( + docId, + token, + ); + const structureData = await structureResponse.json(); + if (!mountedRef.current) return false; + if (structureData && structureData.directory_structure) { + setDirectoryStructure(structureData.directory_structure); + currentOpRef.current = null; + return true; + } + return false; + }; + + const terminal = await new Promise< + 'completed' | 'failed' | 'timeout' | 'unmounted' + >((resolve) => { + if (!mountedRef.current) { + resolve('unmounted'); + return; + } + // Cover the race where the terminal event landed between + // the POST returning and the subscribe call. + const initial = terminalFromSse(); + if (initial) { + resolve(initial); + return; + } + const timer = window.setTimeout(() => { + waitUnsubscribeRef.current?.(); + waitUnsubscribeRef.current = null; + waitTimerRef.current = null; + resolve('timeout'); + }, MAX_WAIT_MS); + waitTimerRef.current = timer; + waitUnsubscribeRef.current = store.subscribe(() => { + if (!mountedRef.current) { + window.clearTimeout(timer); + waitTimerRef.current = null; + waitUnsubscribeRef.current?.(); + waitUnsubscribeRef.current = null; + resolve('unmounted'); + return; + } + const next = terminalFromSse(); + if (next) { + window.clearTimeout(timer); + waitTimerRef.current = null; + waitUnsubscribeRef.current?.(); + waitUnsubscribeRef.current = null; + resolve(next); + } + }); + }); + + if (!mountedRef.current) return false; + + if (terminal === 'completed') { + if (await refreshStructure()) return true; + } else if (terminal === 'failed') { + console.error('Reingest task failed (per SSE)'); + } else if (terminal === 'unmounted') { + return false; + } else { + console.error('Reingest timed out waiting for SSE terminal'); } } else { throw new Error( @@ -374,7 +451,7 @@ const FileTree: React.FC = ({ ? 'delete directory' : 'delete file(s)'; console.error(`Error ${actionText}:`, error); - setError(`Failed to ${errorText}`); + if (mountedRef.current) setError(`Failed to ${errorText}`); } finally { currentOpRef.current = null; } diff --git a/frontend/src/components/MessageInput.tsx b/frontend/src/components/MessageInput.tsx index 3d52a75b..1f066154 100644 --- a/frontend/src/components/MessageInput.tsx +++ b/frontend/src/components/MessageInput.tsx @@ -3,7 +3,7 @@ import { createPortal } from 'react-dom'; import { LoaderCircle, Mic, Square } from 'lucide-react'; import { useDropzone } from 'react-dropzone'; import { useTranslation } from 'react-i18next'; -import { useDispatch, useSelector } from 'react-redux'; +import { useDispatch, useSelector, useStore } from 'react-redux'; import endpoints from '../api/endpoints'; import userService from '../api/services/userService'; @@ -28,6 +28,7 @@ import { selectSelectedDocs, selectToken, } from '../preferences/preferenceSlice'; +import type { RootState } from '../store'; import Upload from '../upload/Upload'; import { getOS, isTouchDevice } from '../utils/browserUtils'; import SourcesPopup from './SourcesPopup'; @@ -316,6 +317,7 @@ export default function MessageInput({ const attachments = useSelector(selectAttachments); const dispatch = useDispatch(); + const store = useStore(); const mediaStreamRef = useRef(null); const audioContextRef = useRef(null); const audioSourceNodeRef = useRef(null); @@ -410,6 +412,86 @@ export default function MessageInput({ }; }, []); + // Recover the race where attachment.* SSE arrives before the upload + // XHR's onload sets ``attachmentId``: walk recentEvents and watchdog + // the row so it can't stay stuck on 'processing'. Mirrors + // Upload.tsx's ``trackTraining``. + const trackAttachment = useCallback( + (clientId: string, attachmentId: string) => { + let handled = false; + + const check = () => { + const state = store.getState(); + const row = state.upload.attachments.find((a) => a.id === clientId); + if (!row) return true; // removed by user; stop tracking + if (row.status === 'completed' || row.status === 'failed') { + handled = true; + return true; + } + for (const event of state.notifications.recentEvents) { + if (event.scope?.id !== attachmentId) continue; + if (event.type === 'attachment.completed') { + const payload = (event.payload || {}) as Record; + const tokenCount = Number(payload.token_count); + handled = true; + dispatch( + updateAttachment({ + id: clientId, + updates: { + status: 'completed', + progress: 100, + ...(Number.isFinite(tokenCount) + ? { token_count: tokenCount } + : {}), + }, + }), + ); + return true; + } + if (event.type === 'attachment.failed') { + handled = true; + dispatch( + updateAttachment({ + id: clientId, + updates: { status: 'failed' }, + }), + ); + return true; + } + } + return false; + }; + + if (check()) return; + const MAX_WAIT_MS = 5 * 60_000; + let unsubscribe: (() => void) | null = null; + const timer = window.setTimeout(() => { + unsubscribe?.(); + if (!handled) { + handled = true; + console.warn( + 'trackAttachment: timed out waiting for terminal SSE', + clientId, + attachmentId, + ); + dispatch( + updateAttachment({ + id: clientId, + updates: { status: 'failed' }, + }), + ); + } + }, MAX_WAIT_MS); + unsubscribe = store.subscribe(() => { + if (check()) { + window.clearTimeout(timer); + unsubscribe?.(); + } + }); + }, + [dispatch, store], + ); + const uploadFiles = useCallback( (files: File[]) => { if (!files || files.length === 0) return; @@ -510,11 +592,19 @@ export default function MessageInput({ id: uiId, updates: { taskId: task.task_id, + // Stash the server's attachment id so SSE + // ``attachment.*`` events can match this + // row by ``scope.id`` and drive the + // per-attachment push-fresh poll gate. + attachmentId: task.attachment_id, status: 'processing', progress: 10, }, }), ); + if (task.attachment_id) { + trackAttachment(uiId, task.attachment_id); + } return; } @@ -545,11 +635,15 @@ export default function MessageInput({ id: uiId, updates: { taskId: t.task_id, + attachmentId: t.attachment_id, status: 'processing', progress: 10, }, }), ); + if (t.attachment_id) { + trackAttachment(uiId, t.attachment_id); + } } else { dispatch( updateAttachment({ @@ -583,11 +677,15 @@ export default function MessageInput({ id: uiId, updates: { taskId: response.task_id, + attachmentId: response.attachment_id, status: 'processing', progress: 10, }, }), ); + if (response.attachment_id) { + trackAttachment(uiId, response.attachment_id); + } } } else { console.warn( @@ -714,11 +812,15 @@ export default function MessageInput({ id: uniqueId, updates: { taskId: response.task_id, + attachmentId: response.attachment_id, status: 'processing', progress: 10, }, }), ); + if (response.attachment_id) { + trackAttachment(uniqueId, response.attachment_id); + } } else { // If backend returned tasks[] for single-file, handle gracefully: if ( @@ -730,11 +832,15 @@ export default function MessageInput({ id: uniqueId, updates: { taskId: response.tasks[0].task_id, + attachmentId: response.tasks[0].attachment_id, status: 'processing', progress: 10, }, }), ); + if (response.tasks[0].attachment_id) { + trackAttachment(uniqueId, response.tasks[0].attachment_id); + } } else { dispatch( updateAttachment({ @@ -781,7 +887,7 @@ export default function MessageInput({ xhr.send(formData); }); }, - [dispatch, token], + [dispatch, token, trackAttachment], ); const handleFileAttachment = (e: React.ChangeEvent) => { @@ -816,65 +922,6 @@ export default function MessageInput({ accept: FILE_UPLOAD_ACCEPT, }); - useEffect(() => { - const checkTaskStatus = () => { - const processingAttachments = attachments.filter( - (att) => att.status === 'processing' && att.taskId, - ); - - processingAttachments.forEach((attachment) => { - userService - .getTaskStatus(attachment.taskId!, null) - .then((data) => data.json()) - .then((data) => { - if (data.status === 'SUCCESS') { - dispatch( - updateAttachment({ - id: attachment.id, - updates: { - status: 'completed', - progress: 100, - id: data.result?.attachment_id, - token_count: data.result?.token_count, - }, - }), - ); - } else if (data.status === 'FAILURE') { - dispatch( - updateAttachment({ - id: attachment.id, - updates: { status: 'failed' }, - }), - ); - } else if (data.status === 'PROGRESS' && data.result?.current) { - dispatch( - updateAttachment({ - id: attachment.id, - updates: { progress: data.result.current }, - }), - ); - } - }) - .catch(() => { - dispatch( - updateAttachment({ - id: attachment.id, - updates: { status: 'failed' }, - }), - ); - }); - }); - }; - - const interval = setInterval(() => { - if (attachments.some((att) => att.status === 'processing')) { - checkTaskStatus(); - } - }, 2000); - - return () => clearInterval(interval); - }, [attachments, dispatch]); - const handleInput = useCallback(() => { if (inputRef.current) { if (window.innerWidth < 350) inputRef.current.style.height = 'auto'; diff --git a/frontend/src/components/UploadToast.tsx b/frontend/src/components/UploadToast.tsx index 7312d7b5..454727b6 100644 --- a/frontend/src/components/UploadToast.tsx +++ b/frontend/src/components/UploadToast.tsx @@ -5,41 +5,54 @@ import { useDispatch, useSelector } from 'react-redux'; import CheckCircleFilled from '../assets/check-circle-filled.svg'; import ChevronDown from '../assets/chevron-down.svg'; import WarnIcon from '../assets/warn.svg'; -import { dismissUploadTask, selectUploadTasks } from '../upload/uploadSlice'; +import { + dismissUploadTask, + selectUploadTasks, + type UploadTask, +} from '../upload/uploadSlice'; const PROGRESS_RADIUS = 10; const PROGRESS_CIRCUMFERENCE = 2 * Math.PI * PROGRESS_RADIUS; -export default function UploadToast() { - const [collapsedTasks, setCollapsedTasks] = useState>( - {}, - ); +const IN_PROGRESS_STATUSES = new Set([ + 'preparing', + 'uploading', + 'training', +]); - const toggleTaskCollapse = (taskId: string) => { - setCollapsedTasks((prev) => ({ - ...prev, - [taskId]: !prev[taskId], - })); - }; +/** + * Single merged upload card — Google-Drive style. Multiple in-flight + * uploads share one toast with a list of rows; the header reflects + * the *primary* task's status (the newest still-running task, or the + * newest task overall if all are terminal). Per-task progress lives + * on each row. + * + * Dismissal: the header X dismisses every visible task at once + * (mirrors the GDrive panel close — keeps the surface tidy without + * per-row controls). The chevron collapses the row list. + */ +export default function UploadToast() { + const [collapsed, setCollapsed] = useState(false); const { t } = useTranslation(); const dispatch = useDispatch(); const uploadTasks = useSelector(selectUploadTasks); - const getStatusHeading = (status: string) => { - switch (status) { - case 'preparing': - return t('modals.uploadDoc.progress.wait'); - case 'uploading': - return t('modals.uploadDoc.progress.upload'); - case 'training': - return t('modals.uploadDoc.progress.upload'); - case 'completed': - return t('modals.uploadDoc.progress.completed'); - case 'failed': - return t('modals.uploadDoc.progress.failed'); - default: - return t('modals.uploadDoc.progress.preparing'); + const visibleTasks = uploadTasks.filter((task) => !task.dismissed); + if (visibleTasks.length === 0) return null; + + // Pick the task that drives the header status: prefer a still- + // running task (most-recent first since the slice unshifts), and + // fall back to whatever's most-recent if everything is terminal. + const primaryTask = + visibleTasks.find((task) => IN_PROGRESS_STATUSES.has(task.status)) ?? + visibleTasks[0]; + + const headerLabel = getStatusHeading(primaryTask.status, t); + + const dismissAll = () => { + for (const task of visibleTasks) { + dispatch(dismissUploadTask(task.id)); } }; @@ -47,180 +60,205 @@ export default function UploadToast() {
e.stopPropagation()} + role="status" + aria-live="polite" + aria-atomic="false" > - {uploadTasks - .filter((task) => !task.dismissed) - .map((task) => { - const shouldShowProgress = [ - 'preparing', - 'uploading', - 'training', - ].includes(task.status); - const rawProgress = Math.min(Math.max(task.progress ?? 0, 0), 100); - const formattedProgress = Math.round(rawProgress); - const progressOffset = - PROGRESS_CIRCUMFERENCE * (1 - rawProgress / 100); - const isCollapsed = collapsedTasks[task.id] ?? false; - - return ( -
+
+

+ {headerLabel} +

+
+ - -
-
+ + + +
+
-
-
-
-

- {task.fileName} -

- -
- {shouldShowProgress && ( - - - - - )} - - {task.status === 'completed' && ( - - )} - - {task.status === 'failed' && ( - - )} -
-
- - {task.status === 'failed' && task.errorMessage && ( - - {task.errorMessage} - - )} -
-
- - - ); - })} +
+
+
    + {visibleTasks.map((task) => ( + + ))} +
+
+
+ ); } + +function UploadRow({ + task, + t, +}: { + task: UploadTask; + t: ReturnType['t']; +}) { + const showProgress = IN_PROGRESS_STATUSES.has(task.status); + const rawProgress = Math.min(Math.max(task.progress ?? 0, 0), 100); + const formattedProgress = Math.round(rawProgress); + const progressOffset = PROGRESS_CIRCUMFERENCE * (1 - rawProgress / 100); + + return ( +
  • +
    +

    + {task.fileName} +

    + +
    + {showProgress && ( + + + + + )} + + {task.status === 'completed' && ( + + )} + + {task.status === 'failed' && ( + + )} +
    +
    + + {task.status === 'failed' && + (task.tokenLimitReached || task.errorMessage) && ( + + {task.tokenLimitReached + ? t('modals.uploadDoc.progress.tokenLimit') + : task.errorMessage} + + )} +
    • + ); +} + +function getStatusHeading( + status: UploadTask['status'], + t: ReturnType['t'], +): string { + switch (status) { + case 'preparing': + return t('modals.uploadDoc.progress.wait'); + case 'uploading': + case 'training': + return t('modals.uploadDoc.progress.upload'); + case 'completed': + return t('modals.uploadDoc.progress.completed'); + case 'failed': + return t('modals.uploadDoc.progress.failed'); + default: + return t('modals.uploadDoc.progress.preparing'); + } +} diff --git a/frontend/src/conversation/conversationHandlers.ts b/frontend/src/conversation/conversationHandlers.ts index 5366d38d..0ca0110c 100644 --- a/frontend/src/conversation/conversationHandlers.ts +++ b/frontend/src/conversation/conversationHandlers.ts @@ -1,8 +1,155 @@ +import { baseURL } from '../api/client'; import conversationService from '../api/services/conversationService'; import { Doc } from '../models/misc'; import { Answer, FEEDBACK, RetrievalPayload } from './conversationModels'; import { ToolCallsType } from './types'; +/** + * Mirrors the backend's ``_SEQUENCE_NO_RE`` (application/api/answer/ + * routes/messages.py) — only non-negative decimal integers are valid + * cursors. Rejects empty strings (Number("") === 0), hex literals, + * exponential notation, and anything else that ``Number(...)`` would + * happily coerce. + */ +const _SEQUENCE_NO_RE = /^\d+$/; + +/** + * Drain an SSE response body, forwarding each ``data:`` line to + * ``onData`` and tracking the most recent ``id:`` header. Returns + * when the body ends, the signal aborts, or ``shouldStop()`` returns + * true (e.g. a terminal ``end``/``error`` event was dispatched — + * the reconnect endpoint is a live tail that doesn't close on its + * own past terminal replay). + */ +/** + * Convert a non-SSE pre-stream HTTP failure (e.g. ``check_usage``'s + * 429 JSON response) into a synthetic typed ``error`` frame so the + * caller's slice sees the actual server message instead of the + * generic "Connection lost" synthesised when the drainer finishes + * with zero events. Returns true if a frame was dispatched and the + * caller should skip ``_drainSseBody`` entirely. + * + * SSE-shaped error bodies (``mimetype="text/event-stream"``) are + * left alone — the drainer parses the typed ``error`` frame they + * carry through the normal path. + */ +async function _handlePreStreamHttpError( + response: Response, + dispatch: (data: string) => void, +): Promise { + if (response.ok) return false; + const contentType = ( + response.headers.get('content-type') ?? '' + ).toLowerCase(); + if (contentType.includes('text/event-stream')) return false; + let message: string | null = null; + try { + const text = await response.text(); + if (text) { + try { + const parsed = JSON.parse(text); + if (parsed && typeof parsed === 'object') { + message = + (typeof parsed.message === 'string' && parsed.message) || + (typeof parsed.error === 'string' && parsed.error) || + (typeof parsed.detail === 'string' && parsed.detail) || + null; + } + } catch { + message = text.slice(0, 500); + } + } + } catch { + // Body already consumed or unreadable — fall through to the + // status-line fallback below. + } + if (!message) { + message = `HTTP ${response.status} ${response.statusText}`.trim(); + } + dispatch(JSON.stringify({ type: 'error', error: message })); + return true; +} + +async function _drainSseBody( + body: ReadableStream, + signal: AbortSignal, + onData: (data: string) => void, + onId: (id: number) => void, + shouldStop?: () => boolean, +): Promise { + const reader = body.getReader(); + const decoder = new TextDecoder('utf-8'); + let buffer = ''; + let stoppedEarly = false; + try { + while (true) { + if (signal.aborted) break; + if (shouldStop?.()) { + stoppedEarly = true; + break; + } + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + // Normalise mixed line terminators so a stray CR can't smuggle + // a record boundary inside a JSON payload. + buffer = buffer.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + let boundary = buffer.indexOf('\n\n'); + while (boundary !== -1) { + const record = buffer.slice(0, boundary); + buffer = buffer.slice(boundary + 2); + boundary = buffer.indexOf('\n\n'); + if (record.length === 0) continue; + const dataParts: string[] = []; + let sawDataField = false; + for (const line of record.split('\n')) { + if (line.length === 0) continue; + if (line.startsWith(':')) continue; // SSE comment / keepalive + const colonIdx = line.indexOf(':'); + const field = colonIdx === -1 ? line : line.slice(0, colonIdx); + let value = colonIdx === -1 ? '' : line.slice(colonIdx + 1); + if (value.startsWith(' ')) value = value.slice(1); + if (field === 'id') { + // Strict regex match — empty value, hex, ``-1`` (the + // backend's terminal snapshot-failure synthetic), and + // exponent forms are all rejected so they can't silently + // rewrite the reconnect cursor. + if (_SEQUENCE_NO_RE.test(value)) onId(parseInt(value, 10)); + } else if (field === 'data') { + sawDataField = true; + dataParts.push(value); + } + } + if (!sawDataField) continue; + const data = dataParts.join('\n').trim(); + if (data.length === 0) continue; + onData(data); + if (shouldStop?.()) { + stoppedEarly = true; + break; + } + } + if (stoppedEarly) break; + } + } finally { + if (stoppedEarly) { + // Ask the runtime to tear the underlying response body down so + // the server-side WSGI thread isn't pinned waiting on + // keepalives. ``releaseLock`` alone leaves the body half-open. + try { + await reader.cancel(); + } catch { + // Already errored / closed. + } + } + try { + reader.releaseLock(); + } catch { + // Already released. + } + } +} + export function handleFetchAnswer( question: string, signal: AbortSignal, @@ -143,54 +290,153 @@ export function handleFetchAnswerSteaming( const headers: Record = {}; if (idempotencyKey) headers['Idempotency-Key'] = idempotencyKey; + + // Per-stream state used for reconnect-after-disconnect. + let messageId: string | null = null; + let lastEventId: number | null = null; + // The single JSON.parse below feeds both the message_id capture and + // the termination flag — cheaper and stricter than substring + // matching the wire bytes. + let endReceived = false; + + const dispatch = (data: string) => { + try { + const parsed = JSON.parse(data); + if (parsed && typeof parsed === 'object') { + if (parsed.type === 'message_id' && parsed.message_id) { + messageId = parsed.message_id; + } else if (parsed.type === 'end' || parsed.type === 'error') { + endReceived = true; + } + } + } catch { + // Not JSON — pass through anyway; the caller handles raw lines. + } + onEvent(new MessageEvent('message', { data })); + }; + + const runInitialPost = async (): Promise => { + const response = await conversationService.answerStream( + payload, + token, + signal, + headers, + ); + // Pre-stream HTTP failures with non-SSE bodies (e.g. ``check_usage`` + // returning a JSON 429) drain as zero events and would otherwise + // be masked by the generic "Connection lost" synthetic. Convert + // them into a typed ``error`` frame so the real message surfaces. + if (await _handlePreStreamHttpError(response, dispatch)) return; + if (!response.body) throw new Error('No response body'); + await _drainSseBody(response.body, signal, dispatch, (id) => { + lastEventId = id; + }); + }; + + // Reconnect's stop predicate: as soon as ``dispatch`` flips + // ``endReceived`` (typed ``end`` or ``error`` event seen — both + // are terminal per the backend's contract). Without this the + // live-tail endpoint would emit keepalives indefinitely and the + // await would never return. + const reconnectShouldStop = () => endReceived; + + const runReconnect = async (): Promise => { + if (!messageId) { + throw new Error('reconnect: no message_id captured'); + } + const url = new URL(`${baseURL}/api/messages/${messageId}/events`); + if (lastEventId !== null) { + url.searchParams.set('last_event_id', String(lastEventId)); + } + const reconnectHeaders: Record = { + Accept: 'text/event-stream', + }; + if (token) reconnectHeaders.Authorization = `Bearer ${token}`; + // NB: there is no slice consumer for a synthetic ``reconnecting`` + // event yet — surface only the underlying network reality. The + // user-visible ``Reconnecting…`` affordance is a follow-up that + // needs ``conversationSlice`` to gain a status case. + const response = await fetch(url.toString(), { + method: 'GET', + headers: reconnectHeaders, + signal, + cache: 'no-store', + }); + if (!response.ok || !response.body) { + throw new Error( + `reconnect: HTTP ${response.status} ${response.statusText}`, + ); + } + await _drainSseBody( + response.body, + signal, + dispatch, + (id) => { + lastEventId = id; + }, + reconnectShouldStop, + ); + }; + return new Promise((resolve, reject) => { - conversationService - .answerStream(payload, token, signal, headers) - .then((response) => { - if (!response.body) throw Error('No response body'); - - let buffer = ''; - const reader = response.body.getReader(); - const decoder = new TextDecoder('utf-8'); - let counterrr = 0; - const processStream = ({ - done, - value, - }: ReadableStreamReadResult) => { - if (done) return; - - counterrr += 1; - - const chunk = decoder.decode(value); - buffer += chunk; - - const events = buffer.split('\n\n'); - buffer = events.pop() ?? ''; - - for (const event of events) { - if (event.trim().startsWith('data:')) { - const dataLine: string = event - .split('\n') - .map((line: string) => line.replace(/^data:\s?/, '')) - .join(''); - - const messageEvent = new MessageEvent('message', { - data: dataLine.trim(), - }); - - onEvent(messageEvent); - } + (async () => { + try { + try { + await runInitialPost(); + } catch (initialErr) { + // Mid-stream network failures (WiFi blip, worker recycle, + // body reader rejecting) surface as a thrown error — not a + // graceful EOF. If the stream had already started (we have a + // ``messageId``), fall through to the reconnect path so the + // journal-backed replay can finish what the live socket + // couldn't. Pre-stream failures (auth, DNS, server 4xx/5xx + // before any yield) lack a messageId and bubble up. + if (signal.aborted || !messageId) throw initialErr; + console.warn( + 'Initial stream failed mid-flight, attempting reconnect:', + initialErr, + ); + } + // The backend ends the stream cleanly with a typed ``end`` + // event. Anything else (network drop, gunicorn worker recycle, + // load-balancer timeout) is a "premature close" — try one + // reconnect via the GET /api/messages//events endpoint. + if (!endReceived && !signal.aborted && messageId) { + try { + await runReconnect(); + } catch (reconnectErr) { + console.warn('Stream reconnect failed:', reconnectErr); } - - reader.read().then(processStream).catch(reject); - }; - - reader.read().then(processStream).catch(reject); - }) - .catch((error) => { + } + // If we never observed a terminal frame (reconnect 4xx/5xx, + // network drop during reconnect, or live tail still silent), + // synthesize one through the same ``dispatch`` path the wire + // events use. Without this the caller's slice never transitions + // out of ``streaming`` and the UI stays in a loading spinner + // forever — the conversationSlice handles ``data.type === 'error'`` + // by setting status=failed. + if (!endReceived && !signal.aborted) { + dispatch( + JSON.stringify({ + type: 'error', + error: + 'Connection lost. The response could not be resumed; please try again.', + }), + ); + } + // The handler historically never explicitly resolved with a + // value — callers consume the streamed events via ``onEvent`` + // and read final state from Redux. Preserve that contract. + resolve(undefined as unknown as Answer); + } catch (error) { + if (signal.aborted) { + resolve(undefined as unknown as Answer); + return; + } console.error('Connection failed:', error); reject(error); - }); + } + })(); }); } @@ -214,52 +460,149 @@ export function handleSubmitToolActions( const headers: Record = {}; if (idempotencyKey) headers['Idempotency-Key'] = idempotencyKey; + + // Tool-action submissions resume against the original + // ``reserved_message_id``, so the backend's continuation path emits + // ``id:`` prefixed records that the legacy parser would silently + // drop. Use the shared SSE drainer — and the same reconnect-on- + // premature-close pattern as ``handleFetchAnswerSteaming`` so a + // dropped tool-action stream can pick up after the disconnect. + let messageId: string | null = null; + let lastEventId: number | null = null; + + // Track whether the typed ``end`` event was observed. The single + // JSON.parse below feeds both the message_id capture and the + // termination flag — cheaper and stricter than substring matching + // the wire bytes. + let endReceived = false; + + const dispatch = (data: string) => { + try { + const parsed = JSON.parse(data); + if (parsed && typeof parsed === 'object') { + if (parsed.type === 'message_id' && parsed.message_id) { + messageId = parsed.message_id; + } else if (parsed.type === 'end' || parsed.type === 'error') { + // Match the backend's terminal set in + // ``application/streaming/event_replay.py``: the agent's + // catch-all path emits ``error`` *without* a trailing + // ``end``, so treating only ``end`` as terminal would + // trigger a reconnect against an already-finished stream + // and hang on keepalives. + endReceived = true; + } + } + } catch { + // Not JSON — pass through anyway; the caller handles raw lines. + } + onEvent(new MessageEvent('message', { data })); + }; + + const runInitial = async (): Promise => { + const response = await conversationService.answerStream( + payload, + token, + signal, + headers, + ); + // See ``handleFetchAnswerSteaming`` for the rationale: non-SSE + // HTTP failures (e.g. ``check_usage`` 429 JSON) need to be lifted + // into a typed ``error`` frame before they reach the drainer. + if (await _handlePreStreamHttpError(response, dispatch)) return; + if (!response.body) throw new Error('No response body'); + await _drainSseBody(response.body, signal, dispatch, (id) => { + lastEventId = id; + }); + }; + + // Reconnect's stop predicate: as soon as ``dispatch`` flips + // ``endReceived`` (typed ``end`` or ``error`` event seen — both + // are terminal per the backend's contract). Without this the + // live-tail endpoint would emit keepalives indefinitely and the + // await would never return. + const reconnectShouldStop = () => endReceived; + + const runReconnect = async (): Promise => { + if (!messageId) { + throw new Error('reconnect: no message_id captured'); + } + const url = new URL(`${baseURL}/api/messages/${messageId}/events`); + if (lastEventId !== null) { + url.searchParams.set('last_event_id', String(lastEventId)); + } + const reconnectHeaders: Record = { + Accept: 'text/event-stream', + }; + if (token) reconnectHeaders.Authorization = `Bearer ${token}`; + const response = await fetch(url.toString(), { + method: 'GET', + headers: reconnectHeaders, + signal, + cache: 'no-store', + }); + if (!response.ok || !response.body) { + throw new Error( + `reconnect: HTTP ${response.status} ${response.statusText}`, + ); + } + await _drainSseBody( + response.body, + signal, + dispatch, + (id) => { + lastEventId = id; + }, + reconnectShouldStop, + ); + }; + return new Promise((resolve, reject) => { - conversationService - .answerStream(payload, token, signal, headers) - .then((response) => { - if (!response.body) throw Error('No response body'); - - let buffer = ''; - const reader = response.body.getReader(); - const decoder = new TextDecoder('utf-8'); - - const processStream = ({ - done, - value, - }: ReadableStreamReadResult) => { - if (done) return; - - const chunk = decoder.decode(value); - buffer += chunk; - - const events = buffer.split('\n\n'); - buffer = events.pop() ?? ''; - - for (const event of events) { - if (event.trim().startsWith('data:')) { - const dataLine: string = event - .split('\n') - .map((line: string) => line.replace(/^data:\s?/, '')) - .join(''); - - const messageEvent = new MessageEvent('message', { - data: dataLine.trim(), - }); - - onEvent(messageEvent); - } + (async () => { + try { + try { + await runInitial(); + } catch (initialErr) { + // Same premature-close handling as + // ``handleFetchAnswerSteaming``: a thrown reader error after + // the message_id frame still warrants one reconnect attempt + // against the journal. Pre-stream failures lack a messageId + // and bubble up. + if (signal.aborted || !messageId) throw initialErr; + console.warn( + 'Tool-actions stream failed mid-flight, attempting reconnect:', + initialErr, + ); + } + if (!endReceived && !signal.aborted && messageId) { + try { + await runReconnect(); + } catch (reconnectErr) { + console.warn('Tool-actions reconnect failed:', reconnectErr); } - - reader.read().then(processStream).catch(reject); - }; - - reader.read().then(processStream).catch(reject); - }) - .catch((error) => { + } + // Synthesize a terminal error if reconnect couldn't deliver one + // (4xx/5xx, network drop, silent live tail). Same reasoning as + // ``handleFetchAnswerSteaming``: the caller's slice only exits + // the streaming state on a terminal frame. + if (!endReceived && !signal.aborted) { + dispatch( + JSON.stringify({ + type: 'error', + error: + 'Connection lost. The tool response could not be resumed; please try again.', + }), + ); + } + resolve(undefined as unknown as Answer); + } catch (error) { + if (signal.aborted) { + resolve(undefined as unknown as Answer); + return; + } console.error('Tool actions submission failed:', error); reject(error); - }); + } + })(); }); } diff --git a/frontend/src/conversation/conversationSlice.test.ts b/frontend/src/conversation/conversationSlice.test.ts new file mode 100644 index 00000000..db942434 --- /dev/null +++ b/frontend/src/conversation/conversationSlice.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from 'vitest'; + +import reducer, { + applyMessageTail, + setConversation, +} from './conversationSlice'; + +const baseQuery = { + prompt: 'tell me a poem', + messageId: 'm-1', + messageStatus: 'pending' as const, +}; + +const seedSlice = () => reducer(undefined, setConversation([baseQuery])); + +describe('applyMessageTail — streaming partial', () => { + it('writes response to the query while status is streaming', () => { + const state = seedSlice(); + const next = reducer( + state, + applyMessageTail({ + index: 0, + tail: { + message_id: 'm-1', + status: 'streaming', + response: 'Hello, par', + thought: null, + sources: [], + tool_calls: [], + }, + }), + ); + expect(next.queries[0].messageStatus).toBe('streaming'); + expect(next.queries[0].response).toBe('Hello, par'); + }); + + it('updates response on each successive tail tick', () => { + let state = seedSlice(); + state = reducer( + state, + applyMessageTail({ + index: 0, + tail: { + message_id: 'm-1', + status: 'streaming', + response: 'Hello', + sources: [], + tool_calls: [], + }, + }), + ); + state = reducer( + state, + applyMessageTail({ + index: 0, + tail: { + message_id: 'm-1', + status: 'streaming', + response: 'Hello, world', + sources: [], + tool_calls: [], + }, + }), + ); + expect(state.queries[0].response).toBe('Hello, world'); + }); + + it('applies sources and tool_calls when they appear mid-stream', () => { + const state = seedSlice(); + const next = reducer( + state, + applyMessageTail({ + index: 0, + tail: { + message_id: 'm-1', + status: 'streaming', + response: 'partial', + sources: [{ id: 's1', title: 'doc' }], + tool_calls: [{ name: 'search' }], + }, + }), + ); + expect(next.queries[0].sources).toEqual([{ id: 's1', title: 'doc' }]); + expect(next.queries[0].tool_calls).toEqual([{ name: 'search' }]); + }); + + it('ignores empty sources / tool_calls arrays so the renderer stays clean', () => { + const state = seedSlice(); + const next = reducer( + state, + applyMessageTail({ + index: 0, + tail: { + message_id: 'm-1', + status: 'streaming', + response: 'partial', + sources: [], + tool_calls: [], + }, + }), + ); + expect(next.queries[0].sources).toBeUndefined(); + expect(next.queries[0].tool_calls).toBeUndefined(); + }); + + it('promotes to complete with the final response and clears any error', () => { + let state = seedSlice(); + state = reducer( + state, + applyMessageTail({ + index: 0, + tail: { + message_id: 'm-1', + status: 'streaming', + response: 'partial', + }, + }), + ); + state = reducer( + state, + applyMessageTail({ + index: 0, + tail: { + message_id: 'm-1', + status: 'complete', + response: 'Final answer.', + }, + }), + ); + expect(state.queries[0].messageStatus).toBe('complete'); + expect(state.queries[0].response).toBe('Final answer.'); + expect(state.queries[0].error).toBeUndefined(); + }); + + it('surfaces failed status as error and clears response', () => { + const state = seedSlice(); + const next = reducer( + state, + applyMessageTail({ + index: 0, + tail: { + message_id: 'm-1', + status: 'failed', + response: 'whatever', + error: 'worker died', + }, + }), + ); + expect(next.queries[0].messageStatus).toBe('failed'); + expect(next.queries[0].error).toBe('worker died'); + expect(next.queries[0].response).toBeUndefined(); + }); +}); diff --git a/frontend/src/conversation/conversationSlice.ts b/frontend/src/conversation/conversationSlice.ts index 8c7f695d..264aa456 100644 --- a/frontend/src/conversation/conversationSlice.ts +++ b/frontend/src/conversation/conversationSlice.ts @@ -957,20 +957,34 @@ export const conversationSlice = createSlice({ const status = tail?.status as MessageStatus | undefined; query.messageStatus = status; query.lastHeartbeatAt = tail?.last_heartbeat_at ?? query.lastHeartbeatAt; - if (status === 'complete') { - query.response = tail?.response ?? ''; - query.thought = tail?.thought ?? query.thought; - query.sources = tail?.sources ?? query.sources; - query.tool_calls = tail?.tool_calls ?? query.tool_calls; - delete query.error; - } else if (status === 'failed') { + if (status === 'failed') { // Surface as error so the placeholder text never renders. query.error = (typeof tail?.error === 'string' && tail.error) || 'Generation failed before completing.'; delete query.response; + return; + } + // /tail returns reconstructed partials mid-stream so a second tab + // can render the in-flight bubble; spinner is driven by status. + const incomingResponse = tail?.response; + if (typeof incomingResponse === 'string') { + query.response = incomingResponse; + } else if (status === 'complete') { + query.response = ''; + } + if (typeof tail?.thought === 'string') { + query.thought = tail.thought; + } + if (Array.isArray(tail?.sources) && tail.sources.length > 0) { + query.sources = tail.sources; + } + if (Array.isArray(tail?.tool_calls) && tail.tool_calls.length > 0) { + query.tool_calls = tail.tool_calls; + } + if (status === 'complete') { + delete query.error; } - // pending / streaming: untouched; spinner keeps showing. }, raiseError( state, diff --git a/frontend/src/events/EventStreamProvider.tsx b/frontend/src/events/EventStreamProvider.tsx new file mode 100644 index 00000000..8b84aa26 --- /dev/null +++ b/frontend/src/events/EventStreamProvider.tsx @@ -0,0 +1,18 @@ +import React from 'react'; + +import { useEventStream } from './useEventStream'; + +/** + * Mount-once provider that opens the user's SSE connection. Place + * inside ``AuthWrapper`` so it sees a populated token, and wrap the + * authenticated-app subtree so the connection lives for the user's + * whole session. + */ +export function EventStreamProvider({ + children, +}: { + children: React.ReactNode; +}): React.ReactElement { + useEventStream(); + return <>{children}; +} diff --git a/frontend/src/events/dispatchEvent.test.ts b/frontend/src/events/dispatchEvent.test.ts new file mode 100644 index 00000000..f0bf5135 --- /dev/null +++ b/frontend/src/events/dispatchEvent.test.ts @@ -0,0 +1,49 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { AppDispatch } from '../store'; +import { + sseEventReceived, + sseLastEventIdReset, +} from '../notifications/notificationsSlice'; +import { dispatchSSEEvent } from './dispatchEvent'; + +describe('dispatchSSEEvent', () => { + let debugSpy: ReturnType; + + beforeEach(() => { + debugSpy = vi.spyOn(console, 'debug').mockImplementation(() => undefined); + }); + + afterEach(() => { + debugSpy.mockRestore(); + }); + + it('dispatches sseLastEventIdReset AND sseEventReceived for backlog.truncated', () => { + const dispatch = vi.fn() as unknown as AppDispatch; + const envelope = { type: 'backlog.truncated' as const }; + + dispatchSSEEvent(envelope, dispatch); + + const calls = (dispatch as unknown as { mock: { calls: unknown[][] } }).mock + .calls; + expect(calls).toHaveLength(2); + expect(calls[0][0]).toEqual(sseLastEventIdReset()); + expect(calls[1][0]).toEqual(sseEventReceived(envelope)); + }); + + it('does not log a debug line for known envelope types', () => { + const dispatch = vi.fn() as unknown as AppDispatch; + dispatchSSEEvent({ id: 'e-1', type: 'source.ingest.progress' }, dispatch); + expect(debugSpy).not.toHaveBeenCalled(); + }); + + it('logs a debug line for unknown envelope types', () => { + const dispatch = vi.fn() as unknown as AppDispatch; + dispatchSSEEvent({ id: 'e-2', type: 'mystery.event' }, dispatch); + expect(debugSpy).toHaveBeenCalledTimes(1); + expect(debugSpy.mock.calls[0]).toEqual([ + '[dispatchSSEEvent] unknown envelope type', + 'mystery.event', + ]); + }); +}); diff --git a/frontend/src/events/dispatchEvent.ts b/frontend/src/events/dispatchEvent.ts new file mode 100644 index 00000000..4139691a --- /dev/null +++ b/frontend/src/events/dispatchEvent.ts @@ -0,0 +1,58 @@ +import type { AppDispatch } from '../store'; +import { + sseEventReceived, + sseLastEventIdReset, + type SSEEvent, +} from '../notifications/notificationsSlice'; + +// Envelope types this build knows about. Hitting an unknown type means +// the backend published something the frontend hasn't been taught yet +// — worth a single debug line so it's visible in devtools without +// drowning the console in known per-progress traffic. +const KNOWN_TYPES: ReadonlySet = new Set([ + 'backlog.truncated', + 'source.ingest.queued', + 'source.ingest.progress', + 'source.ingest.completed', + 'source.ingest.failed', + 'attachment.queued', + 'attachment.progress', + 'attachment.completed', + 'attachment.failed', + 'mcp.oauth.awaiting_redirect', + 'mcp.oauth.in_progress', + 'mcp.oauth.completed', + 'mcp.oauth.failed', + 'tool.approval.required', +]); + +/** + * Single fan-out point for inbound SSE envelopes. Always dispatches + * ``sseEventReceived`` so any slice can ``extraReducers``-listen + * (uploadSlice does this for source-ingest events), then handles the + * small set of envelope-types that need centralised side effects (e.g. + * ``backlog.truncated``). + */ +export function dispatchSSEEvent( + envelope: SSEEvent, + dispatch: AppDispatch, +): void { + if (!KNOWN_TYPES.has(envelope.type)) { + console.debug('[dispatchSSEEvent] unknown envelope type', envelope.type); + } + + switch (envelope.type) { + case 'backlog.truncated': + // Backlog window slid past the client's Last-Event-ID. Drop the + // cursor so the next reconnect doesn't try to resume past the + // retained window. Slices that care about full-state freshness + // can subscribe to ``sseEventReceived`` and refetch. + dispatch(sseLastEventIdReset()); + break; + default: + // No central side effect; rely on slice-level extraReducers. + break; + } + + dispatch(sseEventReceived(envelope)); +} diff --git a/frontend/src/events/eventStreamClient.ts b/frontend/src/events/eventStreamClient.ts new file mode 100644 index 00000000..badd8e07 --- /dev/null +++ b/frontend/src/events/eventStreamClient.ts @@ -0,0 +1,386 @@ +import { baseURL } from '../api/client'; +import type { SSEEvent } from '../notifications/notificationsSlice'; + +/** + * Connection state surfaced to the consumer. Maps directly to the + * ``PushHealth`` machine in ``notificationsSlice``. + */ +export type EventStreamHealth = 'connecting' | 'healthy' | 'unhealthy'; + +export interface EventStreamOptions { + /** Bearer token; ``null`` short-circuits to ``unhealthy`` (auth required). */ + token: string | null; + /** + * Lazy getter for the current ``Last-Event-ID``. Called once at the + * top of each connect attempt so token rotations / remounts read + * the freshest cursor from Redux instead of a stale mount-time + * snapshot. Return ``null`` for a fresh connect. + */ + getLastEventId: () => string | null; + onEvent: (event: SSEEvent) => void; + onHealthChange: (health: EventStreamHealth) => void; + /** Called with the most recently received id so the caller can persist it. */ + onLastEventId?: (id: string) => void; + /** + * Called when the server emitted an ``id:`` line with an empty value + * (WHATWG SSE cursor reset). Distinct from ``onLastEventId('')`` so + * callers can dispatch ``sseLastEventIdReset`` without overloading + * the normal advance path. + */ + onLastEventIdReset?: () => void; + /** + * Invoked once after ``MAX_CONSECUTIVE_401`` back-to-back 401s. The + * reconnect loop then bails out, so the caller is responsible for + * refreshing the token / signalling logout. Without this, an expired + * token spins forever at the 30s backoff cap. + */ + onAuthFailure?: () => void; + /** + * Invoked once when the reconnect loop bails out after + * ``MAX_CONSECUTIVE_ERRORS`` non-401 failures. Lets the caller surface + * a warning instead of the connection silently going dark. + */ + onPermanentFailure?: () => void; +} + +export interface EventStreamConnection { + close(): void; +} + +/** + * Backoff schedule (ms) for reconnect attempts. Capped at 30s so a long + * outage doesn't push retries past Cloudflare's typical 100s idle-close + * envelope. The schedule resets to 0 after a stream stays healthy for + * ``HEALTHY_DEBOUNCE_MS``. + */ +const BACKOFF_SCHEDULE_MS = [0, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000]; +const HEALTHY_DEBOUNCE_MS = 2_000; +/** + * Reconnect ceilings. Without these, the ``while (!closed)`` loop spins + * forever on a persistently-failing endpoint — expired token (401s) or + * sustained server outage (5xx). Both counters reset on a successful + * stream open. Untested (no frontend test harness); behaviour verified + * by manual trace of the loop in ``connectEventStream``. + */ +const MAX_CONSECUTIVE_401 = 3; +const MAX_CONSECUTIVE_ERRORS = 20; + +/** Up-to-±20% random jitter so N tabs reconnecting in lockstep stagger. */ +function withJitter(delayMs: number): number { + if (delayMs <= 0) return 0; + const span = delayMs * 0.2; + return Math.max(0, Math.round(delayMs + (Math.random() * 2 - 1) * span)); +} + +/** + * Open a long-lived SSE connection to ``GET /api/events`` with + * fetch-streaming semantics that mirror ``conversationHandlers.ts``. + * + * Returns immediately with an opaque handle; the connection lives in a + * background async loop until ``close()`` is called or the underlying + * ``AbortController`` fires. + * + * The ``Last-Event-ID`` cursor rides on the URL (``?last_event_id=...``) + * rather than as a header so the request stays a CORS-simple GET — a + * custom header would force a preflight OPTIONS that the production + * cross-origin deployment isn't allowlisted for. + */ +export function connectEventStream( + opts: EventStreamOptions, +): EventStreamConnection { + const controller = new AbortController(); + let closed = false; + let attempt = 0; + let consecutive401 = 0; + let consecutiveErrors = 0; + // Closure cursor. Seeded from the store on each connect attempt so + // mid-session reconnects use the freshest id, but kept here too so + // an in-flight stream's reconnect doesn't lose progress between ids + // that the store hasn't seen yet (e.g. id-only frames). + let lastEventId: string | null = opts.getLastEventId(); + + const notifyHealth = (h: EventStreamHealth) => { + if (closed) return; + opts.onHealthChange(h); + }; + + void (async () => { + while (!closed) { + const baseDelay = + BACKOFF_SCHEDULE_MS[Math.min(attempt, BACKOFF_SCHEDULE_MS.length - 1)]; + const delay = withJitter(baseDelay); + if (delay > 0) { + try { + await sleep(delay, controller.signal); + } catch { + return; // aborted while waiting + } + if (closed) return; + } + + notifyHealth('connecting'); + + // Always re-read the store cursor before reconnecting and copy + // it verbatim — including null. A null cursor isn't "leave + // alone": ``backlog.truncated`` events fire ``sseLastEventIdReset`` + // to clear the slice, and the client must respect that on the + // next attempt by sending no cursor (full-backlog replay) instead + // of resending the stale one and re-tripping the same truncation. + lastEventId = opts.getLastEventId(); + + const url = new URL(`${baseURL}/api/events`); + if (lastEventId) url.searchParams.set('last_event_id', lastEventId); + + // Auth header is omitted when token is null. Self-hosted dev + // installs run with ``AUTH_TYPE`` unset; the backend maps those + // requests to ``{"sub": "local"}`` so the SSE connection works + // tokenless. When auth IS required, a missing header surfaces + // as a 401 and the response.ok check below flips the health + // back to unhealthy. + const headers: Record = { + Accept: 'text/event-stream', + }; + if (opts.token) { + headers.Authorization = `Bearer ${opts.token}`; + } + + try { + const response = await fetch(url.toString(), { + method: 'GET', + headers, + signal: controller.signal, + // SSE must not be cached. + cache: 'no-store', + }); + + if (!response.ok || !response.body) { + notifyHealth('unhealthy'); + // 401 typically means token expired. Bail out after N + // consecutive 401s so the loop doesn't spin forever at the + // 30s backoff cap with a stale token; the caller is + // responsible for refreshing auth via ``onAuthFailure``. + if (response.status === 401) { + consecutive401 += 1; + consecutiveErrors += 1; + if (consecutive401 >= MAX_CONSECUTIVE_401) { + opts.onAuthFailure?.(); + return; + } + } else { + consecutive401 = 0; + consecutiveErrors += 1; + } + if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { + opts.onPermanentFailure?.(); + return; + } + // 429: server-side per-user concurrency cap; backoff harder. + if (response.status === 429) attempt = Math.max(attempt, 4); + else attempt = Math.min(attempt + 1, BACKOFF_SCHEDULE_MS.length - 1); + continue; + } + consecutive401 = 0; + + // Connection is open. Mark healthy after either: + // - 2s of open response body (covers servers that go silent), or + // - first record received past the 2s mark. + // The setTimeout path means a backend that never emits a single + // record after sending the 200 still flips us out of `connecting`. + let healthyMarked = false; + const markHealthy = () => { + if (healthyMarked) return; + healthyMarked = true; + notifyHealth('healthy'); + attempt = 0; + consecutiveErrors = 0; + }; + const debounceTimer = setTimeout(markHealthy, HEALTHY_DEBOUNCE_MS); + + try { + await readSSEStream(response.body, controller.signal, (record) => { + if (record.id !== undefined) { + lastEventId = record.id || null; + if (record.id) opts.onLastEventId?.(record.id); + else opts.onLastEventIdReset?.(); + } + if (record.data === undefined) { + // Keepalive comment, id-only frame, or comment line. + // The cursor was already advanced via ``onLastEventId`` + // above so the slice tracks ids even on frames we don't + // dispatch as events. + return; + } + // Empty data line is technically valid SSE but useless; skip. + if (record.data.trim().length === 0) return; + let envelope: SSEEvent | null = null; + try { + envelope = JSON.parse(record.data) as SSEEvent; + } catch { + // Malformed payload; skip. + return; + } + // Defensive shape validation — the cast above lies if the + // server (or a man-in-the-middle) sends garbage. + if ( + !envelope || + typeof envelope !== 'object' || + typeof envelope.type !== 'string' + ) { + return; + } + if (record.id && !envelope.id) envelope.id = record.id; + // Receiving a real envelope post-debounce-window flips + // healthy if the timer hasn't already. + markHealthy(); + // Every tab dispatches every envelope it receives into its + // own Redux store. With N tabs open this means N copies of + // the same toast — accepted as a v1 limitation; cross-tab + // dedup via BroadcastChannel + navigator.locks is future + // work. Toast-level suppression can be handled per surface. + opts.onEvent(envelope); + }); + } finally { + clearTimeout(debounceTimer); + } + + // The reader returned without abort — server closed the stream. + // Fall through to reconnect. + notifyHealth('unhealthy'); + consecutiveErrors += 1; + if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { + opts.onPermanentFailure?.(); + return; + } + attempt = Math.min(attempt + 1, BACKOFF_SCHEDULE_MS.length - 1); + } catch (err) { + if ( + closed || + (err instanceof DOMException && err.name === 'AbortError') + ) { + return; + } + notifyHealth('unhealthy'); + consecutiveErrors += 1; + if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { + opts.onPermanentFailure?.(); + return; + } + attempt = Math.min(attempt + 1, BACKOFF_SCHEDULE_MS.length - 1); + } + } + })(); + + return { + close() { + if (closed) return; + closed = true; + controller.abort(); + }, + }; +} + +interface ParsedSSERecord { + /** + * ``undefined`` when the record had no ``id`` field at all. An empty + * string means the server explicitly reset the cursor (an ``id:`` + * line with no value, per WHATWG SSE). + */ + id?: string; + /** ``undefined`` for keepalive comments / id-only frames. */ + data?: string; +} + +/** + * Drain a ``ReadableStream`` of ``\n\n``-delimited SSE records, + * forwarding each parsed record to ``onRecord``. Honours the WHATWG SSE + * spec's mixed line-terminator handling and SSE comment lines. + */ +async function readSSEStream( + body: ReadableStream, + signal: AbortSignal, + onRecord: (record: ParsedSSERecord) => void, +): Promise { + const reader = body.getReader(); + const decoder = new TextDecoder('utf-8'); + let buffer = ''; + + try { + while (true) { + if (signal.aborted) return; + const { done, value } = await reader.read(); + if (done) return; + buffer += decoder.decode(value, { stream: true }); + + // SSE records are separated by a blank line. WHATWG spec accepts + // CRLF, CR, or LF — normalise so a stray CR can't smuggle a + // boundary mid-record. + buffer = buffer.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + + let boundary = buffer.indexOf('\n\n'); + while (boundary !== -1) { + const raw = buffer.slice(0, boundary); + buffer = buffer.slice(boundary + 2); + const record = parseSSERecord(raw); + if (record) onRecord(record); + boundary = buffer.indexOf('\n\n'); + } + } + } finally { + try { + reader.releaseLock(); + } catch { + // Already released. + } + } +} + +function parseSSERecord(raw: string): ParsedSSERecord | null { + if (raw.length === 0) return null; + const lines = raw.split('\n'); + let id: string | undefined; + const dataParts: string[] = []; + let sawDataField = false; + + for (const line of lines) { + if (line.length === 0) continue; + if (line.startsWith(':')) continue; // SSE comment / keepalive + const colonIdx = line.indexOf(':'); + const field = colonIdx === -1 ? line : line.slice(0, colonIdx); + let value = colonIdx === -1 ? '' : line.slice(colonIdx + 1); + // SSE: value may be prefixed by exactly one optional space. + if (value.startsWith(' ')) value = value.slice(1); + + if (field === 'id') { + id = value; + } else if (field === 'data') { + sawDataField = true; + dataParts.push(value); + } + // Other field names ('event', 'retry') are ignored for now. + } + + if (!sawDataField && id === undefined) return null; + return { + id, + data: sawDataField ? dataParts.join('\n') : undefined, + }; +} + +function sleep(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal.aborted) { + reject(new DOMException('Aborted', 'AbortError')); + return; + } + const timer = setTimeout(() => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + signal.removeEventListener('abort', onAbort); + reject(new DOMException('Aborted', 'AbortError')); + }; + signal.addEventListener('abort', onAbort, { once: true }); + }); +} diff --git a/frontend/src/events/useEventStream.ts b/frontend/src/events/useEventStream.ts new file mode 100644 index 00000000..805581aa --- /dev/null +++ b/frontend/src/events/useEventStream.ts @@ -0,0 +1,85 @@ +import { useEffect } from 'react'; +import { useDispatch, useSelector, useStore } from 'react-redux'; + +import { + selectLastEventId, + sseHealthChanged, + sseLastEventIdAdvanced, + sseLastEventIdReset, +} from '../notifications/notificationsSlice'; +import { selectToken, setToken } from '../preferences/preferenceSlice'; +import type { AppDispatch, RootState } from '../store'; + +import { connectEventStream } from './eventStreamClient'; +import { dispatchSSEEvent } from './dispatchEvent'; + +/** + * Open the SSE connection for the current token and keep it alive for + * the lifetime of the host component. Recreates the connection on + * token change (login / refresh). + * + * The ``lastEventId`` cursor is read lazily from the slice on each + * connect attempt via ``store.getState()`` — capturing it at mount time + * would silently re-replay the entire 24h backlog on token rotation, + * since the slice's id advances during the previous connection's + * lifetime but a snapshot ref would still hold the value seen at + * first mount. + */ +export function useEventStream(): void { + const dispatch = useDispatch(); + const token = useSelector(selectToken); + const store = useStore(); + + useEffect(() => { + // Connect even when token is null. Self-hosted dev installs run + // with ``AUTH_TYPE`` unset, where ``handle_auth`` maps every + // request to ``{"sub": "local"}`` regardless of headers — gating + // the connection on a populated token would silently disable push + // notifications for the most common configuration. When auth IS + // required and token is null, the backend will 401 and the + // health state will flip to ``unhealthy`` via the response check + // inside ``connectEventStream``. + const conn = connectEventStream({ + token, + getLastEventId: () => selectLastEventId(store.getState()), + onEvent: (envelope) => dispatchSSEEvent(envelope, dispatch), + // Advance the slice cursor for every id-bearing frame. Each tab + // owns an independent SSE connection and Redux store, so every + // active tab tracks its own replay cursor. + onLastEventId: (id) => dispatch(sseLastEventIdAdvanced(id)), + // Server emitted ``id:`` with an empty value — WHATWG cursor reset. + // Mirror the slice so the next reconnect doesn't resend a stale id. + onLastEventIdReset: () => dispatch(sseLastEventIdReset()), + onHealthChange: (health) => dispatch(sseHealthChanged(health)), + // SSE 401 loop bail-out. Clear the stored token AND dispatch + // ``setToken(null)`` so ``useAuth`` regenerates a fresh + // ``session_jwt`` in-session; the Redux change also flips this + // hook's ``[token]`` dep, tearing down and respawning the + // connection with the new token. Without the dispatch a + // ``session_jwt`` user is stuck until a hard reload. + onAuthFailure: () => { + console.error( + '[useEventStream] giving up after repeated 401s on /api/events', + ); + try { + localStorage.removeItem('authToken'); + } catch { + // localStorage unavailable (private mode, etc.) — nothing to do. + } + dispatch(setToken(null)); + }, + // Surface a warning when the non-401 error budget is exhausted so + // the connection going dark isn't completely silent. Doesn't block + // UI — just observable in devtools. + onPermanentFailure: () => { + console.warn( + '[useEventStream] SSE connection failed permanently after repeated errors', + ); + }, + }); + + return () => { + conn.close(); + }; + }, [token, dispatch, store]); +} diff --git a/frontend/src/hooks/useTokenAuth.ts b/frontend/src/hooks/useTokenAuth.ts index 8f408600..5fd8eb42 100644 --- a/frontend/src/hooks/useTokenAuth.ts +++ b/frontend/src/hooks/useTokenAuth.ts @@ -15,24 +15,39 @@ export default function useAuth() { const generateNewToken = async () => { if (isGeneratingToken.current) return; isGeneratingToken.current = true; - const response = await userService.getNewToken(); - const { token: newToken } = await response.json(); - localStorage.setItem('authToken', newToken); - dispatch(setToken(newToken)); - setIsAuthLoading(false); - return newToken; + try { + const response = await userService.getNewToken(); + const { token: newToken } = await response.json(); + localStorage.setItem('authToken', newToken); + dispatch(setToken(newToken)); + setIsAuthLoading(false); + return newToken; + } finally { + // Reset so a subsequent ``setToken(null)`` (SSE 401 recovery) + // can trigger another generation. Without this the in-flight + // guard would latch true forever after the first call. + isGeneratingToken.current = false; + } }; useEffect(() => { + // Re-fires when ``token`` flips to null mid-session (e.g. + // ``useEventStream`` dispatches ``setToken(null)`` after repeated + // SSE 401s) so ``session_jwt`` users get a fresh token without a + // hard reload. ``authType`` short-circuits on subsequent runs. const initializeAuth = async () => { try { - const configRes = await userService.getConfig(); - const config = await configRes.json(); - setAuthType(config.auth_type); + let resolvedAuthType = authType; + if (resolvedAuthType === null) { + const configRes = await userService.getConfig(); + const config = await configRes.json(); + resolvedAuthType = config.auth_type; + setAuthType(resolvedAuthType); + } - if (config.auth_type === 'session_jwt' && !token) { + if (resolvedAuthType === 'session_jwt' && !token) { await generateNewToken(); - } else if (config.auth_type === 'simple_jwt' && !token) { + } else if (resolvedAuthType === 'simple_jwt' && !token) { setShowTokenModal(true); setIsAuthLoading(false); } else { @@ -44,7 +59,7 @@ export default function useAuth() { } }; initializeAuth(); - }, []); + }, [token, authType]); const handleTokenSubmit = (enteredToken: string) => { localStorage.setItem('authToken', enteredToken); diff --git a/frontend/src/modals/MCPServerModal.tsx b/frontend/src/modals/MCPServerModal.tsx index 9c6857a6..44b47625 100644 --- a/frontend/src/modals/MCPServerModal.tsx +++ b/frontend/src/modals/MCPServerModal.tsx @@ -15,6 +15,7 @@ import { SelectValue, } from '../components/ui/select'; import { ActiveState } from '../models/misc'; +import { selectRecentEvents } from '../notifications/notificationsSlice'; import { selectToken } from '../preferences/preferenceSlice'; import WrapperComponent from './WrapperModal'; @@ -33,6 +34,7 @@ export default function MCPServerModal({ }: MCPServerModalProps) { const { t } = useTranslation(); const token = useSelector(selectToken); + const recentEvents = useSelector(selectRecentEvents); const authTypes = [ { label: t('settings.tools.mcp.authTypes.none'), value: 'none' }, @@ -71,17 +73,29 @@ export default function MCPServerModal({ >([]); const [errors, setErrors] = useState<{ [key: string]: string }>({}); const oauthPopupRef = useRef(null); - const pollingCancelledRef = useRef(false); - const pollTimerRef = useRef | null>(null); + // Set after ``test_mcp_connection`` returns ``task_id``. The SSE + // effect filters ``recentEvents`` to envelopes matching this id and + // drives the OAuth UI (popup open / completion / failure) from the + // push stream rather than polling the legacy status endpoint. + const [oauthTaskId, setOauthTaskId] = useState(null); + // Highest event id we have already reacted to for this taskId. Each + // mcp.oauth.* envelope must fire its side-effect once; without this + // any later re-render that re-evaluates ``recentEvents`` would + // re-open the popup or re-fire onComplete. + const handledEventIdsRef = useRef>(new Set()); + // Holds the ``testConnection`` ``onComplete`` for the current + // task id so the SSE effect can invoke it when the terminal event + // lands. Reset to ``null`` on cancel / new test / unmount. + const onCompleteRef = useRef<((result: any) => void) | null>(null); + const popupOpenedRef = useRef(false); const [oauthCompleted, setOAuthCompleted] = useState(false); const [saveActive, setSaveActive] = useState(false); - const cleanupPolling = useCallback(() => { - pollingCancelledRef.current = true; - if (pollTimerRef.current) { - clearTimeout(pollTimerRef.current); - pollTimerRef.current = null; - } + const cleanupOAuthListener = useCallback(() => { + setOauthTaskId(null); + handledEventIdsRef.current = new Set(); + onCompleteRef.current = null; + popupOpenedRef.current = false; if (oauthPopupRef.current && !oauthPopupRef.current.closed) { oauthPopupRef.current.close(); } @@ -89,8 +103,8 @@ export default function MCPServerModal({ }, []); useEffect(() => { - return cleanupPolling; - }, [cleanupPolling]); + return cleanupOAuthListener; + }, [cleanupOAuthListener]); useEffect(() => { if (modalState === 'ACTIVE' && server) { @@ -119,7 +133,7 @@ export default function MCPServerModal({ }, [modalState, server]); const resetForm = () => { - cleanupPolling(); + cleanupOAuthListener(); setFormData({ name: t('settings.tools.mcp.defaultServerName'), server_url: '', @@ -228,114 +242,123 @@ export default function MCPServerModal({ return config; }; - const pollOAuthStatus = async ( - taskId: string, - onComplete: (result: any) => void, - ) => { - let attempts = 0; - const maxAttempts = 60; - let popupOpened = false; - pollingCancelledRef.current = false; + /** + * Drive the OAuth handshake straight from the SSE stream: + * + * - ``mcp.oauth.awaiting_redirect`` → open the popup with the + * ``authorization_url`` carried on the envelope. Previously this URL + * came from polling ``/api/mcp_server/oauth_status/``; the + * worker now publishes it inline so we never need to poll. + * - ``mcp.oauth.completed`` → enable Save, surface discovered tools, + * invoke ``onComplete`` (resolves ``testConnection``'s pending state). + * - ``mcp.oauth.failed`` → surface the error and reset Save. + * + * Each event is matched to the active task id via ``scope.id``. The + * publisher is best-effort: a lost ``awaiting_redirect`` envelope + * means the popup never opens, the user retries, and we accept that + * over the prior 1s × 60 polling loop. + */ + useEffect(() => { + if (!oauthTaskId) return; + // ``recentEvents`` is newest-first (the slice ``unshift``s on + // arrival). Walk it oldest-first so we observe the natural OAuth + // ordering (``awaiting_redirect`` → ``completed``) when both + // arrive between effect runs — otherwise we would short-circuit + // on ``completed`` and never open the popup for the + // ``awaiting_redirect`` envelope that was already buffered. + for (let i = recentEvents.length - 1; i >= 0; i--) { + const event = recentEvents[i]; + if (event.scope?.id !== oauthTaskId) continue; + if (!event.id || handledEventIdsRef.current.has(event.id)) continue; - const poll = async () => { - if (pollingCancelledRef.current) return; - try { - const resp = await userService.getMCPOAuthStatus(taskId, token); - if (pollingCancelledRef.current) return; - const data = await resp.json(); - if (pollingCancelledRef.current) return; + const payload = (event.payload || {}) as Record; - if (data.authorization_url && !popupOpened) { + if (event.type === 'mcp.oauth.awaiting_redirect') { + handledEventIdsRef.current.add(event.id); + const authUrl = payload.authorization_url as string | undefined; + if (authUrl && !popupOpenedRef.current) { + popupOpenedRef.current = true; if (oauthPopupRef.current && !oauthPopupRef.current.closed) { oauthPopupRef.current.close(); } oauthPopupRef.current = window.open( - data.authorization_url, + authUrl, 'oauthPopup', 'width=600,height=700', ); - popupOpened = true; - if (!oauthPopupRef.current) { + // Popup blocked — surface the URL inline so the user can + // click through manually. Browsers gate ``window.open`` + // outside of a user gesture, and the SSE event arrives + // asynchronously, so a blocked popup is expected on + // some browsers / configs. setTestResult({ success: true, message: t('settings.tools.mcp.oauthPopupBlocked', { defaultValue: 'Popup blocked by browser. Click below to authorize:', }), - authorization_url: data.authorization_url, + authorization_url: authUrl, }); } } + continue; + } - const callbackReceived = - data.status === 'callback_received' || data.status === 'completed'; - - if (data.status === 'completed') { - setOAuthCompleted(true); - setSaveActive(true); - onComplete({ - ...data, + if (event.type === 'mcp.oauth.completed') { + handledEventIdsRef.current.add(event.id); + const tools = Array.isArray(payload.tools) ? payload.tools : []; + const toolsCount = + (payload.tools_count as number | undefined) ?? tools.length; + setOAuthCompleted(true); + setSaveActive(true); + if (oauthPopupRef.current && !oauthPopupRef.current.closed) { + oauthPopupRef.current.close(); + } + const cb = onCompleteRef.current; + onCompleteRef.current = null; + setOauthTaskId(null); + if (cb) { + cb({ + status: 'completed', + task_id: oauthTaskId, + tools, + tools_count: toolsCount, success: true, message: t('settings.tools.mcp.oauthCompleted'), }); - if (oauthPopupRef.current && !oauthPopupRef.current.closed) { - oauthPopupRef.current.close(); - } - } else if (data.status === 'error' || data.success === false) { - setSaveActive(false); - onComplete({ - ...data, - success: false, - message: data.message || t('settings.tools.mcp.errors.oauthFailed'), - }); - if (oauthPopupRef.current && !oauthPopupRef.current.closed) { - oauthPopupRef.current.close(); - } - } else { - if (++attempts < maxAttempts) { - if ( - oauthPopupRef.current && - oauthPopupRef.current.closed && - popupOpened && - !callbackReceived - ) { - setSaveActive(false); - onComplete({ - success: false, - message: t('settings.tools.mcp.errors.oauthFailed'), - }); - return; - } - pollTimerRef.current = setTimeout(poll, 1000); - } else { - setSaveActive(false); - cleanupPolling(); - onComplete({ - success: false, - message: t('settings.tools.mcp.errors.oauthTimeout'), - }); - } - } - } catch { - if (pollingCancelledRef.current) return; - if (++attempts < maxAttempts) { - pollTimerRef.current = setTimeout(poll, 1000); - } else { - cleanupPolling(); - onComplete({ - success: false, - message: t('settings.tools.mcp.errors.oauthTimeout'), - }); } + continue; } - }; - poll(); - }; + + if (event.type === 'mcp.oauth.failed') { + handledEventIdsRef.current.add(event.id); + const message = + (payload.error as string) ?? + t('settings.tools.mcp.errors.oauthFailed'); + setSaveActive(false); + if (oauthPopupRef.current && !oauthPopupRef.current.closed) { + oauthPopupRef.current.close(); + } + const cb = onCompleteRef.current; + onCompleteRef.current = null; + setOauthTaskId(null); + if (cb) { + cb({ + status: 'error', + task_id: oauthTaskId, + success: false, + message, + }); + } + continue; + } + } + }, [recentEvents, oauthTaskId, t]); const testConnection = async () => { if (!validateForm()) return; - cleanupPolling(); + cleanupOAuthListener(); setTesting(true); setTestResult(null); setDiscoveredTools([]); @@ -355,7 +378,7 @@ export default function MCPServerModal({ message: t('settings.tools.mcp.oauthInProgress'), }); setSaveActive(false); - pollOAuthStatus(result.task_id, (finalResult) => { + onCompleteRef.current = (finalResult: any) => { setTestResult(finalResult); if (finalResult.tools && Array.isArray(finalResult.tools)) { setDiscoveredTools(finalResult.tools); @@ -365,7 +388,11 @@ export default function MCPServerModal({ oauth_task_id: result.task_id || '', })); setTesting(false); - }); + }; + // Activate the SSE listener for this task id. The effect above + // will react when ``mcp.oauth.{awaiting_redirect,completed,failed}`` + // arrives. + setOauthTaskId(result.task_id); } else { setTestResult(result); if (result.success && result.tools && Array.isArray(result.tools)) { diff --git a/frontend/src/notifications/ToolApprovalToast.tsx b/frontend/src/notifications/ToolApprovalToast.tsx new file mode 100644 index 00000000..60653682 --- /dev/null +++ b/frontend/src/notifications/ToolApprovalToast.tsx @@ -0,0 +1,174 @@ +import { useDispatch, useSelector } from 'react-redux'; +import { useMatch, useNavigate } from 'react-router-dom'; + +import WarnIcon from '../assets/warn.svg'; +import type { RootState } from '../store'; + +import { + dismissToolApproval, + selectDismissedToolApprovals, + selectRecentEvents, +} from './notificationsSlice'; + +/** + * Surface ``tool.approval.required`` events as toasts that look like + * ``UploadToast`` (same fixed bottom-right rail) — but only when the + * user is NOT already on the conversation that needs the approval. + * + * - Dedup by ``conversation_id`` (the SSE ``scope.id``): keep only + * the newest pending event per conversation, so multiple paused + * tools in one conversation collapse to one toast. + * - Dismissal is per-event-id so a *new* pause of the same + * conversation will re-surface (different event id). + * - Clicking "Review" navigates to ``/c/`` and dismisses. + */ +export default function ToolApprovalToast() { + const dispatch = useDispatch(); + const navigate = useNavigate(); + const events = useSelector(selectRecentEvents); + const dismissed = useSelector(selectDismissedToolApprovals); + + // Pull the active conversation id off the route. Two route shapes + // place a conversation in view: the bare ``/c/:conversationId`` and + // the agent-scoped ``/agents/:agentId/c/:conversationId``. Hooks + // are unconditional; the toast just respects whichever matches. + // + // ``/c/new`` is the conversation route's literal-string placeholder + // for "unknown / not-yet-loaded conversation" (see the rewrite in + // the conversation route). Treat it the same as no match — the + // user isn't viewing any specific conversation yet, so an approval + // toast for any conversation should still surface. + const plainMatch = useMatch('/c/:conversationId'); + const agentMatch = useMatch('/agents/:agentId/c/:conversationId'); + const matchedConversationId = + plainMatch?.params.conversationId ?? + agentMatch?.params.conversationId ?? + null; + const currentConversationId = + matchedConversationId === 'new' ? null : matchedConversationId; + + // Conversation name lookup — best-effort. The slice's + // ``preference.conversations.data`` is populated by + // ``useDataInitializer`` once auth resolves; until then we fall + // back to the conversation id. + const conversations = useSelector( + (state: RootState) => state.preference.conversations.data, + ); + + const dismissedSet = new Set(dismissed); + const pendingByConversation = new Map< + string, + { eventId: string; conversationId: string } + >(); + for (const event of events) { + if (event.type !== 'tool.approval.required') continue; + if (!event.id) continue; // can't dismiss without an id + if (dismissedSet.has(event.id)) continue; + const conversationId = event.scope?.id; + if (!conversationId) continue; + if (currentConversationId && conversationId === currentConversationId) { + continue; + } + if (pendingByConversation.has(conversationId)) continue; + // ``recentEvents`` is newest-first, so the first match per convId + // is the most recent unhandled approval. + pendingByConversation.set(conversationId, { + eventId: event.id, + conversationId, + }); + } + + if (pendingByConversation.size === 0) return null; + + const conversationName = (conversationId: string): string => { + const found = conversations?.find((c) => c.id === conversationId); + return found?.name ?? 'Conversation'; + }; + + return ( + // Sit above ``UploadToast`` (which owns ``bottom-4 right-4``) + // rather than overlapping it. ``bottom-24`` ≈ 96px clears one + // standard-height upload toast; multiple in-flight uploads will + // stack into the gap, at which point the approval toast still + // floats on top via ``z-50``. Acceptable v1 layout — the two + // surfaces are rarely competing. +
    e.stopPropagation()} + role="status" + aria-live="polite" + aria-atomic="true" + > + {Array.from(pendingByConversation.values()).map( + ({ eventId, conversationId }) => ( +
    +
    +

    + Tool approval needed +

    + +
    +
    +
    + +

    + {conversationName(conversationId)} +

    +
    + +
    +
    + ), + )} +
    + ); +} diff --git a/frontend/src/notifications/dismissedPersistence.test.ts b/frontend/src/notifications/dismissedPersistence.test.ts new file mode 100644 index 00000000..b6b4e9f5 --- /dev/null +++ b/frontend/src/notifications/dismissedPersistence.test.ts @@ -0,0 +1,71 @@ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { + isDismissed, + loadDismissed, + saveDismissed, +} from './dismissedPersistence'; + +const KEY = 'test:dismissed'; +const TTL = 24 * 60 * 60 * 1000; + +describe('dismissedPersistence', () => { + beforeEach(() => { + localStorage.clear(); + }); + + it('saveDismissed + loadDismissed round-trips entries', () => { + const now = Date.now(); + saveDismissed(KEY, [ + { id: 'a', at: now }, + { id: 'b', at: now - 1000 }, + ]); + const loaded = loadDismissed(KEY, TTL); + expect(loaded).toEqual([ + { id: 'a', at: now }, + { id: 'b', at: now - 1000 }, + ]); + }); + + it('loadDismissed returns [] when key absent', () => { + expect(loadDismissed(KEY, TTL)).toEqual([]); + }); + + it('loadDismissed drops entries past the TTL cutoff', () => { + const now = Date.now(); + saveDismissed(KEY, [ + { id: 'fresh', at: now - 1000 }, + { id: 'stale', at: now - (TTL + 1000) }, + ]); + const loaded = loadDismissed(KEY, TTL); + expect(loaded.map((e) => e.id)).toEqual(['fresh']); + }); + + it('loadDismissed returns [] on malformed JSON without throwing', () => { + localStorage.setItem(KEY, '{not json'); + expect(loadDismissed(KEY, TTL)).toEqual([]); + }); + + it('loadDismissed filters out entries with wrong shape', () => { + const now = Date.now(); + localStorage.setItem( + KEY, + JSON.stringify([ + { id: 'good', at: now }, + { id: 123, at: now }, + { id: 'bad-at', at: 'nope' }, + null, + 'string-entry', + ]), + ); + const loaded = loadDismissed(KEY, TTL); + expect(loaded.map((e) => e.id)).toEqual(['good']); + }); + + it('isDismissed matches by id', () => { + const list = [{ id: 'a', at: 1 }]; + expect(isDismissed(list, 'a')).toBe(true); + expect(isDismissed(list, 'b')).toBe(false); + expect(isDismissed([], 'a')).toBe(false); + }); +}); diff --git a/frontend/src/notifications/dismissedPersistence.ts b/frontend/src/notifications/dismissedPersistence.ts new file mode 100644 index 00000000..c5ae6300 --- /dev/null +++ b/frontend/src/notifications/dismissedPersistence.ts @@ -0,0 +1,42 @@ +// Persisted dismissal lists for SSE-driven toasts. Without persistence +// the next page's backlog replay re-fires the events and pops the +// toast back. TTL matches the backend's stream retention. + +export interface DismissedEntry { + id: string; + at: number; +} + +export function loadDismissed(key: string, ttlMs: number): DismissedEntry[] { + if (typeof localStorage === 'undefined') return []; + try { + const raw = localStorage.getItem(key); + if (!raw) return []; + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return []; + const cutoff = Date.now() - ttlMs; + return parsed.filter( + (e): e is DismissedEntry => + !!e && + typeof e === 'object' && + typeof (e as DismissedEntry).id === 'string' && + typeof (e as DismissedEntry).at === 'number' && + (e as DismissedEntry).at >= cutoff, + ); + } catch { + return []; + } +} + +export function saveDismissed(key: string, list: DismissedEntry[]): void { + if (typeof localStorage === 'undefined') return; + try { + localStorage.setItem(key, JSON.stringify(list)); + } catch { + // Best-effort: ignore quota / private-mode errors. + } +} + +export function isDismissed(list: DismissedEntry[], id: string): boolean { + return list.some((e) => e.id === id); +} diff --git a/frontend/src/notifications/notificationsSlice.test.ts b/frontend/src/notifications/notificationsSlice.test.ts new file mode 100644 index 00000000..27839d47 --- /dev/null +++ b/frontend/src/notifications/notificationsSlice.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it, vi, afterEach } from 'vitest'; + +import reducer, { + dismissToolApproval, + sseEventReceived, + sseLastEventIdReset, + type SSEEvent, +} from './notificationsSlice'; + +const baseEvent = (overrides: Partial = {}): SSEEvent => ({ + id: 'evt-1', + type: 'source.ingest.progress', + ...overrides, +}); + +const seedState = () => reducer(undefined, { type: '@@INIT' }); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('sseEventReceived', () => { + it('dedupes by id when the same envelope arrives twice', () => { + let state = seedState(); + state = reducer(state, sseEventReceived(baseEvent({ id: 'a' }))); + state = reducer(state, sseEventReceived(baseEvent({ id: 'a' }))); + expect(state.recentEvents).toHaveLength(1); + expect(state.recentEvents[0].id).toBe('a'); + }); + + it('does not update lastEventId for envelopes without an id (backlog.truncated)', () => { + let state = seedState(); + state = reducer(state, sseEventReceived(baseEvent({ id: 'cursor-1' }))); + expect(state.lastEventId).toBe('cursor-1'); + state = reducer( + state, + sseEventReceived({ type: 'backlog.truncated' } as SSEEvent), + ); + expect(state.lastEventId).toBe('cursor-1'); + }); + + it('caps recentEvents at 100 entries (oldest evicted)', () => { + let state = seedState(); + for (let i = 0; i < 105; i += 1) { + state = reducer(state, sseEventReceived(baseEvent({ id: `e-${i}` }))); + } + expect(state.recentEvents).toHaveLength(100); + // Newest first. + expect(state.recentEvents[0].id).toBe('e-104'); + expect(state.recentEvents[state.recentEvents.length - 1].id).toBe('e-5'); + }); +}); + +describe('sseLastEventIdReset', () => { + it('clears lastEventId back to null', () => { + let state = seedState(); + state = reducer(state, sseEventReceived(baseEvent({ id: 'x' }))); + expect(state.lastEventId).toBe('x'); + state = reducer(state, sseLastEventIdReset()); + expect(state.lastEventId).toBeNull(); + }); +}); + +describe('dismissToolApproval', () => { + it('dedupes by id and refreshes the timestamp', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')); + let state = seedState(); + state = reducer(state, dismissToolApproval('approval-1')); + const firstAt = state.dismissedToolApprovals[0].at; + + vi.setSystemTime(new Date('2026-01-01T00:05:00Z')); + state = reducer(state, dismissToolApproval('approval-1')); + expect(state.dismissedToolApprovals).toHaveLength(1); + expect(state.dismissedToolApprovals[0].at).toBeGreaterThan(firstAt); + }); + + it('evicts entries older than the 24h TTL', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')); + let state = seedState(); + state = reducer(state, dismissToolApproval('old-1')); + + // Move past the 24h TTL window. + vi.setSystemTime(new Date('2026-01-02T00:00:01Z')); + state = reducer(state, dismissToolApproval('fresh-1')); + const ids = state.dismissedToolApprovals.map((entry) => entry.id); + expect(ids).toEqual(['fresh-1']); + }); + + it('applies the 200-entry cap as a backstop after TTL filtering', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')); + let state = seedState(); + // Insert 205 distinct ids within the TTL window. + for (let i = 0; i < 205; i += 1) { + // Advance time slightly so the at-values are distinct but well + // inside the 24h TTL. + vi.setSystemTime( + new Date(`2026-01-01T00:00:${(i % 60).toString().padStart(2, '0')}Z`), + ); + state = reducer(state, dismissToolApproval(`id-${i}`)); + } + expect(state.dismissedToolApprovals).toHaveLength(200); + // The 200-cap keeps the most recently pushed ids. + expect(state.dismissedToolApprovals[0].id).toBe('id-5'); + expect(state.dismissedToolApprovals[199].id).toBe('id-204'); + }); +}); diff --git a/frontend/src/notifications/notificationsSlice.ts b/frontend/src/notifications/notificationsSlice.ts new file mode 100644 index 00000000..7708d312 --- /dev/null +++ b/frontend/src/notifications/notificationsSlice.ts @@ -0,0 +1,200 @@ +import { createSelector, createSlice, PayloadAction } from '@reduxjs/toolkit'; + +import { RootState } from '../store'; +import { loadDismissed, saveDismissed } from './dismissedPersistence'; + +const DISMISSED_TOOL_APPROVALS_STORAGE_KEY = 'docsgpt:dismissedToolApprovals'; + +/** + * Envelope shape published by the backend SSE endpoint + * (`application/events/publisher.py`). Mirrors the wire JSON 1:1. + */ +export interface SSEEvent

    > { + id?: string; + type: string; + ts?: string; + user_id?: string; + topic?: string; + scope?: { kind: string; id: string }; + payload?: P; +} + +/** + * Connection-health state machine the rest of the app reads via + * ``selectPushChannelHealthy`` to gate polling-fallback behaviour. + * + * - ``connecting`` — initial fetch in flight, or reconnecting after drop. + * - ``healthy`` — at least one event (data or keepalive) received and + * the stream has been open >2s. + * - ``unhealthy`` — last attempt failed or has been dropped without a + * successful re-establish; fall back to polling. + */ +export type PushHealth = 'connecting' | 'healthy' | 'unhealthy'; + +interface NotificationsState { + health: PushHealth; + /** Most-recent server-issued id; sent back as ``Last-Event-ID`` on reconnect. */ + lastEventId: string | null; + /** Bounded ring of recent events for the in-app notifications surface. */ + recentEvents: SSEEvent[]; + /** + * Wallclock ms of last received data-bearing event. SSE keepalives + * are comment lines (no ``id:``/``data:``) and do NOT update this — + * they're filtered out at the parser level. + */ + lastEventReceivedAt: number | null; + /** + * Event ids of ``tool.approval.required`` notifications the user + * dismissed (close button or by navigating into the conversation), + * each tagged with the wallclock ms at which it was dismissed. + * Keyed by the SSE event id so a *new* approval for the same + * conversation re-surfaces; the dismissal only suppresses the one + * specific paused-tool prompt. + * + * Entries are evicted by TTL first (anything older than + * ``DISMISSED_TOOL_APPROVALS_TTL_MS``), then by FIFO cap. The TTL + * matters because a pure FIFO with a small cap can evict a *still- + * pending* approval id before the user acts on it — re-popping the + * toast on the next dispatch. The 24h ceiling is longer than any + * plausible approval-pending window. + */ + dismissedToolApprovals: Array<{ id: string; at: number }>; +} + +const RECENT_EVENTS_CAP = 100; +const DISMISSED_TOOL_APPROVALS_CAP = 200; +const DISMISSED_TOOL_APPROVALS_TTL_MS = 24 * 60 * 60 * 1000; + +const initialState: NotificationsState = { + health: 'connecting', + lastEventId: null, + recentEvents: [], + lastEventReceivedAt: null, + // Hydrate from localStorage: SSE backlog replay re-delivers the + // originating ``tool.approval.required`` envelopes on reload. + dismissedToolApprovals: loadDismissed( + DISMISSED_TOOL_APPROVALS_STORAGE_KEY, + DISMISSED_TOOL_APPROVALS_TTL_MS, + ), +}; + +export const notificationsSlice = createSlice({ + name: 'notifications', + initialState, + reducers: { + sseEventReceived: (state, action: PayloadAction) => { + const e = action.payload; + // Drop immediate duplicates. Snapshot replay + live tail can + // both deliver the same id when the live pubsub frame and the + // replay XRANGE overlap, and consumers that walk + // ``recentEvents`` (FileTree, ConnectorTree, MCPServerModal, + // ToolApprovalToast) would otherwise act on the same envelope + // twice. The route's dedup floor catches the common case; this + // is a belt-and-suspenders for in-tab StrictMode double-mounts + // and any envelope that slips through with the same id. + if (e.id && state.recentEvents[0]?.id === e.id) return; + state.recentEvents.unshift(e); + if (state.recentEvents.length > RECENT_EVENTS_CAP) { + state.recentEvents.length = RECENT_EVENTS_CAP; + } + if (e.id) state.lastEventId = e.id; + state.lastEventReceivedAt = Date.now(); + }, + sseHealthChanged: (state, action: PayloadAction) => { + state.health = action.payload; + }, + /** + * Lifecycle helper used by reconnect bookkeeping — does not record + * an event, just stamps "we heard from the server" so the polling + * fallback stays disabled while keepalives arrive. + */ + sseHeartbeat: (state) => { + state.lastEventReceivedAt = Date.now(); + }, + sseLastEventIdReset: (state) => { + // Backlog truncated — drop the cursor so the next reconnect + // doesn't try to resume past the retained window. + state.lastEventId = null; + }, + /** + * Advance the cursor without recording an event. Fired for every + * id-bearing frame including keepalives and id-only comments, + * so the slice cursor tracks the freshest id the wire has + * delivered even when no envelope was dispatched. Without this, + * ``lastEventId`` would only advance via ``sseEventReceived`` and + * a long quiet period of keepalives would leave it stale — + * eventually re-snapshotting the same backlog on each reconnect + * and exhausting the per-user replay budget. + */ + sseLastEventIdAdvanced: (state, action: PayloadAction) => { + state.lastEventId = action.payload; + }, + clearRecentEvents: (state) => { + state.recentEvents = []; + }, + /** + * Suppress a ``tool.approval.required`` notification by the SSE + * event id. The toast surface filters dismissed ids out; a *new* + * approval event for the same conversation has a different id + * and re-surfaces, which is the desired UX (each pause is its + * own decision). + */ + dismissToolApproval: (state, action: PayloadAction) => { + const id = action.payload; + const now = Date.now(); + // Evict expired entries first so the TTL — not the FIFO cap — + // governs when stale ids drop, keeping still-pending approvals + // suppressed. + const cutoff = now - DISMISSED_TOOL_APPROVALS_TTL_MS; + state.dismissedToolApprovals = state.dismissedToolApprovals.filter( + (entry) => entry.at >= cutoff && entry.id !== id, + ); + state.dismissedToolApprovals.push({ id, at: now }); + if (state.dismissedToolApprovals.length > DISMISSED_TOOL_APPROVALS_CAP) { + state.dismissedToolApprovals = state.dismissedToolApprovals.slice( + -DISMISSED_TOOL_APPROVALS_CAP, + ); + } + saveDismissed( + DISMISSED_TOOL_APPROVALS_STORAGE_KEY, + state.dismissedToolApprovals, + ); + }, + }, +}); + +export const { + sseEventReceived, + sseHealthChanged, + sseHeartbeat, + sseLastEventIdReset, + sseLastEventIdAdvanced, + clearRecentEvents, + dismissToolApproval, +} = notificationsSlice.actions; + +export const selectSseHealth = (state: RootState): PushHealth => + state.notifications.health; + +export const selectPushChannelHealthy = (state: RootState): boolean => + state.notifications.health === 'healthy'; + +export const selectLastEventId = (state: RootState): string | null => + state.notifications.lastEventId; + +export const selectLastEventReceivedAt = (state: RootState): number | null => + state.notifications.lastEventReceivedAt; + +export const selectRecentEvents = (state: RootState): SSEEvent[] => + state.notifications.recentEvents; + +// Memoised so ``useSelector`` consumers don't re-render on every +// unrelated ``notifications`` state change — the underlying ``{id,at}`` +// array only changes when ``dismissToolApproval`` runs, but the +// projected ``.map`` would otherwise return a fresh array each call. +export const selectDismissedToolApprovals = createSelector( + (state: RootState) => state.notifications.dismissedToolApprovals, + (entries) => entries.map((entry) => entry.id), +); + +export default notificationsSlice.reducer; diff --git a/frontend/src/store.ts b/frontend/src/store.ts index 571a367c..1778cd6f 100644 --- a/frontend/src/store.ts +++ b/frontend/src/store.ts @@ -4,6 +4,7 @@ import agentPreviewReducer from './agents/agentPreviewSlice'; import workflowPreviewReducer from './agents/workflow/workflowPreviewSlice'; import { conversationSlice } from './conversation/conversationSlice'; import { sharedConversationSlice } from './conversation/sharedConversationSlice'; +import notificationsReducer from './notifications/notificationsSlice'; import { getStoredRecentDocs } from './preferences/preferenceApi'; import { Preference, @@ -67,6 +68,7 @@ const store = configureStore({ upload: uploadReducer, agentPreview: agentPreviewReducer, workflowPreview: workflowPreviewReducer, + notifications: notificationsReducer, }, middleware: (getDefaultMiddleware) => getDefaultMiddleware().concat(prefListenerMiddleware.middleware), diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 86fd9a84..0e618248 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -2,9 +2,9 @@ import { useCallback, useState } from 'react'; import { nanoid } from '@reduxjs/toolkit'; import { useDropzone } from 'react-dropzone'; import { useTranslation } from 'react-i18next'; -import { useDispatch, useSelector } from 'react-redux'; +import { useDispatch, useSelector, useStore } from 'react-redux'; -import userService from '../api/services/userService'; +import type { RootState } from '../store'; import { getSessionToken } from '../utils/providerUtils'; import Dropdown from '../components/Dropdown'; import Input from '../components/Input'; @@ -298,6 +298,7 @@ function Upload({ const { t } = useTranslation(); const dispatch = useDispatch(); + const store = useStore(); const ingestorOptions: IngestorOption[] = IngestorFormSchemas.filter( (schema) => (schema.validate ? schema.validate() : true), @@ -334,110 +335,113 @@ function Upload({ [dispatch], ); + /** + * Wait for the source.ingest.* SSE pipeline to flip this task into a + * terminal state, then run the post-completion side effects: refresh + * the global source list, auto-select the new doc, and fire the + * caller's ``onSuccessfulUpload`` hook. The slice's extraReducer + * (uploadSlice.ts) is the sole driver of the task's status; we only + * subscribe so the side effects can fire after the modal has closed. + */ const trackTraining = useCallback( - (backendTaskId: string, clientTaskId: string) => { - let timeoutId: number | null = null; + (clientTaskId: string) => { + let handled = false; - const poll = () => { - userService - .getTaskStatus(backendTaskId, null) - .then((response) => response.json()) - .then(async (data) => { - if (!data.success && data.message) { - if (timeoutId !== null) { - clearTimeout(timeoutId); - timeoutId = null; - } - handleTaskFailure(clientTaskId, data.message); - return; - } - - if (data.status === 'SUCCESS') { - if (timeoutId !== null) { - clearTimeout(timeoutId); - timeoutId = null; - } - - const docs = await getDocs(token); - dispatch(setSourceDocs(docs)); - - if (Array.isArray(docs)) { - const existingDocIds = new Set( - (Array.isArray(sourceDocs) ? sourceDocs : []) - .map((doc: Doc) => doc?.id) - .filter((id): id is string => Boolean(id)), - ); - const newDoc = docs.find( - (doc: Doc) => doc.id && !existingDocIds.has(doc.id), - ); - if (newDoc) { - // If only one doc is selected, replace it completely - // If multiple docs are selected, append the new doc - if (selectedDocs.length === 1) { - dispatch(setSelectedDocs([newDoc])); - } else { - dispatch(setSelectedDocs([...selectedDocs, newDoc])); - } + const handleTerminal = (status: 'completed' | 'failed') => { + if (handled) return; + handled = true; + if (status !== 'completed') return; + getDocs(token) + .then((docs) => { + dispatch(setSourceDocs(docs)); + if (Array.isArray(docs)) { + const existingDocIds = new Set( + (Array.isArray(sourceDocs) ? sourceDocs : []) + .map((doc: Doc) => doc?.id) + .filter((id): id is string => Boolean(id)), + ); + const newDoc = docs.find( + (doc: Doc) => doc.id && !existingDocIds.has(doc.id), + ); + if (newDoc) { + // If only one doc is selected, replace it completely + // If multiple docs are selected, append the new doc + if (selectedDocs.length === 1) { + dispatch(setSelectedDocs([newDoc])); + } else { + dispatch(setSelectedDocs([...selectedDocs, newDoc])); } } - - if (data.result?.limited) { - dispatch( - updateUploadTask({ - id: clientTaskId, - updates: { - status: 'failed', - progress: 100, - errorMessage: t('modals.uploadDoc.progress.tokenLimit'), - }, - }), - ); - } else { - dispatch( - updateUploadTask({ - id: clientTaskId, - updates: { - status: 'completed', - progress: 100, - errorMessage: undefined, - }, - }), - ); - onSuccessfulUpload?.(); - } - } else if (data.status === 'FAILURE') { - if (timeoutId !== null) { - clearTimeout(timeoutId); - timeoutId = null; - } - handleTaskFailure(clientTaskId, data.result?.message); - } else if (data.status === 'PROGRESS') { - dispatch( - updateUploadTask({ - id: clientTaskId, - updates: { - status: 'training', - progress: Math.min(100, data.result?.current ?? 0), - }, - }), - ); - timeoutId = window.setTimeout(poll, 5000); - } else { - timeoutId = window.setTimeout(poll, 5000); } + onSuccessfulUpload?.(); }) - .catch((error) => { - if (timeoutId !== null) { - clearTimeout(timeoutId); - timeoutId = null; - } - handleTaskFailure(clientTaskId, error?.message); + .catch((err) => { + console.error( + 'SSE-driven post-completion source-list refresh failed:', + err, + ); }); }; - timeoutId = window.setTimeout(poll, 3000); + const check = () => { + const state = store.getState(); + const task = state.upload.tasks.find((t) => t.id === clientTaskId); + if (!task) return false; + if (task.status === 'completed' || task.status === 'failed') { + handleTerminal(task.status); + return true; + } + // Recover from the race where the terminal SSE landed before + // ``xhr.onload`` populated ``task.sourceId`` — the slice + // silently drops such events (no task to match by sourceId). + // Mirrors ConnectorTree/FileTree's ``recentEvents`` walk. + if (task.sourceId) { + for (const event of state.notifications.recentEvents) { + if (event.scope?.id !== task.sourceId) continue; + if (event.type === 'source.ingest.completed') { + handleTerminal('completed'); + return true; + } + if (event.type === 'source.ingest.failed') { + handleTerminal('failed'); + return true; + } + } + } + return false; + }; + + if (check()) return; + const MAX_WAIT_MS = 5 * 60_000; + let unsubscribe: (() => void) | null = null; + const timer = window.setTimeout(() => { + unsubscribe?.(); + if (!handled) { + handled = true; + console.warn( + 'trackTraining: timed out waiting for terminal SSE', + clientTaskId, + ); + dispatch( + updateUploadTask({ + id: clientTaskId, + updates: { + status: 'failed', + errorMessage: + 'Timed out waiting for ingest completion. The ingest may still be running — please refresh to check.', + }, + }), + ); + } + }, MAX_WAIT_MS); + unsubscribe = store.subscribe(() => { + if (check()) { + window.clearTimeout(timer); + unsubscribe?.(); + } + }); }, - [dispatch, handleTaskFailure, onSuccessfulUpload, sourceDocs, t, token], + [dispatch, onSuccessfulUpload, selectedDocs, sourceDocs, store, token], ); const onDrop = useCallback( @@ -499,19 +503,23 @@ function Upload({ xhr.onload = () => { if (xhr.status >= 200 && xhr.status < 300) { try { - const parsed = JSON.parse(xhr.responseText) as { task_id?: string }; + const parsed = JSON.parse(xhr.responseText) as { + task_id?: string; + source_id?: string; + }; if (parsed.task_id) { dispatch( updateUploadTask({ id: clientTaskId, updates: { taskId: parsed.task_id, + sourceId: parsed.source_id, status: 'training', progress: 0, }, }), ); - trackTraining(parsed.task_id, clientTaskId); + trackTraining(clientTaskId); } else { dispatch( updateUploadTask({ @@ -627,19 +635,23 @@ function Upload({ xhr.onload = () => { if (xhr.status >= 200 && xhr.status < 300) { try { - const response = JSON.parse(xhr.responseText) as { task_id?: string }; + const response = JSON.parse(xhr.responseText) as { + task_id?: string; + source_id?: string; + }; if (response.task_id) { dispatch( updateUploadTask({ id: clientTaskId, updates: { taskId: response.task_id, + sourceId: response.source_id, status: 'training', progress: 0, }, }), ); - trackTraining(response.task_id, clientTaskId); + trackTraining(clientTaskId); } else { dispatch( updateUploadTask({ diff --git a/frontend/src/upload/uploadSlice.test.ts b/frontend/src/upload/uploadSlice.test.ts new file mode 100644 index 00000000..29c11ed4 --- /dev/null +++ b/frontend/src/upload/uploadSlice.test.ts @@ -0,0 +1,458 @@ +import { beforeEach, describe, expect, it } from 'vitest'; + +import notificationsReducer, { + sseEventReceived, + type SSEEvent, +} from '../notifications/notificationsSlice'; +import reducer, { + addAttachment, + addUploadTask, + dismissUploadTask, + updateAttachment, + updateUploadTask, + type Attachment, + type UploadTask, +} from './uploadSlice'; + +const SOURCE_ID = 'src-1'; + +const makeTask = (overrides: Partial = {}): UploadTask => ({ + id: 't-1', + fileName: 'doc.pdf', + progress: 0, + status: 'preparing', + sourceId: SOURCE_ID, + ...overrides, +}); + +const stateWithTask = (task: UploadTask) => + reducer(undefined, addUploadTask(task)); + +const ingest = ( + type: string, + payload: Record = {}, + scopeId = SOURCE_ID, +) => + sseEventReceived({ + id: `id-${type}`, + type, + scope: { kind: 'source', id: scopeId }, + payload, + }); + +describe('dismissal persistence across reload', () => { + const STORAGE_KEY = 'docsgpt:dismissedUploadSourceIds'; + const SRC = 'src-persisted'; + + // Mirrors initialState as if hydrated from localStorage. + const seedState = (entries: { id: string; at: number }[]) => + reducer( + { attachments: [], tasks: [], dismissedSourceIds: entries }, + { type: '@@INIT' }, + ); + + beforeEach(() => { + localStorage.clear(); + }); + + it('dismissUploadTask writes the task sourceId to localStorage', () => { + let state = stateWithTask( + makeTask({ id: 't-dismiss', sourceId: SRC, status: 'completed' }), + ); + state = reducer(state, dismissUploadTask('t-dismiss')); + expect(state.tasks[0].dismissed).toBe(true); + expect(state.dismissedSourceIds).toHaveLength(1); + expect(state.dismissedSourceIds[0].id).toBe(SRC); + const persisted = JSON.parse(localStorage.getItem(STORAGE_KEY) ?? '[]'); + expect(persisted).toHaveLength(1); + expect(persisted[0].id).toBe(SRC); + }); + + it('skips persistence when the task has no sourceId yet', () => { + let state = stateWithTask( + makeTask({ id: 't-no-src', sourceId: undefined, status: 'preparing' }), + ); + state = reducer(state, dismissUploadTask('t-no-src')); + expect(state.tasks[0].dismissed).toBe(true); + expect(state.dismissedSourceIds).toHaveLength(0); + expect(localStorage.getItem(STORAGE_KEY)).toBeNull(); + }); + + it('auto-create on refresh marks the task dismissed when sourceId is in the persisted list', () => { + const state = seedState([{ id: SRC, at: Date.now() }]); + const next = reducer( + state, + ingest('source.ingest.progress', { current: 40 }, SRC), + ); + const recovered = next.tasks.find((t) => t.sourceId === SRC); + expect(recovered).toBeDefined(); + expect(recovered!.dismissed).toBe(true); + expect(recovered!.progress).toBe(40); + }); + + it('terminal events do NOT un-dismiss a task whose sourceId was previously dismissed', () => { + const state = seedState([{ id: SRC, at: Date.now() }]); + let next = reducer(state, ingest('source.ingest.queued', {}, SRC)); + expect(next.tasks[0].dismissed).toBe(true); + next = reducer(next, ingest('source.ingest.completed', {}, SRC)); + // Even though `wasTerminal` is false (just transitioned), the + // persisted dismissal keeps the toast closed. + expect(next.tasks[0].status).toBe('completed'); + expect(next.tasks[0].dismissed).toBe(true); + }); + + it('updateUploadTask does not un-dismiss on terminal when sourceId was previously dismissed', () => { + const state = reducer( + { + attachments: [], + tasks: [ + makeTask({ + id: 't-1', + sourceId: SRC, + status: 'training', + dismissed: true, + }), + ], + dismissedSourceIds: [{ id: SRC, at: Date.now() }], + }, + { type: '@@INIT' }, + ); + const next = reducer( + state, + updateUploadTask({ + id: 't-1', + updates: { status: 'completed' }, + }), + ); + expect(next.tasks[0].status).toBe('completed'); + expect(next.tasks[0].dismissed).toBe(true); + }); + + it('un-dismisses normally for sourceIds NOT in the persisted list', () => { + const state = reducer(undefined, { type: '@@INIT' }); + const populated = reducer( + state, + addUploadTask( + makeTask({ + id: 't-fresh', + sourceId: 'src-fresh', + status: 'training', + dismissed: true, + }), + ), + ); + const next = reducer( + populated, + updateUploadTask({ id: 't-fresh', updates: { status: 'completed' } }), + ); + expect(next.tasks[0].dismissed).toBe(false); + }); +}); + +describe('refresh recovery — auto-create from SSE when no task matches', () => { + it('creates a task on queued for an unknown sourceId', () => { + let state = reducer(undefined, addUploadTask(makeTask({ id: 'other' }))); + state = reducer( + state, + ingest( + 'source.ingest.queued', + { filename: 'crawler.json', job_name: 'docs' }, + 'src-recovery', + ), + ); + const recovered = state.tasks.find((t) => t.sourceId === 'src-recovery'); + expect(recovered).toBeDefined(); + expect(recovered!.status).toBe('training'); + expect(recovered!.fileName).toBe('crawler.json'); + expect(recovered!.dismissed).toBe(false); + }); + + it('creates a task on progress for an unknown sourceId and applies the percent', () => { + let state: ReturnType = reducer(undefined, { + type: '@@INIT', + }); + state = reducer( + state, + ingest( + 'source.ingest.progress', + { current: 55, total: 10, embedded_chunks: 5 }, + 'src-progress', + ), + ); + expect(state.tasks).toHaveLength(1); + expect(state.tasks[0].sourceId).toBe('src-progress'); + expect(state.tasks[0].status).toBe('training'); + expect(state.tasks[0].progress).toBe(55); + }); + + it('does NOT create a task on completed for an unknown sourceId (avoids backlog toast spam)', () => { + let state: ReturnType = reducer(undefined, { + type: '@@INIT', + }); + state = reducer( + state, + ingest('source.ingest.completed', { tokens: 16 }, 'src-stale'), + ); + expect(state.tasks).toHaveLength(0); + }); + + it('creates a task on failed for an unknown sourceId so error surfaces post-refresh', () => { + let state: ReturnType = reducer(undefined, { + type: '@@INIT', + }); + state = reducer( + state, + ingest( + 'source.ingest.failed', + { error: 'embed worker died' }, + 'src-failed', + ), + ); + expect(state.tasks).toHaveLength(1); + expect(state.tasks[0].status).toBe('failed'); + expect(state.tasks[0].errorMessage).toBe('embed worker died'); + expect(state.tasks[0].dismissed).toBe(false); + }); + + it('falls back to job_name then sourceId when filename is absent', () => { + let state: ReturnType = reducer(undefined, { + type: '@@INIT', + }); + state = reducer( + state, + ingest('source.ingest.queued', { job_name: 'docs-docs' }, 'src-jn'), + ); + expect(state.tasks[0].fileName).toBe('docs-docs'); + state = reducer(state, ingest('source.ingest.queued', {}, 'src-only')); + expect(state.tasks.find((t) => t.sourceId === 'src-only')!.fileName).toBe( + 'src-only', + ); + }); + + it('subsequent progress events update the recovered task in place', () => { + let state: ReturnType = reducer(undefined, { + type: '@@INIT', + }); + state = reducer( + state, + ingest('source.ingest.queued', { filename: 'a.txt' }, 'src-flow'), + ); + state = reducer( + state, + ingest('source.ingest.progress', { current: 30 }, 'src-flow'), + ); + state = reducer( + state, + ingest('source.ingest.progress', { current: 80 }, 'src-flow'), + ); + state = reducer( + state, + ingest('source.ingest.completed', { tokens: 100 }, 'src-flow'), + ); + expect(state.tasks).toHaveLength(1); + expect(state.tasks[0].status).toBe('completed'); + expect(state.tasks[0].progress).toBe(100); + }); +}); + +describe('source.ingest.queued', () => { + it('does not regress a task already in training', () => { + let state = stateWithTask(makeTask({ status: 'training', progress: 42 })); + state = reducer(state, ingest('source.ingest.queued')); + expect(state.tasks[0].status).toBe('training'); + expect(state.tasks[0].progress).toBe(42); + }); + + it('transitions preparing -> training and zeros progress', () => { + let state = stateWithTask(makeTask({ status: 'preparing', progress: 12 })); + state = reducer(state, ingest('source.ingest.queued')); + expect(state.tasks[0].status).toBe('training'); + expect(state.tasks[0].progress).toBe(0); + }); +}); + +describe('source.ingest.progress', () => { + it('clamps to 0..100 and is monotonic', () => { + let state = stateWithTask(makeTask({ status: 'training' })); + state = reducer(state, ingest('source.ingest.progress', { current: 30 })); + expect(state.tasks[0].progress).toBe(30); + // Higher value advances. + state = reducer(state, ingest('source.ingest.progress', { current: 150 })); + expect(state.tasks[0].progress).toBe(100); + // Lower value never regresses. + state = reducer(state, ingest('source.ingest.progress', { current: 50 })); + expect(state.tasks[0].progress).toBe(100); + // Negative gets clamped at 0 but still cannot regress already-higher. + state = reducer(state, ingest('source.ingest.progress', { current: -10 })); + expect(state.tasks[0].progress).toBe(100); + }); +}); + +describe('source.ingest.completed', () => { + it('transitions training -> completed and sets dismissed=false', () => { + let state = stateWithTask( + makeTask({ status: 'training', dismissed: true }), + ); + state = reducer(state, ingest('source.ingest.completed')); + expect(state.tasks[0].status).toBe('completed'); + expect(state.tasks[0].progress).toBe(100); + expect(state.tasks[0].dismissed).toBe(false); + expect(state.tasks[0].tokenLimitReached).toBe(false); + }); + + it('with limited=true transitions training -> failed and flags tokenLimitReached', () => { + let state = stateWithTask( + makeTask({ status: 'training', dismissed: true }), + ); + state = reducer( + state, + ingest('source.ingest.completed', { limited: true }), + ); + expect(state.tasks[0].status).toBe('failed'); + expect(state.tasks[0].progress).toBe(100); + expect(state.tasks[0].tokenLimitReached).toBe(true); + expect(state.tasks[0].dismissed).toBe(false); + }); + + it('does not re-un-dismiss when a duplicate terminal event arrives', () => { + // Initial terminal — wasTerminal=false, dismissed flipped to false. + let state = stateWithTask(makeTask({ status: 'training' })); + state = reducer(state, ingest('source.ingest.completed')); + expect(state.tasks[0].dismissed).toBe(false); + + // User dismisses the toast manually. + state = { + ...state, + tasks: state.tasks.map((t) => ({ ...t, dismissed: true })), + }; + + // Duplicate terminal envelope (StrictMode remount, reconnect overlap). + state = reducer(state, ingest('source.ingest.completed')); + expect(state.tasks[0].status).toBe('completed'); + expect(state.tasks[0].dismissed).toBe(true); + }); +}); + +describe('source.ingest.failed', () => { + it('transitions training -> failed with the error message', () => { + let state = stateWithTask(makeTask({ status: 'training' })); + state = reducer( + state, + ingest('source.ingest.failed', { error: 'parser blew up' }), + ); + expect(state.tasks[0].status).toBe('failed'); + expect(state.tasks[0].errorMessage).toBe('parser blew up'); + expect(state.tasks[0].dismissed).toBe(false); + }); + + it('does not re-un-dismiss when a duplicate failed event arrives', () => { + let state = stateWithTask(makeTask({ status: 'training' })); + state = reducer(state, ingest('source.ingest.failed', { error: 'oops' })); + state = { + ...state, + tasks: state.tasks.map((t) => ({ ...t, dismissed: true })), + }; + state = reducer(state, ingest('source.ingest.failed', { error: 'oops' })); + expect(state.tasks[0].dismissed).toBe(true); + }); +}); + +describe('attachment race recovery', () => { + const ATTACHMENT_ID = 'att-1'; + const CLIENT_ID = 'ui-1'; + + const attEvent = ( + type: string, + payload: Record = {}, + ): SSEEvent => ({ + id: `id-${type}`, + type, + scope: { kind: 'attachment', id: ATTACHMENT_ID }, + payload, + }); + + const makeAttachment = (overrides: Partial = {}): Attachment => ({ + id: CLIENT_ID, + fileName: 'small.pdf', + progress: 10, + status: 'processing', + taskId: 'celery-1', + attachmentId: ATTACHMENT_ID, + ...overrides, + }); + + it('drops attachment.completed silently when no row matches attachmentId', () => { + const state = reducer( + undefined, + sseEventReceived(attEvent('attachment.completed', { token_count: 42 })), + ); + expect(state.attachments).toHaveLength(0); + }); + + it('lands the terminal envelope in notifications.recentEvents for later recovery', () => { + const notifState = notificationsReducer( + undefined, + sseEventReceived(attEvent('attachment.completed', { token_count: 7 })), + ); + expect(notifState.recentEvents).toHaveLength(1); + expect(notifState.recentEvents[0].scope?.id).toBe(ATTACHMENT_ID); + expect(notifState.recentEvents[0].type).toBe('attachment.completed'); + }); + + it('reconciler dispatch flips the row to completed after the late row addition', () => { + // Full race: terminal SSE first, then xhr.onload adds the row, + // then trackAttachment.check() walks recentEvents and dispatches. + const terminal = attEvent('attachment.completed', { token_count: 99 }); + const notifState = notificationsReducer( + undefined, + sseEventReceived(terminal), + ); + + let uploadState = reducer(undefined, addAttachment(makeAttachment())); + expect(uploadState.attachments[0].status).toBe('processing'); + + const found = notifState.recentEvents.find( + (e) => e.scope?.id === ATTACHMENT_ID && e.type === 'attachment.completed', + ); + expect(found).toBeDefined(); + const tokenCount = Number( + (found?.payload as { token_count?: unknown })?.token_count, + ); + uploadState = reducer( + uploadState, + updateAttachment({ + id: CLIENT_ID, + updates: { + status: 'completed', + progress: 100, + ...(Number.isFinite(tokenCount) ? { token_count: tokenCount } : {}), + }, + }), + ); + + expect(uploadState.attachments[0].status).toBe('completed'); + expect(uploadState.attachments[0].progress).toBe(100); + expect(uploadState.attachments[0].token_count).toBe(99); + }); + + it('attachment.failed envelope can drive a stuck row to failed via reconciler', () => { + const failed = attEvent('attachment.failed', { error: 'docling boom' }); + const notifState = notificationsReducer( + undefined, + sseEventReceived(failed), + ); + + let uploadState = reducer(undefined, addAttachment(makeAttachment())); + const found = notifState.recentEvents.find( + (e) => e.scope?.id === ATTACHMENT_ID && e.type === 'attachment.failed', + ); + expect(found).toBeDefined(); + + uploadState = reducer( + uploadState, + updateAttachment({ id: CLIENT_ID, updates: { status: 'failed' } }), + ); + + expect(uploadState.attachments[0].status).toBe('failed'); + }); +}); diff --git a/frontend/src/upload/uploadSlice.ts b/frontend/src/upload/uploadSlice.ts index 44c67843..b66978cc 100644 --- a/frontend/src/upload/uploadSlice.ts +++ b/frontend/src/upload/uploadSlice.ts @@ -1,12 +1,46 @@ import { createSelector, createSlice, PayloadAction } from '@reduxjs/toolkit'; + +import { + DismissedEntry, + isDismissed, + loadDismissed, + saveDismissed, +} from '../notifications/dismissedPersistence'; +import { sseEventReceived } from '../notifications/notificationsSlice'; import { RootState } from '../store'; +const DISMISSED_SOURCE_IDS_STORAGE_KEY = 'docsgpt:dismissedUploadSourceIds'; +const DISMISSED_SOURCE_IDS_TTL_MS = 24 * 60 * 60 * 1000; +const DISMISSED_SOURCE_IDS_CAP = 200; + +function recordDismissedSourceId( + list: DismissedEntry[], + sourceId: string, +): DismissedEntry[] { + const now = Date.now(); + const cutoff = now - DISMISSED_SOURCE_IDS_TTL_MS; + const next = list.filter( + (entry) => entry.at >= cutoff && entry.id !== sourceId, + ); + next.push({ id: sourceId, at: now }); + return next.length > DISMISSED_SOURCE_IDS_CAP + ? next.slice(-DISMISSED_SOURCE_IDS_CAP) + : next; +} + export interface Attachment { - id: string; // Unique identifier for the attachment (required for state management) + id: string; // Client-side state-management id (uuid generated in MessageInput) fileName: string; progress: number; status: 'uploading' | 'processing' | 'completed' | 'failed'; - taskId: string; // Server-assigned task ID (used for API calls) + taskId: string; // Server-assigned celery task ID (used for API calls) + /** + * Server-assigned attachment id (stable across the lifecycle — + * ``attachment.*`` SSE events use this in ``scope.id``). Set as + * soon as the upload response returns. Distinct from ``id`` + * (client) and ``taskId`` (celery). + */ + attachmentId?: string; token_count?: number; } @@ -23,18 +57,40 @@ export interface UploadTask { progress: number; status: UploadTaskStatus; taskId?: string; + /** + * Server-derived source id (uuid5 over the idempotency key) returned by + * the upload endpoint. Used to correlate inbound SSE ingest events + * (``source.ingest.*``) back to this task without consulting the + * polling endpoint. + */ + sourceId?: string; errorMessage?: string; dismissed?: boolean; + /** + * Flipped when ``source.ingest.completed`` carries + * ``payload.limited === true`` (the worker hit a token cap during + * ingest). The slice routes such events to a failed status and + * sets this flag so ``UploadToast`` can surface a translated + * token-limit message instead of a generic error. Forward-looking: + * no worker code path sets ``limited: true`` today. + */ + tokenLimitReached?: boolean; } interface UploadState { attachments: Attachment[]; tasks: UploadTask[]; + /** Persisted dismissed sourceIds; keeps backlog-replay auto-creates silent. */ + dismissedSourceIds: DismissedEntry[]; } const initialState: UploadState = { attachments: [], tasks: [], + dismissedSourceIds: loadDismissed( + DISMISSED_SOURCE_IDS_STORAGE_KEY, + DISMISSED_SOURCE_IDS_TTL_MS, + ), }; export const uploadSlice = createSlice({ @@ -103,9 +159,19 @@ export const uploadSlice = createSlice({ ); if (index !== -1) { const updates = action.payload.updates; + const existingSourceId = state.tasks[index].sourceId; + const incomingSourceId = updates.sourceId; + const effectiveSourceId = incomingSourceId ?? existingSourceId; + const sourceWasDismissed = + !!effectiveSourceId && + isDismissed(state.dismissedSourceIds, effectiveSourceId); - // When task completes or fails, set dismissed to false to notify user - if (updates.status === 'completed' || updates.status === 'failed') { + // Re-surface on terminal status, except when the sourceId was + // dismissed in a prior session (don't re-pop after reload). + if ( + (updates.status === 'completed' || updates.status === 'failed') && + !sourceWasDismissed + ) { state.tasks[index] = { ...state.tasks[index], ...updates, @@ -126,12 +192,184 @@ export const uploadSlice = createSlice({ ...state.tasks[index], dismissed: true, }; + // Persist by sourceId so a reload doesn't re-pop via the SSE + // backlog replay's auto-create branch. + const sourceId = state.tasks[index].sourceId; + if (sourceId) { + state.dismissedSourceIds = recordDismissedSourceId( + state.dismissedSourceIds, + sourceId, + ); + saveDismissed( + DISMISSED_SOURCE_IDS_STORAGE_KEY, + state.dismissedSourceIds, + ); + } } }, removeUploadTask: (state, action: PayloadAction) => { state.tasks = state.tasks.filter((task) => task.id !== action.payload); }, }, + extraReducers: (builder) => { + // Consume backend SSE ingest events for sub-second progress and + // terminal-status updates. The match is by ``sourceId`` (set by + // the upload endpoint's response). Polling stays as the + // correctness-of-record fallback in Upload.tsx. + builder.addCase(sseEventReceived, (state, action) => { + const e = action.payload; + const scopeId = + typeof e.scope?.id === 'string' && e.scope.id.length > 0 + ? e.scope.id + : undefined; + + // Attachment events flow through the same SSE pipe; route them + // to ``state.attachments`` matched by ``attachmentId``. SSE is + // the sole driver of attachment state transitions — polling + // has been removed. Events for attachments uploaded in another + // session are silently dropped. + if (e.type.startsWith('attachment.') && scopeId) { + const attachment = state.attachments.find( + (a) => a.attachmentId === scopeId, + ); + if (attachment) { + const payload = (e.payload || {}) as Record; + switch (e.type) { + case 'attachment.queued': + case 'attachment.progress': { + if ( + attachment.status === 'completed' || + attachment.status === 'failed' + ) { + break; + } + attachment.status = 'processing'; + const current = Number(payload.current); + if (Number.isFinite(current)) { + const clamped = Math.max(0, Math.min(100, current)); + if (clamped > attachment.progress) { + attachment.progress = clamped; + } + } + break; + } + case 'attachment.completed': { + attachment.status = 'completed'; + attachment.progress = 100; + // Replace the client-generated uuid with the server's + // attachment id so question submission + // (Conversation.tsx:174) sends an id the backend can + // resolve. Without this the backend would silently drop + // the attachment from the message context. + attachment.id = scopeId; + const tokenCount = Number(payload.token_count); + if (Number.isFinite(tokenCount)) { + attachment.token_count = tokenCount; + } + break; + } + case 'attachment.failed': { + attachment.status = 'failed'; + break; + } + default: + break; + } + } + return; + } + + if (!e.type.startsWith('source.ingest.')) return; + if (!scopeId) return; + // ``source.ingest.*`` payloads do not carry a celery task_id + // (see ``application/worker.py`` — only ``source_id`` / + // ``job_name`` / ``filename`` are published), so ``sourceId`` + // is the only correlation key. + const payload = (e.payload || {}) as Record; + let task = state.tasks.find((t) => t.sourceId === scopeId); + if (!task) { + // Auto-create on backlog/live SSE when the local task is gone + // (page refresh mid-ingest). Skip ``completed`` so the historical + // backlog doesn't pop toasts for already-finished sources. + if (e.type === 'source.ingest.completed') return; + const fileName = + (typeof payload.filename === 'string' && payload.filename) || + (typeof payload.job_name === 'string' && payload.job_name) || + scopeId; + const previouslyDismissed = isDismissed( + state.dismissedSourceIds, + scopeId, + ); + task = { + id: scopeId, + fileName, + progress: 0, + status: 'training', + sourceId: scopeId, + dismissed: previouslyDismissed, + }; + state.tasks.push(task); + task = state.tasks[state.tasks.length - 1]; + } + const wasTerminal = + task.status === 'completed' || task.status === 'failed'; + const sourceWasDismissed = + !!task.sourceId && isDismissed(state.dismissedSourceIds, task.sourceId); + + switch (e.type) { + case 'source.ingest.queued': + // Don't regress a task already past 'training' (e.g. the + // queued event arrives after the upload XHR finished and + // status flipped to 'training'). Idempotent on retries. + if (task.status === 'preparing' || task.status === 'uploading') { + task.status = 'training'; + task.progress = 0; + } + break; + case 'source.ingest.progress': { + const current = Number(payload.current); + if (!Number.isFinite(current)) break; + // Clamp + monotonic — never regress an already-higher value. + const clamped = Math.max(0, Math.min(100, current)); + if (task.status === 'completed' || task.status === 'failed') break; + task.status = 'training'; + if (clamped > task.progress) task.progress = clamped; + break; + } + case 'source.ingest.completed': + if (payload.limited === true) { + // Token-cap reached during ingest — surface as a failure + // so the toast shows the translated limit message rather + // than a misleading success state. + task.status = 'failed'; + task.progress = 100; + task.tokenLimitReached = true; + task.errorMessage = undefined; + // Only un-dismiss on the initial terminal transition; + // duplicate envelopes (StrictMode remount, reconnect + // overlap) must not re-pop a user-dismissed toast. + if (!wasTerminal && !sourceWasDismissed) task.dismissed = false; + } else { + task.status = 'completed'; + task.progress = 100; + task.errorMessage = undefined; + task.tokenLimitReached = false; + if (!wasTerminal && !sourceWasDismissed) task.dismissed = false; + } + break; + case 'source.ingest.failed': + task.status = 'failed'; + task.errorMessage = + typeof payload.error === 'string' + ? payload.error + : 'Ingestion failed.'; + if (!wasTerminal && !sourceWasDismissed) task.dismissed = false; + break; + default: + break; + } + }); + }, }); export const { diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index c2b499ce..766e6df8 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -12,7 +12,7 @@ "module": "ESNext", "moduleResolution": "Bundler", "resolveJsonModule": true, - "types": ["vite-plugin-svgr/client"], + "types": ["vite-plugin-svgr/client", "vitest/globals"], "isolatedModules": true, "noEmit": true, "jsx": "react-jsx", diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index f0b03175..31a2720c 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -1,3 +1,4 @@ +/// import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; import svgr from 'vite-plugin-svgr'; @@ -11,4 +12,10 @@ export default defineConfig({ '@': path.resolve(__dirname, './src'), }, }, + test: { + environment: 'happy-dom', + globals: true, + include: ['src/**/*.test.{ts,tsx}'], + setupFiles: ['./vitest.setup.ts'], + }, }); diff --git a/frontend/vitest.setup.ts b/frontend/vitest.setup.ts new file mode 100644 index 00000000..b43b9b89 --- /dev/null +++ b/frontend/vitest.setup.ts @@ -0,0 +1,24 @@ +// happy-dom's localStorage lives on `window`; slices read the bare +// global at module-load. Install a Map-backed shim on globalThis. +const store = new Map(); +const shim = { + getItem: (k: string) => (store.has(k) ? store.get(k)! : null), + setItem: (k: string, v: string) => { + store.set(k, String(v)); + }, + removeItem: (k: string) => { + store.delete(k); + }, + clear: () => store.clear(), + key: (i: number) => Array.from(store.keys())[i] ?? null, + get length() { + return store.size; + }, +}; +// Force-override: some happy-dom versions expose a Storage stub +// without getItem, so simple assignment isn't enough. +Object.defineProperty(globalThis, 'localStorage', { + value: shim, + writable: true, + configurable: true, +}); diff --git a/tests/agents/test_mcp_tool.py b/tests/agents/test_mcp_tool.py index 028c2fa3..933305ad 100644 --- a/tests/agents/test_mcp_tool.py +++ b/tests/agents/test_mcp_tool.py @@ -492,7 +492,7 @@ class TestMCPOAuthManager: from application.agents.tools.mcp_tool import MCPOAuthManager manager = MCPOAuthManager(MagicMock()) - result = manager.get_oauth_status("") + result = manager.get_oauth_status("", "alice") assert result["status"] == "not_started" diff --git a/tests/agents/tools/test_mcp_tool.py b/tests/agents/tools/test_mcp_tool.py index 51f84a09..35690bcc 100644 --- a/tests/agents/tools/test_mcp_tool.py +++ b/tests/agents/tools/test_mcp_tool.py @@ -8,6 +8,7 @@ MCPOAuthManager. import asyncio import concurrent.futures +import json from unittest.mock import MagicMock, patch import pytest @@ -869,17 +870,110 @@ class TestMCPOAuthManager: from application.agents.tools.mcp_tool import MCPOAuthManager manager = MCPOAuthManager(MagicMock()) - result = manager.get_oauth_status("") + result = manager.get_oauth_status("", "alice") assert result["status"] == "not_started" - def test_get_oauth_status_with_task(self): + def test_get_oauth_status_no_user(self): from application.agents.tools.mcp_tool import MCPOAuthManager - with patch("application.agents.tools.mcp_tool.mcp_oauth_status_task", - return_value={"status": "complete"}): - manager = MCPOAuthManager(MagicMock()) - result = manager.get_oauth_status("task123") - assert result["status"] == "complete" + manager = MCPOAuthManager(MagicMock()) + result = manager.get_oauth_status("task123", "") + # Without a user id we can't address the per-user SSE journal, + # so the manager refuses rather than scanning every user's + # stream. + assert result["status"] == "not_found" + + def test_get_oauth_status_reads_completed_envelope_from_journal(self): + """The manager walks the user's SSE Streams journal and + surfaces the latest ``mcp.oauth.*`` envelope for the task. + + Verifies the full polling-contract surface: ``status`` is + derived from the event-type suffix, and the completed + payload's ``tools`` / ``tools_count`` fields are passed + through unchanged so ``mcp.py``'s ``connect_mcp`` can use them + without further plumbing. + """ + from application.agents.tools.mcp_tool import MCPOAuthManager + + completed_envelope = json.dumps( + { + "type": "mcp.oauth.completed", + "scope": {"kind": "mcp_oauth", "id": "task123"}, + "payload": { + "task_id": "task123", + "tools": [{"name": "t1", "description": "d"}], + "tools_count": 1, + }, + } + ).encode("utf-8") + # xrevrange yields newest-first; an unrelated older entry + # should be skipped on the way to the matching one. + unrelated_envelope = json.dumps( + { + "type": "source.ingest.completed", + "scope": {"kind": "source", "id": "abc"}, + "payload": {}, + } + ).encode("utf-8") + mock_redis = MagicMock() + mock_redis.xrevrange.return_value = [ + (b"1735682400000-0", {b"event": completed_envelope}), + (b"1735682300000-0", {b"event": unrelated_envelope}), + ] + + manager = MCPOAuthManager(mock_redis) + result = manager.get_oauth_status("task123", "alice") + + assert result["status"] == "completed" + assert result["tools"] == [{"name": "t1", "description": "d"}] + assert result["tools_count"] == 1 + # The stream key is scoped per-user — never global. + mock_redis.xrevrange.assert_called_once() + call_args = mock_redis.xrevrange.call_args + assert "user:alice:stream" in call_args.args + # Scan window must cover the full bounded stream so a flood of + # concurrent source-ingest events between popup-completed and + # Save can't push the OAuth envelope out of view. + from application.core.settings import settings + + assert call_args.kwargs.get("count") >= settings.EVENTS_STREAM_MAXLEN + + def test_get_oauth_status_scan_covers_events_stream_maxlen(self): + """Regression: count must scale with ``EVENTS_STREAM_MAXLEN`` + so the OAuth completion envelope is reachable even after + concurrent source-ingest events flood the user stream.""" + from application.agents.tools.mcp_tool import MCPOAuthManager + from application.core.settings import settings + + mock_redis = MagicMock() + mock_redis.xrevrange.return_value = [] + + manager = MCPOAuthManager(mock_redis) + manager.get_oauth_status("task123", "alice") + + call_args = mock_redis.xrevrange.call_args + assert call_args.kwargs.get("count") >= settings.EVENTS_STREAM_MAXLEN + + def test_get_oauth_status_returns_not_found_when_no_match(self): + from application.agents.tools.mcp_tool import MCPOAuthManager + + mock_redis = MagicMock() + # Stream is non-empty but has nothing matching this task. + unrelated_envelope = json.dumps( + { + "type": "mcp.oauth.completed", + "scope": {"kind": "mcp_oauth", "id": "other-task"}, + "payload": {"task_id": "other-task"}, + } + ).encode("utf-8") + mock_redis.xrevrange.return_value = [ + (b"1735682400000-0", {b"event": unrelated_envelope}), + ] + + manager = MCPOAuthManager(mock_redis) + result = manager.get_oauth_status("task123", "alice") + + assert result["status"] == "not_found" # ===================================================================== @@ -1325,6 +1419,7 @@ class TestSetupClientExtended: tool._client = None tool.available_tools = [] tool.user_id = "user1" + tool.oauth_redirect_publish = None mock_client = MagicMock() with patch.object(MCPTool, "_create_transport", return_value=MagicMock()), \ @@ -2257,16 +2352,21 @@ class TestMCPOAuthManagerExtended: # Should store error in redis mock_redis.setex.assert_called() - def test_get_oauth_status_task_error(self): + def test_get_oauth_status_returns_not_found_on_redis_error(self): + """A failure inside ``xrevrange`` (Redis down, network blip) is + swallowed — the manager returns ``not_found`` so the caller + can present a clean "OAuth failed, try again" message rather + than a 500. + """ from application.agents.tools.mcp_tool import MCPOAuthManager - with patch( - "application.agents.tools.mcp_tool.mcp_oauth_status_task", - side_effect=Exception("task failed"), - ): - manager = MCPOAuthManager(MagicMock()) - with pytest.raises(Exception, match="task failed"): - manager.get_oauth_status("task123") + mock_redis = MagicMock() + mock_redis.xrevrange.side_effect = Exception("Redis went away") + + manager = MCPOAuthManager(mock_redis) + result = manager.get_oauth_status("task123", "alice") + + assert result["status"] == "not_found" # ===================================================================== diff --git a/tests/api/answer/services/test_conversation_service.py b/tests/api/answer/services/test_conversation_service.py index 136d0268..e016bcb5 100644 --- a/tests/api/answer/services/test_conversation_service.py +++ b/tests/api/answer/services/test_conversation_service.py @@ -449,6 +449,9 @@ class TestFinalizeMessage: question="q", decoded_token={"sub": user}, ) + from application.storage.db.repositories.conversations import ( + MessageUpdateOutcome, + ) assert svc.finalize_message( res["message_id"], "real answer", @@ -458,7 +461,7 @@ class TestFinalizeMessage: model_id="gpt-4", metadata={"foo": "bar"}, status="complete", - ) is True + ) is MessageUpdateOutcome.UPDATED msgs = ConversationsRepository(pg_conn).get_messages( res["conversation_id"], @@ -488,12 +491,15 @@ class TestFinalizeMessage: decoded_token={"sub": user}, ) err = RuntimeError("provider down") + from application.storage.db.repositories.conversations import ( + MessageUpdateOutcome, + ) assert svc.finalize_message( res["message_id"], "fallback text", status="failed", error=err, - ) is True + ) is MessageUpdateOutcome.UPDATED msgs = ConversationsRepository(pg_conn).get_messages( res["conversation_id"], @@ -526,9 +532,12 @@ class TestFinalizeMessage: ), {"cid": "c1", "mid": res["message_id"]}, ) + from application.storage.db.repositories.conversations import ( + MessageUpdateOutcome, + ) assert svc.finalize_message( res["message_id"], "ans", status="complete", - ) is True + ) is MessageUpdateOutcome.UPDATED status = pg_conn.execute( sql_text("SELECT status FROM tool_call_attempts WHERE call_id = :cid"), @@ -536,16 +545,19 @@ class TestFinalizeMessage: ).scalar() assert status == "confirmed" - def test_finalize_returns_false_for_unknown_message(self, pg_conn): + def test_finalize_returns_not_found_for_unknown_message(self, pg_conn): from application.api.answer.services.conversation_service import ( ConversationService, ) + from application.storage.db.repositories.conversations import ( + MessageUpdateOutcome, + ) with _patch_db(pg_conn): assert ConversationService().finalize_message( "00000000-0000-0000-0000-000000000000", "x", status="complete", - ) is False + ) is MessageUpdateOutcome.NOT_FOUND def test_finalize_rolls_back_tool_call_confirm_on_message_update_failure( self, pg_conn @@ -651,6 +663,9 @@ class TestFinalizeMessage: question="long question that becomes the fallback name", decoded_token={"sub": user}, ) + from application.storage.db.repositories.conversations import ( + MessageUpdateOutcome, + ) assert svc.finalize_message( res["message_id"], "answer", @@ -664,7 +679,7 @@ class TestFinalizeMessage: "long question that becomes the fallback name"[:50] ), }, - ) is True + ) is MessageUpdateOutcome.UPDATED repo = ConversationsRepository(pg_conn) conv = repo.get_any(res["conversation_id"], user) diff --git a/tests/api/answer/test_base_routes.py b/tests/api/answer/test_base_routes.py index 7fa2a52c..1c4a7f9a 100644 --- a/tests/api/answer/test_base_routes.py +++ b/tests/api/answer/test_base_routes.py @@ -343,6 +343,50 @@ class TestProcessResponseStreamExtended: result = resource.process_response_stream(iter(stream)) assert result["thought"] == "thinking..." + def test_handles_id_prefixed_chunks(self, mock_mongo_db, flask_app): + """``complete_stream`` emits ``id: \\n`` before each + ``data:`` line so reconnects can resume. The non-streaming + ``/api/answer`` consumer must skip the ``id:`` header (and the + informational ``message_id`` event) without breaking JSON + decoding. + """ + from application.api.answer.routes.base import BaseAnswerResource + + with flask_app.app_context(): + resource = BaseAnswerResource() + stream = [ + 'id: 0\n' + f'data: {json.dumps({"type": "message_id", "message_id": "abc"})}\n\n', + 'id: 1\n' + f'data: {json.dumps({"type": "answer", "answer": "Hello "})}\n\n', + 'id: 2\n' + f'data: {json.dumps({"type": "answer", "answer": "world"})}\n\n', + 'id: 3\n' + f'data: {json.dumps({"type": "id", "id": "conv-1"})}\n\n', + 'id: 4\n' + f'data: {json.dumps({"type": "end"})}\n\n', + ] + result = resource.process_response_stream(iter(stream)) + assert result["error"] is None + assert result["answer"] == "Hello world" + assert result["conversation_id"] == "conv-1" + + def test_skips_keepalive_comment_lines(self, mock_mongo_db, flask_app): + from application.api.answer.routes.base import BaseAnswerResource + + with flask_app.app_context(): + resource = BaseAnswerResource() + stream = [ + ': keepalive\n\n', + 'id: 0\n' + f'data: {json.dumps({"type": "answer", "answer": "ok"})}\n\n', + 'id: 1\n' + f'data: {json.dumps({"type": "end"})}\n\n', + ] + result = resource.process_response_stream(iter(stream)) + assert result["answer"] == "ok" + assert result["error"] is None + @pytest.mark.unit class TestCheckUsageStringBooleans: @@ -593,6 +637,251 @@ class TestCompleteStreamGeneratorExit: next(gen) gen.close() # Should not crash even with save error + def test_generator_exit_before_any_response_journals_error_not_end( + self, mock_mongo_db, flask_app, + ): + """A client disconnect right after the early ``message_id`` frame + leaves ``response_full`` empty, so finalize never runs. The abort + handler must journal ``error`` (not ``end``) and flip the row to + ``failed`` — otherwise a reconnecting client sees a terminal + ``end`` for a row whose DB status is still non-terminal and the + UI parks on a blank successful answer. + """ + from application.api.answer.routes.base import BaseAnswerResource + + with flask_app.app_context(): + resource = BaseAnswerResource() + mock_agent = MagicMock() + + # An agent that's primed but never yields anything before the + # client disconnects — keeps ``response_full`` empty. + def gen_never_yields(): + if False: + yield # pragma: no cover + return + + mock_agent.gen.return_value = gen_never_yields() + mock_agent.compression_metadata = None + mock_agent.compression_saved = False + mock_agent.tool_calls = [] + + resource.conversation_service = MagicMock() + resource.conversation_service.save_user_question.return_value = { + "conversation_id": "conv1", + "message_id": "msg1", + "request_id": "req1", + } + + journaled: list[tuple] = [] + + def _capture_record(message_id, sequence_no, event_type, payload): + journaled.append((message_id, sequence_no, event_type, payload)) + return True + + with patch( + "application.api.answer.routes.base.record_event", + side_effect=_capture_record, + ): + gen = resource.complete_stream( + question="Q", + agent=mock_agent, + conversation_id="conv1", + user_api_key=None, + decoded_token={"sub": "u"}, + should_save_conversation=True, + model_id="gpt-4", + ) + # Drain the early ``message_id`` event, then close before + # the agent yields anything. + next(gen) + gen.close() + + # The early message_id frame got journaled (seq 0); the abort + # handler must follow with an ``error`` event (NOT ``end``). + terminal_events = [ + (et, pl) for (_, _, et, pl) in journaled if et in ("end", "error") + ] + assert len(terminal_events) == 1, ( + f"expected exactly one terminal journal write, got {terminal_events}" + ) + assert terminal_events[0][0] == "error", ( + f"expected ``error`` terminal but got ``end``: {terminal_events}" + ) + payload = terminal_events[0][1] + assert payload.get("type") == "error" + assert payload.get("code") == "client_disconnect" + + # And the DB row should have been flipped to ``failed`` via + # finalize_message. The mocked service records the call. + finalize_calls = ( + resource.conversation_service.finalize_message.call_args_list + ) + assert len(finalize_calls) == 1 + assert finalize_calls[0].kwargs.get("status") == "failed" + + def test_generator_exit_after_response_still_journals_end( + self, mock_mongo_db, flask_app, + ): + """Regression guard: a disconnect AFTER partial response was + produced and ``finalize_message`` succeeded must still journal + ``end`` (the row matches ``complete``). Only the empty-response + branch flips to ``error``. + """ + from application.api.answer.routes.base import BaseAnswerResource + from application.storage.db.repositories.conversations import ( + MessageUpdateOutcome, + ) + + with flask_app.app_context(): + resource = BaseAnswerResource() + mock_agent = MagicMock() + + def gen_answers(): + yield {"answer": "partial"} + + mock_agent.gen.return_value = gen_answers() + mock_agent.compression_metadata = None + mock_agent.compression_saved = False + mock_agent.tool_calls = [] + + resource.conversation_service = MagicMock() + resource.conversation_service.finalize_message.return_value = ( + MessageUpdateOutcome.UPDATED + ) + resource.conversation_service.save_user_question.return_value = { + "conversation_id": "conv1", + "message_id": "msg1", + "request_id": "req1", + } + + journaled: list[tuple] = [] + + def _capture_record(message_id, sequence_no, event_type, payload): + journaled.append((message_id, sequence_no, event_type, payload)) + return True + + with patch( + "application.api.answer.routes.base.record_event", + side_effect=_capture_record, + ): + gen = resource.complete_stream( + question="Q", + agent=mock_agent, + conversation_id="conv1", + user_api_key=None, + decoded_token={"sub": "u"}, + should_save_conversation=True, + model_id="gpt-4", + ) + next(gen) # message_id frame + next(gen) # answer frame (consumes ``partial``) + gen.close() + + terminal_events = [ + (et, pl) for (_, _, et, pl) in journaled if et in ("end", "error") + ] + assert terminal_events and terminal_events[0][0] == "end", ( + f"finalize succeeded but abort journaled {terminal_events}" + ) + + def test_generator_exit_after_normal_finalize_already_complete_journals_end( + self, mock_mongo_db, flask_app, + ): + """Regression for the race where the normal-path finalize wins + against a client disconnect. + + Trace: agent finishes, ``complete_stream`` runs the normal-path + ``finalize_message`` at base.py:632 and flips the row to + ``complete``. The client TCP-resets before the ``end`` frame can + be journaled. The GeneratorExit handler calls ``finalize_message`` + again — and the repository, gated by ``only_if_non_terminal``, + reports ``ALREADY_COMPLETE`` because the row is already at the + target state. The abort handler must journal ``end``, not + ``error``: the DB says ``complete``, the reconnecting client + must see the same. + + Without the fix the abort handler treated ``ALREADY_COMPLETE`` + as a failure and journaled ``error`` — a reconnect would then + replay a successful completion as a failed answer. + """ + from application.api.answer.routes.base import BaseAnswerResource + from application.storage.db.repositories.conversations import ( + MessageUpdateOutcome, + ) + + with flask_app.app_context(): + resource = BaseAnswerResource() + mock_agent = MagicMock() + + def gen_answers(): + yield {"answer": "partial"} + + mock_agent.gen.return_value = gen_answers() + mock_agent.compression_metadata = None + mock_agent.compression_saved = False + mock_agent.tool_calls = [] + + resource.conversation_service = MagicMock() + # Normal-path call returns UPDATED; abort-handler call sees + # the row already terminal and returns ALREADY_COMPLETE. + resource.conversation_service.finalize_message.side_effect = [ + MessageUpdateOutcome.UPDATED, + MessageUpdateOutcome.ALREADY_COMPLETE, + ] + resource.conversation_service.save_user_question.return_value = { + "conversation_id": "conv1", + "message_id": "msg1", + "request_id": "req1", + } + + journaled: list[tuple] = [] + + def _capture_record(message_id, sequence_no, event_type, payload): + journaled.append((message_id, sequence_no, event_type, payload)) + return True + + with patch( + "application.api.answer.routes.base.record_event", + side_effect=_capture_record, + ): + gen = resource.complete_stream( + question="Q", + agent=mock_agent, + conversation_id="conv1", + user_api_key=None, + decoded_token={"sub": "u"}, + should_save_conversation=True, + model_id="gpt-4", + ) + next(gen) # message_id frame + next(gen) # answer frame + # Pull the ``id`` frame so the normal-path finalize at + # line 632 actually runs (it sits between the answer + # frame yield and the id frame yield). + next(gen) # id frame — runs normal-path finalize first + gen.close() # GeneratorExit at the id frame yield + + terminal_events = [ + (et, pl) for (_, _, et, pl) in journaled if et in ("end", "error") + ] + assert len(terminal_events) == 1, ( + f"expected exactly one terminal journal write, got " + f"{terminal_events}" + ) + assert terminal_events[0][0] == "end", ( + f"row was already ``complete`` but abort journaled " + f"{terminal_events[0]} — reconnect would surface a " + f"successful answer as failed" + ) + + # Both finalize_message calls were made: the normal-path + # one and the abort-handler one. Asserting on side_effect + # consumption ensures the test really exercised both + # branches. + assert ( + resource.conversation_service.finalize_message.call_count == 2 + ) + @contextmanager def _patch_db_session(conn): @@ -606,10 +895,34 @@ def _patch_db_session(conn): ), patch( "application.api.answer.services.conversation_service.db_readonly", _yield, + ), patch( + # ``record_event`` opens its own short-lived ``db_session`` for + # cross-connection visibility. In tests we route it back to the + # same ``pg_conn`` so the journal write can see the message row + # the conversation_service just wrote in this transaction. + "application.streaming.message_journal.db_session", + _yield, + ), patch( + # ``complete_stream`` reads ``latest_sequence_no`` via + # ``db_readonly`` to seed continuation runs. Same patch reason + # as the journal — keep the read on the same pg_conn so it sees + # uncommitted writes from this transaction. + "application.api.answer.routes.base.db_readonly", + _yield, ): yield +def _extract_sse_data(chunk: str) -> str: + """Pull the ``data:`` payload from an SSE record, ignoring any + ``id:`` header introduced by the journal wiring. + """ + for line in chunk.split("\n"): + if line.startswith("data:"): + return line[len("data:") :].lstrip() + return "" + + @pytest.mark.unit class TestCompleteStreamWalAcceptance: """Acceptance for the WAL pre-persist behaviour: when the LLM raises @@ -659,6 +972,158 @@ class TestCompleteStreamWalAcceptance: assert "RuntimeError" in msgs[0]["metadata"]["error"] assert "LLM upstream failed" in msgs[0]["metadata"]["error"] + def test_tool_approval_event_only_fires_when_state_saved( + self, pg_conn, flask_app, + ): + """A `tool.approval.required` notification with no resumable + ``pending_tool_state`` row would deep-link the user to a 404. + Gate the publish on save_state actually committing. + """ + from application.api.answer.routes.base import BaseAnswerResource + + with flask_app.app_context(): + resource = BaseAnswerResource() + mock_agent = MagicMock() + # Drive the agent into the paused branch with a single + # ``tool_calls_pending`` event. + mock_agent.gen.return_value = iter( + [ + { + "type": "tool_calls_pending", + "data": {"pending_tool_calls": [{"call_id": "c1"}]}, + } + ] + ) + mock_agent._pending_continuation = { + "messages": [], + "tools_dict": {}, + "pending_tool_calls": [{"call_id": "c1"}], + } + mock_agent.tool_calls = [] + mock_agent.compression_metadata = None + mock_agent.compression_saved = False + + published: list = [] + + def _capture(*args, **kwargs): + published.append(args) + + with _patch_db_session(pg_conn), patch( + "application.api.answer.routes.base.publish_user_event", + side_effect=_capture, + ), patch( + "application.api.answer.services.continuation_service." + "ContinuationService.save_state", + side_effect=RuntimeError("PG outage"), + ): + list( + resource.complete_stream( + question="run my tool", + agent=mock_agent, + conversation_id=None, + user_api_key=None, + decoded_token={"sub": "u-tap"}, + should_save_conversation=True, + model_id="gpt-4", + ) + ) + + # No tool.approval.required publish when save_state failed. + event_types = [a[1] for a in published if len(a) >= 2] + assert "tool.approval.required" not in event_types + + def test_continuation_seeds_sequence_no_from_journal_high_water_mark( + self, pg_conn, flask_app, + ): + """A resumed (tool-actions) stream must continue numbering past + the original run's max ``sequence_no``. Otherwise the second + invocation collides on the duplicate-PK and silently drops + every journal write past the resume point. + + We pre-seed the journal with synthetic rows simulating an + original run, then invoke ``complete_stream`` with + ``_continuation`` set and assert seq numbering picks up past + the high-water mark. + """ + import uuid as _uuid + + from sqlalchemy import text as sql_text + + from application.api.answer.routes.base import BaseAnswerResource + from application.storage.db.repositories.message_events import ( + MessageEventsRepository, + ) + + # Pre-seed: insert a parent conversation + message + a few + # journal rows so latest_sequence_no returns 7. + user_id = "u-resume" + conv_id = _uuid.uuid4() + message_id = _uuid.uuid4() + pg_conn.execute( + sql_text("INSERT INTO users (user_id) VALUES (:u)"), + {"u": user_id}, + ) + pg_conn.execute( + sql_text( + "INSERT INTO conversations (id, user_id, name) " + "VALUES (:id, :u, 'pre-seed')" + ), + {"id": conv_id, "u": user_id}, + ) + pg_conn.execute( + sql_text( + "INSERT INTO conversation_messages " + "(id, conversation_id, user_id, position, prompt) " + "VALUES (:id, :c, :u, 0, 'q')" + ), + {"id": message_id, "c": conv_id, "u": user_id}, + ) + repo = MessageEventsRepository(pg_conn) + for seq in range(8): # rows 0..7 + repo.record(str(message_id), seq, "answer", {"chunk": str(seq)}) + original_max = repo.latest_sequence_no(str(message_id)) + assert original_max == 7 + + with flask_app.app_context(): + resource = BaseAnswerResource() + cont_agent = MagicMock() + cont_agent.gen_continuation.return_value = iter( + [{"answer": "d"}, {"answer": "e"}] + ) + cont_agent.tool_calls = [] + cont_agent.compression_metadata = None + cont_agent.compression_saved = False + cont_agent.tool_executor = None + + with _patch_db_session(pg_conn): + list( + resource.complete_stream( + question="", + agent=cont_agent, + conversation_id=str(conv_id), + user_api_key=None, + decoded_token={"sub": user_id}, + should_save_conversation=True, + model_id="gpt-4", + _continuation={ + "messages": [], + "tools_dict": {}, + "pending_tool_calls": [], + "tool_actions": [], + "reserved_message_id": str(message_id), + "request_id": "req-resume", + }, + ) + ) + + new_max = MessageEventsRepository(pg_conn).latest_sequence_no( + str(message_id) + ) + # Continuation extended the journal — the new high-water mark + # is strictly greater than the seeded ``original_max=7``, + # confirming the allocator picked up past the resume point. + assert new_max is not None and new_max > original_max + def test_request_id_consistent_across_sse_event_and_wal_row( self, pg_conn, flask_app, ): @@ -694,9 +1159,9 @@ class TestCompleteStreamWalAcceptance: ) sse_events = [ - json.loads(s.replace("data: ", "").strip()) + json.loads(_extract_sse_data(s)) for s in stream - if s.startswith("data: ") + if "data:" in s ] early_events = [e for e in sse_events if e.get("type") == "message_id"] assert len(early_events) == 1 @@ -712,3 +1177,102 @@ class TestCompleteStreamWalAcceptance: msgs = ConversationsRepository(pg_conn).get_messages(str(convs[0][0])) assert len(msgs) == 1 assert msgs[0]["request_id"] == sse_request_id + + +@pytest.mark.unit +class TestStreamingHeartbeatSeed: + """Regression guard: when the row flips to ``streaming`` we must seed + ``last_heartbeat_at`` so the watchdog doesn't fall back to + ``timestamp`` (creation time) on slow LLM cold-starts (>idle_secs). + """ + + def test_heartbeat_seeded_on_first_chunk(self, mock_mongo_db, flask_app): + from application.api.answer.routes.base import BaseAnswerResource + + with flask_app.app_context(): + resource = BaseAnswerResource() + mock_agent = MagicMock() + mock_agent.gen.return_value = iter( + [{"answer": "a"}, {"answer": "b"}] + ) + mock_agent.compression_metadata = None + mock_agent.compression_saved = False + mock_agent.tool_calls = [] + mock_agent.tool_executor = None + + resource.conversation_service = MagicMock() + resource.conversation_service.save_conversation.return_value = "conv1" + resource.conversation_service.save_user_question.return_value = { + "conversation_id": "conv1", + "message_id": "msg1", + "request_id": "req1", + } + + list( + resource.complete_stream( + question="Q", + agent=mock_agent, + conversation_id="conv1", + user_api_key=None, + decoded_token={"sub": "u"}, + should_save_conversation=True, + model_id="gpt-4", + ) + ) + + # update_message_status flips the row to ``streaming`` exactly + # once (idempotent via streaming_marked). + status_calls = [ + c for c in resource.conversation_service + .update_message_status.call_args_list + if c.args[1] == "streaming" + ] + assert len(status_calls) == 1 + assert status_calls[0].args[0] == "msg1" + + # heartbeat seed runs exactly once at the same flip — multiple + # chunks don't re-stamp inside this window (the throttled + # _heartbeat_streaming path is gated by STREAM_HEARTBEAT_INTERVAL). + hb_calls = ( + resource.conversation_service.heartbeat_message.call_args_list + ) + assert len(hb_calls) == 1 + assert hb_calls[0].args[0] == "msg1" + + def test_heartbeat_seed_skipped_without_reserved_message_id( + self, mock_mongo_db, flask_app, + ): + """No DB-backed message row → no heartbeat call (and no error).""" + from application.api.answer.routes.base import BaseAnswerResource + + with flask_app.app_context(): + resource = BaseAnswerResource() + mock_agent = MagicMock() + mock_agent.gen.return_value = iter([{"answer": "a"}]) + mock_agent.compression_metadata = None + mock_agent.compression_saved = False + mock_agent.tool_calls = [] + mock_agent.tool_executor = None + + resource.conversation_service = MagicMock() + resource.conversation_service.save_conversation.return_value = "conv1" + # save_user_question returns no message_id → reservation absent + resource.conversation_service.save_user_question.return_value = { + "conversation_id": "conv1", + "message_id": None, + "request_id": "req1", + } + + list( + resource.complete_stream( + question="Q", + agent=mock_agent, + conversation_id="conv1", + user_api_key=None, + decoded_token={"sub": "u"}, + should_save_conversation=True, + model_id="gpt-4", + ) + ) + + resource.conversation_service.heartbeat_message.assert_not_called() diff --git a/tests/api/answer/test_snapshot_tail_integration.py b/tests/api/answer/test_snapshot_tail_integration.py new file mode 100644 index 00000000..4d6005df --- /dev/null +++ b/tests/api/answer/test_snapshot_tail_integration.py @@ -0,0 +1,233 @@ +"""Integration tests for the end-to-end snapshot+tail handoff. + +Exercises the publisher → journal → reconnect endpoint round-trip +without mocking the journal layer, so a regression in any of: +- complete_stream's _emit closure +- record_event's commit-per-call contract +- build_message_event_stream's snapshot-from-DB path +- the reconnect route's auth + ownership gates +- message_events repo SQL + +would surface here as a failed integration assertion. +""" + +from __future__ import annotations + +import uuid as _uuid +from contextlib import contextmanager +from unittest.mock import patch + +import pytest +from sqlalchemy import text as sql_text + + +@contextmanager +def _patch_journal_session(conn): + @contextmanager + def _yield(): + yield conn + + with patch( + "application.streaming.message_journal.db_session", _yield + ), patch( + "application.streaming.event_replay.db_readonly", _yield + ): + yield + + +def _seed_message(conn, user_id: str | None = None): + user_id = user_id or f"u-{_uuid.uuid4().hex[:6]}" + conv_id = _uuid.uuid4() + msg_id = _uuid.uuid4() + conn.execute(sql_text("INSERT INTO users (user_id) VALUES (:u)"), {"u": user_id}) + conn.execute( + sql_text( + "INSERT INTO conversations (id, user_id, name) VALUES (:id, :u, 't')" + ), + {"id": conv_id, "u": user_id}, + ) + conn.execute( + sql_text( + "INSERT INTO conversation_messages (id, conversation_id, user_id, position) " + "VALUES (:id, :c, :u, 0)" + ), + {"id": msg_id, "c": conv_id, "u": user_id}, + ) + return user_id, str(msg_id) + + +@pytest.mark.integration +class TestSnapshotPlusTailRoundTrip: + def test_record_event_then_snapshot_returns_what_was_journaled( + self, pg_conn, + ): + """End-to-end of the journal half: ``record_event`` writes through + a real ``MessageEventsRepository``, ``build_message_event_stream`` + reads the snapshot back via the same repo + a stub Topic. + """ + from application.streaming import event_replay + from application.streaming.message_journal import record_event + + _, message_id = _seed_message(pg_conn) + + with _patch_journal_session(pg_conn): + # Three events stamp the journal. + record_event(message_id, 0, "answer", {"type": "answer", "answer": "A"}) + record_event(message_id, 1, "answer", {"type": "answer", "answer": "B"}) + record_event(message_id, 2, "end", {"type": "end"}) + + # Reconnect path: subscribe yields nothing (Redis-down + # branch); the post-loop fallback runs the snapshot read + # synchronously and yields the journal contents. + def _empty_subscribe(self, on_subscribe=None, poll_timeout=1.0): + return + yield # pragma: no cover + + with patch.object( + event_replay.Topic, + "subscribe", + _empty_subscribe, + create=False, + ): + gen = event_replay.build_message_event_stream( + message_id, + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = list(gen) + + # Prelude + 3 snapshot frames. + assert out[0] == ": connected\n\n" + assert "id: 0" in out[1] and '"answer": "A"' in out[1] + assert "id: 1" in out[2] and '"answer": "B"' in out[2] + assert "id: 2" in out[3] and '"type": "end"' in out[3] + + def test_snapshot_resumes_past_last_event_id(self, pg_conn): + from application.streaming import event_replay + from application.streaming.message_journal import record_event + + _, message_id = _seed_message(pg_conn) + + with _patch_journal_session(pg_conn): + for seq in range(5): + record_event( + message_id, seq, "answer", {"type": "answer", "answer": str(seq)} + ) + + def _empty_subscribe(self, on_subscribe=None, poll_timeout=1.0): + return + yield # pragma: no cover + + with patch.object( + event_replay.Topic, + "subscribe", + _empty_subscribe, + create=False, + ): + # Client says it has seen up through seq=2; expect 3 + 4. + out = list( + event_replay.build_message_event_stream( + message_id, + last_event_id=2, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + ) + + ids_seen = [line for line in out if line.startswith("id: ")] + # Multi-line records: extract the id integers we delivered. + emitted = sorted( + int(line.split(": ", 1)[1].split("\n")[0]) + for line in out + if line.startswith("id: ") + ) + # Filter for non-negative (the snapshot-failure synthetic uses -1). + emitted = [e for e in emitted if e >= 0] + assert emitted == [3, 4] + assert ids_seen # sanity + + def test_reconnect_route_round_trip(self, pg_conn, flask_app): + """``/api/messages//events`` returns the journaled events + for an authenticated owner. + """ + from flask import Flask, request + + from application.api.answer.routes.messages import messages_bp + from application.streaming.message_journal import record_event + + # Build a fresh Flask app routing to the reconnect blueprint + # plus a tiny auth shim that injects the test user. + user_id, message_id = _seed_message(pg_conn) + app = Flask(__name__) + app.register_blueprint(messages_bp) + app.config["TESTING"] = True + + @app.before_request + def _shim_auth(): + request.decoded_token = {"sub": user_id} + + with _patch_journal_session(pg_conn): + record_event(message_id, 0, "answer", {"type": "answer", "answer": "x"}) + record_event(message_id, 1, "end", {"type": "end"}) + + from application.streaming import event_replay + + def _empty_subscribe(self, on_subscribe=None, poll_timeout=1.0): + return + yield # pragma: no cover + + with patch.object( + event_replay.Topic, + "subscribe", + _empty_subscribe, + create=False, + ), patch( + "application.api.answer.routes.messages.db_readonly" + ) as ro: + ro.return_value.__enter__.return_value = pg_conn + + with app.test_client() as c: + r = c.get(f"/api/messages/{message_id}/events") + assert r.status_code == 200 + body = b"" + for chunk in r.iter_encoded(): + body += chunk + if body.count(b"\n\n") >= 4: + break + r.close() + # Both journaled events present in the response. + text = body.decode("utf-8") + assert ": connected" in text + assert '"answer": "x"' in text + assert '"type": "end"' in text + # The seq lines are correct. + assert "id: 0" in text and "id: 1" in text + + def test_reconnect_rejects_non_owner(self, pg_conn, flask_app): + from flask import Flask, request + + from application.api.answer.routes.messages import messages_bp + + user_id, message_id = _seed_message(pg_conn) + app = Flask(__name__) + app.register_blueprint(messages_bp) + + @app.before_request + def _shim_auth(): + request.decoded_token = {"sub": "different-user"} + + # Make the ownership check use the test connection. + with patch( + "application.api.answer.routes.messages.db_readonly" + ) as ro: + from contextlib import contextmanager as _cm + + @_cm + def _yield(): + yield pg_conn + + ro.side_effect = lambda: _yield() + with app.test_client() as c: + r = c.get(f"/api/messages/{message_id}/events") + assert r.status_code == 404 diff --git a/tests/api/test_events_routes.py b/tests/api/test_events_routes.py new file mode 100644 index 00000000..629acd39 --- /dev/null +++ b/tests/api/test_events_routes.py @@ -0,0 +1,649 @@ +"""Tests for application/api/events/routes.py — the SSE endpoint. + +The SSE generator runs in a separate thread under the WSGI test client; +we drive it with mocked Redis (the ``pubsub.get_message`` and ``xrange`` +sequences) and read the response body until we have enough records to +assert on, then close the response to terminate the generator. +""" + +from __future__ import annotations + +import json +import threading +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from flask import Flask, request + + +def _make_app(): + """Mount the events blueprint on a bare Flask app + JWT shim. + + The shim mimics ``application/app.py`` populating + ``request.decoded_token`` so the SSE handler's auth gate sees a + user-id without requiring the full app stack. + """ + from application.api.events.routes import events + + app = Flask(__name__) + app.register_blueprint(events) + app.config["TESTING"] = True + + @app.before_request + def _shim_auth(): # noqa: D401 + header = request.headers.get("X-Test-Sub") + request.decoded_token = {"sub": header} if header else None + + return app + + +class _FakePubSub: + """Minimal Redis pub/sub stand-in for the SSE handler. + + ``messages`` is a list of message dicts the generator should see in + order. After exhausting it, ``get_message`` returns ``None`` (poll + timeout) so the generator stays alive emitting keepalives until the + test closes the response. + """ + + def __init__(self, messages: list[dict[str, Any]]): + self._messages = list(messages) + self.subscribed: list[str] = [] + self.unsubscribed: list[str] = [] + self.closed = False + self._lock = threading.Lock() + + def subscribe(self, name: str): + self.subscribed.append(name) + + def unsubscribe(self, name: str): + self.unsubscribed.append(name) + + def close(self): + self.closed = True + + def get_message(self, timeout: float = 0): + with self._lock: + if self._messages: + return self._messages.pop(0) + return None + + +def _drain_until(response, predicate, max_chunks: int = 200) -> bytes: + """Consume the streamed response until ``predicate(buf)`` is true. + + Returns the accumulated bytes. Closes the response so the generator + exits cleanly via GeneratorExit. + """ + buf = b"" + iterator = response.iter_encoded() + for _ in range(max_chunks): + try: + chunk = next(iterator) + except StopIteration: + break + if not chunk: + continue + buf += chunk + if predicate(buf): + break + response.close() + return buf + + +# ── auth gate ─────────────────────────────────────────────────────────── + + +class TestAuthGate: + def test_rejects_when_no_decoded_token(self): + app = _make_app() + with app.test_client() as c: + r = c.get("/api/events") + assert r.status_code == 401 + + def test_rejects_when_decoded_token_missing_sub(self): + from application.api.events import routes as events_module + + app = _make_app() + + # Clear the shim's behavior — supply a decoded_token without sub. + @app.before_request + def _override(): + request.decoded_token = {"email": "x@y.z"} + + with patch.object(events_module, "get_redis_instance", return_value=None): + with app.test_client() as c: + r = c.get("/api/events") + assert r.status_code == 401 + + +# ── streaming response shape ──────────────────────────────────────────── + + +class TestStreamShape: + def test_returns_event_stream_mimetype_and_no_buffering_header(self): + from application.api.events import routes as events_module + + app = _make_app() + with patch.object(events_module, "get_redis_instance", return_value=None): + with app.test_client() as c: + r = c.get("/api/events", headers={"X-Test-Sub": "alice"}) + assert r.status_code == 200 + assert r.mimetype == "text/event-stream" + assert r.headers.get("Cache-Control") == "no-store" + assert r.headers.get("X-Accel-Buffering") == "no" + # Drain enough to see the prelude comment then close. + body = _drain_until(r, lambda b: b": connected" in b) + assert b": connected" in body + + def test_emits_push_disabled_when_setting_off(self): + from application.api.events import routes as events_module + + app = _make_app() + with patch.object(events_module, "get_redis_instance", return_value=None), \ + patch.object(events_module.settings, "ENABLE_SSE_PUSH", False): + with app.test_client() as c: + r = c.get("/api/events", headers={"X-Test-Sub": "alice"}) + body = _drain_until(r, lambda b: b": push_disabled" in b) + assert b": push_disabled" in body + assert b": connected" in body # prelude still emitted + + +# ── concurrency cap ───────────────────────────────────────────────────── + + +class TestConcurrencyCap: + def test_returns_429_when_user_over_cap(self): + from application.api.events import routes as events_module + + app = _make_app() + redis_client = MagicMock() + # First INCR returns 9 (over cap of 8). + redis_client.incr.return_value = 9 + + with patch.object(events_module, "get_redis_instance", return_value=redis_client), \ + patch.object(events_module.settings, "SSE_MAX_CONCURRENT_PER_USER", 8): + with app.test_client() as c: + r = c.get("/api/events", headers={"X-Test-Sub": "alice"}) + assert r.status_code == 429 + # DECR fired to release the over-cap increment. + redis_client.decr.assert_called_once_with("user:alice:sse_count") + + def test_skips_cap_when_zero_disabled(self): + from application.api.events import routes as events_module + + app = _make_app() + redis_client = MagicMock() + + with patch.object(events_module, "get_redis_instance", return_value=redis_client), \ + patch.object(events_module.settings, "SSE_MAX_CONCURRENT_PER_USER", 0), \ + patch.object(events_module, "Topic") as mock_topic_cls: + mock_topic = MagicMock() + mock_topic.subscribe.return_value = iter([]) + mock_topic_cls.return_value = mock_topic + redis_client.xinfo_stream.side_effect = Exception("no stream") + redis_client.xrange.return_value = [] + with app.test_client() as c: + r = c.get("/api/events", headers={"X-Test-Sub": "alice"}) + assert r.status_code == 200 + # Concurrency counter not touched when cap is 0. The + # replay-budget INCR is unrelated and may still fire. + incr_keys = [ + call.args[0] for call in redis_client.incr.call_args_list + ] + assert "user:alice:sse_count" not in incr_keys + _drain_until(r, lambda b: b": connected" in b) + + +# ── replay + live tail ────────────────────────────────────────────────── + + +class TestReplayAndTail: + def test_replay_yields_xrange_entries_with_injected_id(self): + from application.api.events import routes as events_module + + app = _make_app() + redis_client = MagicMock() + redis_client.incr.return_value = 1 + # Empty stream (no truncation). + redis_client.xinfo_stream.side_effect = Exception("nope") + # XRANGE returns one stored envelope (without ``id``); the route + # injects the entry id on the way out. + stored_event = json.dumps( + { + "type": "source.ingest.progress", + "ts": "2026-04-28T00:00:00.000Z", + "user_id": "alice", + "topic": "user:alice", + "scope": {"kind": "source", "id": "src-1"}, + "payload": {"current": 25, "total": 100}, + } + ).encode() + redis_client.xrange.return_value = [ + (b"1735682400000-0", {b"event": stored_event}), + ] + + # Topic.subscribe yields an immediate timeout so the generator + # keeps running long enough to flush replay; subsequent calls + # also return None. + from application.api.events.routes import _SSE_LINE_SPLIT # noqa: F401 + + # Fake the broadcast Topic to invoke on_subscribe immediately + # then yield None ticks until close. + def _fake_subscribe(self, on_subscribe=None, poll_timeout=1.0): + if on_subscribe is not None: + on_subscribe() + while True: + yield None + + with patch.object(events_module, "get_redis_instance", return_value=redis_client), \ + patch.object( + events_module.Topic, "subscribe", _fake_subscribe, create=False + ): + with app.test_client() as c: + r = c.get( + "/api/events", + headers={"X-Test-Sub": "alice", "Last-Event-ID": "1735682300000-0"}, + ) + body = _drain_until( + r, + lambda b: b'"current": 25' in b or b'"current":25' in b, + max_chunks=80, + ) + # Replay yields the entry id as the SSE id field. + assert b"id: 1735682400000-0" in body + # Envelope was rewritten to include the injected id. + assert b'"id": "1735682400000-0"' in body or b'"id":"1735682400000-0"' in body + # The connect log fires before replay. + assert b": connected" in body + + def test_snapshot_flushed_when_subscribe_dies_after_callback(self): + """Regression: if ``on_subscribe`` populated ``replay_lines`` but + ``Topic.subscribe`` exits before yielding once (transient Redis + hiccup between SUBSCRIBE-ack and the first poll), the snapshot + must still reach the client. Prior to the fix the in-loop flush + was the only flush, so the backlog was silently dropped. + """ + from application.api.events import routes as events_module + + app = _make_app() + redis_client = MagicMock() + redis_client.incr.return_value = 1 + redis_client.xinfo_stream.side_effect = Exception("nope") + stored_event = json.dumps( + { + "type": "notification", + "payload": {"text": "from snapshot"}, + } + ).encode() + redis_client.xrange.return_value = [ + (b"1735682400000-0", {b"event": stored_event}), + ] + + # Mimic the broadcast_channel race: SUBSCRIBE acks, on_subscribe + # runs, then the next get_message raises and the generator + # returns without ever yielding. + def _subscribe_dies_after_callback( + self, on_subscribe=None, poll_timeout=1.0 + ): + if on_subscribe is not None: + on_subscribe() + return + yield # pragma: no cover (make the function a generator) + + with patch.object(events_module, "get_redis_instance", return_value=redis_client), \ + patch.object( + events_module.Topic, + "subscribe", + _subscribe_dies_after_callback, + create=False, + ): + with app.test_client() as c: + r = c.get( + "/api/events", + headers={ + "X-Test-Sub": "alice", + "Last-Event-ID": "1735682300000-0", + }, + ) + body = _drain_until( + r, + lambda b: b"from snapshot" in b, + max_chunks=80, + ) + # Snapshot frame must have been flushed via the post-loop + # safety net even though Topic.subscribe exited before + # the in-loop flush could fire. + assert b"id: 1735682400000-0" in body + assert b"from snapshot" in body + # XRANGE was issued exactly once (no double-flush). + redis_client.xrange.assert_called_once() + + def test_invalid_last_event_id_emits_truncation_notice(self): + from application.api.events import routes as events_module + + app = _make_app() + redis_client = MagicMock() + redis_client.incr.return_value = 1 + redis_client.xinfo_stream.return_value = {"first-entry": [b"1-0", []]} + redis_client.xrange.return_value = [] + + def _fake_subscribe(self, on_subscribe=None, poll_timeout=1.0): + if on_subscribe is not None: + on_subscribe() + while True: + yield None + + with patch.object(events_module, "get_redis_instance", return_value=redis_client), \ + patch.object(events_module.Topic, "subscribe", _fake_subscribe, create=False): + with app.test_client() as c: + r = c.get( + "/api/events", + headers={"X-Test-Sub": "alice", "Last-Event-ID": "definitely-not-an-id"}, + ) + body = _drain_until( + r, lambda b: b"backlog.truncated" in b, max_chunks=80 + ) + assert b"backlog.truncated" in body + + def test_live_tail_rejects_malformed_event_id_for_dedupe(self): + """A pub/sub envelope carrying a non-Redis-Streams ``id`` must not + seed the dedup floor. Otherwise an adversarial or buggy publisher + could ship ``id="9999999999999-9"`` (lex-greater than any real + id) and pin every subsequent legitimate event below the floor, + silently dropping the user's notifications. + + The event itself should still be delivered to the client — we + just refuse to use the bogus id for ordering, so it ships + without an SSE ``id:`` header and ``max_replayed_id`` stays put. + """ + from application.api.events import routes as events_module + + app = _make_app() + redis_client = MagicMock() + redis_client.incr.return_value = 1 + redis_client.xinfo_stream.side_effect = Exception("nope") + # Snapshot covers ids up to 1735682400000-0; max_replayed_id + # becomes that value after the in-loop flush. + replay_event = json.dumps({ + "type": "source.ingest.progress", + "payload": {"step": "replay"}, + }).encode() + redis_client.xrange.return_value = [ + (b"1735682400000-0", {b"event": replay_event}), + ] + + live_bogus = json.dumps({ + "id": "definitely-not-an-id", + "type": "source.ingest.completed", + "payload": {"step": "live-bogus"}, + }) + live_real = json.dumps({ + "id": "1735682500000-0", + "type": "source.ingest.completed", + "payload": {"step": "live-real"}, + }) + + def _fake_subscribe(self, on_subscribe=None, poll_timeout=1.0): + # ``Topic.subscribe`` already unpacks redis-py pubsub dicts + # and yields the raw ``data`` bytes (or ``None`` on poll + # timeout). Mirror that contract. + if on_subscribe is not None: + on_subscribe() + yield live_bogus.encode() + yield live_real.encode() + while True: + yield None + + with patch.object( + events_module, "get_redis_instance", return_value=redis_client + ), patch.object( + events_module.Topic, "subscribe", _fake_subscribe, create=False + ): + with app.test_client() as c: + r = c.get( + "/api/events", + headers={ + "X-Test-Sub": "alice", + "Last-Event-ID": "1735682300000-0", + }, + ) + body = _drain_until( + r, lambda b: b"live-real" in b, max_chunks=80 + ) + + # Live-real arrived (its id is strictly greater than the + # replayed snapshot's id), with its valid id surfaced as + # the SSE ``id:`` header so the frontend can advance. + assert b"live-real" in body + assert b"id: 1735682500000-0" in body + + # The bogus-id event was still delivered to the client, + # but no ``id: definitely-not-an-id`` line was emitted — + # the malformed id never reached the SSE wire and so + # could not pin the dedup floor. + assert b"live-bogus" in body + assert b"id: definitely-not-an-id" not in body + + +# ── format helpers (already covered in test_events_substrate but +# duplicated here as a smoke for the route's surface) ───────────────── + + +class TestReplayRateLimit: + """Enumeration defenses on the per-user backlog.""" + + def test_allow_replay_returns_true_when_budget_disabled(self): + from application.api.events.routes import _allow_replay + + with patch("application.api.events.routes.settings") as mock_settings: + mock_settings.EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW = 0 + mock_settings.EVENTS_REPLAY_BUDGET_WINDOW_SECONDS = 60 + assert _allow_replay(MagicMock(), "alice", "1735682400000-0") is True + + def test_allow_replay_returns_true_when_redis_unavailable(self): + from application.api.events.routes import _allow_replay + + with patch("application.api.events.routes.settings") as mock_settings: + mock_settings.EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW = 5 + mock_settings.EVENTS_REPLAY_BUDGET_WINDOW_SECONDS = 60 + assert _allow_replay(None, "alice", "1735682400000-0") is True + + def test_allow_replay_skips_incr_when_no_cursor_and_empty_backlog(self): + """Fresh client with no cursor and an empty user stream cannot + do snapshot work — INCR'ing the counter would needlessly + burn budget. Catches the React-StrictMode dev-burst case where + double-mounted components would otherwise 429 in 5 connects. + """ + from application.api.events.routes import _allow_replay + + with patch("application.api.events.routes.settings") as mock_settings: + mock_settings.EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW = 3 + mock_settings.EVENTS_REPLAY_BUDGET_WINDOW_SECONDS = 60 + redis = MagicMock() + redis.xlen.return_value = 0 + + # 5 connects in a row, all with no cursor — none consume + # budget because the backlog is empty. + for _ in range(5): + assert _allow_replay(redis, "alice", None) is True + + redis.xlen.assert_called() + redis.incr.assert_not_called() + + def test_allow_replay_incrs_when_no_cursor_but_backlog_present(self): + """A no-cursor connect against a non-empty backlog *will* do + snapshot work, so it consumes budget normally. + """ + from application.api.events.routes import _allow_replay + + with patch("application.api.events.routes.settings") as mock_settings: + mock_settings.EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW = 5 + mock_settings.EVENTS_REPLAY_BUDGET_WINDOW_SECONDS = 60 + redis = MagicMock() + redis.xlen.return_value = 42 + redis.incr.return_value = 1 + + assert _allow_replay(redis, "alice", None) is True + redis.incr.assert_called_once() + + def test_allow_replay_passes_until_budget_exhausted(self): + from application.api.events.routes import _allow_replay + + with patch("application.api.events.routes.settings") as mock_settings: + mock_settings.EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW = 3 + mock_settings.EVENTS_REPLAY_BUDGET_WINDOW_SECONDS = 60 + redis = MagicMock() + counter = {"v": 0} + + def _incr(_key): + counter["v"] += 1 + return counter["v"] + + redis.incr.side_effect = _incr + + # Cursor set → XLEN short-circuit doesn't fire, INCR always runs. + cursor = "1735682400000-0" + # First three pass. + assert _allow_replay(redis, "alice", cursor) is True + assert _allow_replay(redis, "alice", cursor) is True + assert _allow_replay(redis, "alice", cursor) is True + # Fourth refused. + assert _allow_replay(redis, "alice", cursor) is False + # TTL re-seeded on every successful INCR so a transient + # EXPIRE failure on the seeding call can't wedge the key. + assert redis.expire.call_count == 4 + for call in redis.expire.call_args_list: + assert call.args[1] == 60 + + def test_allow_replay_fail_open_on_redis_error(self): + from application.api.events.routes import _allow_replay + + with patch("application.api.events.routes.settings") as mock_settings: + mock_settings.EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW = 5 + mock_settings.EVENTS_REPLAY_BUDGET_WINDOW_SECONDS = 60 + redis = MagicMock() + redis.incr.side_effect = Exception("redis down") + assert _allow_replay(redis, "alice", "1735682400000-0") is True + + def test_allow_replay_recovers_when_seeding_expire_raises(self): + """Regression: INCR=1 then EXPIRE raising must not wedge the key. + + Earlier code only called EXPIRE when ``used == 1``. If that EXPIRE + raised, the counter persisted with no TTL and every subsequent + call hit ``used > 1`` without re-seeding — locking the user out + until an operator DEL'd the key. The fix calls EXPIRE on every + successful INCR so the next call still re-seeds the TTL. + """ + from application.api.events.routes import _allow_replay + + with patch("application.api.events.routes.settings") as mock_settings: + mock_settings.EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW = 5 + mock_settings.EVENTS_REPLAY_BUDGET_WINDOW_SECONDS = 60 + redis = MagicMock() + counter = {"v": 0} + + def _incr(_key): + counter["v"] += 1 + return counter["v"] + + redis.incr.side_effect = _incr + # First EXPIRE raises (the seeding call that would have + # wedged the key under the old gated logic). Second EXPIRE + # succeeds — and crucially, must still run. + redis.expire.side_effect = [Exception("expire blip"), True] + + cursor = "1735682400000-0" + # First call: INCR=1 succeeds, EXPIRE raises -> outer except + # returns True (fail-open) for this call. + assert _allow_replay(redis, "alice", cursor) is True + # Second call: INCR=2, EXPIRE succeeds -> still under budget, + # and the TTL is now seeded (no permanent lockout). + assert _allow_replay(redis, "alice", cursor) is True + + assert redis.expire.call_count == 2 + # Both EXPIRE calls used the configured window. + for call in redis.expire.call_args_list: + assert call.args[1] == 60 + + def test_replay_backlog_passes_count_to_xrange(self): + from application.api.events.routes import _replay_backlog + + redis = MagicMock() + redis.xrange.return_value = [] + # Drain the iterator so xrange is actually called. + list(_replay_backlog(redis, "alice", None, 200)) + redis.xrange.assert_called_once() + kwargs = redis.xrange.call_args.kwargs + assert kwargs.get("count") == 200 + + def test_returns_429_when_replay_budget_exhausted(self): + """Route refuses the connection rather than serving live tail + only. Earlier behavior silently skipped replay and let the + client advance ``lastEventId`` via id-bearing live frames, + permanently stranding the un-replayed window. The 429 keeps + the cursor pinned so the next reconnect (after the budget + window slides) can replay normally. + """ + from application.api.events import routes as events_module + + app = _make_app() + redis_client = MagicMock() + + def _incr(key): + if key == "user:alice:sse_count": + return 1 + # Budget counter: report over-limit. + return 31 + + redis_client.incr.side_effect = _incr + + with patch.object( + events_module, "get_redis_instance", return_value=redis_client + ), patch.object( + events_module.settings, + "EVENTS_REPLAY_BUDGET_REQUESTS_PER_WINDOW", + 30, + ): + with app.test_client() as c: + r = c.get( + "/api/events", + headers={ + "X-Test-Sub": "alice", + "Last-Event-ID": "1735682300000-0", + }, + ) + assert r.status_code == 429 + # Concurrency slot is released so a budget-denied request + # doesn't permanently consume a connection from the cap. + redis_client.decr.assert_called_once_with("user:alice:sse_count") + + +class TestFormatHelpers: + def test_format_sse_two_terminating_newlines(self): + from application.api.events.routes import _format_sse + + out = _format_sse("hello", event_id="1-0") + assert out.endswith("\n\n") + # Exactly one ``id:`` and one ``data:``. + lines = out.rstrip("\n").split("\n") + assert lines == ["id: 1-0", "data: hello"] + + @pytest.mark.parametrize( + "candidate, expected", + [ + ("1234", "1234"), + ("1234-5", "1234-5"), + (" 1234-0 ", "1234-0"), + (None, None), + ("", None), + (" ", None), + ("nope", None), + ("1234-foo", None), + ], + ) + def test_normalize_last_event_id(self, candidate, expected): + from application.api.events.routes import _normalize_last_event_id + + assert _normalize_last_event_id(candidate) == expected diff --git a/tests/api/test_message_stream_routes.py b/tests/api/test_message_stream_routes.py new file mode 100644 index 00000000..7da6bc30 --- /dev/null +++ b/tests/api/test_message_stream_routes.py @@ -0,0 +1,220 @@ +"""Tests for ``application/api/answer/routes/messages.py``. + +Reconnect endpoint: GET /api/messages//events. Auth gate, ownership +gate, malformed-id rejection, Last-Event-ID normalisation, and a smoke +test that the SSE response shape matches the user-events endpoint. +""" + +from __future__ import annotations + +from unittest.mock import patch + +from flask import Flask, request + +from application.api.answer.routes.messages import ( + _MESSAGE_ID_RE, + _normalise_last_event_id, + messages_bp, +) + + +def _make_app(decoded_token=None): + app = Flask(__name__) + app.register_blueprint(messages_bp) + app.config["TESTING"] = True + + @app.before_request + def _shim_auth(): + request.decoded_token = decoded_token + + return app + + +VALID_UUID = "67d65e8f-e7fb-4df1-9e6e-99ea6c830206" + + +class TestNormaliseLastEventId: + def test_none_passthrough(self): + assert _normalise_last_event_id(None) is None + + def test_empty_string(self): + assert _normalise_last_event_id("") is None + + def test_whitespace_only(self): + assert _normalise_last_event_id(" ") is None + + def test_valid_int(self): + assert _normalise_last_event_id("42") == 42 + + def test_stripped_whitespace(self): + assert _normalise_last_event_id(" 7 ") == 7 + + def test_zero_is_valid(self): + assert _normalise_last_event_id("0") == 0 + + def test_negative_rejected(self): + # We expose only non-negative cursors; -1 is reserved for the + # snapshot-failure synthetic terminal event and shouldn't + # round-trip back. + assert _normalise_last_event_id("-1") is None + + def test_non_numeric_rejected(self): + for bad in ("foo", "1.5", "1e3", "abc-123", "null"): + assert _normalise_last_event_id(bad) is None, bad + + +class TestMessageIdRegex: + def test_canonical_uuid_accepted(self): + assert _MESSAGE_ID_RE.match(VALID_UUID) + + def test_uppercase_uuid_accepted(self): + assert _MESSAGE_ID_RE.match(VALID_UUID.upper()) + + def test_no_dashes_rejected(self): + assert not _MESSAGE_ID_RE.match(VALID_UUID.replace("-", "")) + + def test_legacy_mongo_id_rejected(self): + # 24-char hex with no dashes — a Mongo objectid-shaped string + # that happened to leak through somewhere. + assert not _MESSAGE_ID_RE.match("507f1f77bcf86cd799439011") + + +class TestAuthGate: + def test_401_when_no_decoded_token(self): + app = _make_app(decoded_token=None) + with app.test_client() as c: + r = c.get(f"/api/messages/{VALID_UUID}/events") + assert r.status_code == 401 + + def test_401_when_decoded_token_missing_sub(self): + app = _make_app(decoded_token={"email": "x@y"}) + with app.test_client() as c: + r = c.get(f"/api/messages/{VALID_UUID}/events") + assert r.status_code == 401 + + +class TestMessageIdValidation: + def test_400_on_malformed_id(self): + app = _make_app(decoded_token={"sub": "alice"}) + with app.test_client() as c: + r = c.get("/api/messages/not-a-uuid/events") + assert r.status_code == 400 + + +class TestOwnershipGate: + def test_404_when_user_does_not_own_message(self): + from application.api.answer.routes import messages as messages_module + + app = _make_app(decoded_token={"sub": "alice"}) + with patch.object( + messages_module, "_user_owns_message", return_value=False + ): + with app.test_client() as c: + r = c.get(f"/api/messages/{VALID_UUID}/events") + assert r.status_code == 404 + + def test_200_when_user_owns_message(self): + from application.api.answer.routes import messages as messages_module + + app = _make_app(decoded_token={"sub": "alice"}) + + # Have build_message_event_stream yield just the prelude then + # exit so the test can drain the response without blocking on + # a live pubsub subscription. + def _fake_builder(message_id, last_event_id=None, **kwargs): + yield ": connected\n\n" + + with patch.object( + messages_module, "_user_owns_message", return_value=True + ), patch.object( + messages_module, "build_message_event_stream", _fake_builder + ): + with app.test_client() as c: + r = c.get(f"/api/messages/{VALID_UUID}/events") + assert r.status_code == 200 + assert r.mimetype == "text/event-stream" + assert r.headers.get("Cache-Control") == "no-store" + assert r.headers.get("X-Accel-Buffering") == "no" + body = b"" + for chunk in r.iter_encoded(): + body += chunk + if b": connected" in body: + break + r.close() + assert b": connected" in body + + +class TestLastEventIdParsing: + def test_header_passes_through_to_builder(self): + from application.api.answer.routes import messages as messages_module + + captured = {} + + def _fake_builder(message_id, last_event_id=None, **kwargs): + captured["message_id"] = message_id + captured["last_event_id"] = last_event_id + yield ": connected\n\n" + + app = _make_app(decoded_token={"sub": "alice"}) + with patch.object( + messages_module, "_user_owns_message", return_value=True + ), patch.object( + messages_module, "build_message_event_stream", _fake_builder + ): + with app.test_client() as c: + r = c.get( + f"/api/messages/{VALID_UUID}/events", + headers={"Last-Event-ID": "12"}, + ) + # Drain a tick. + next(iter(r.iter_encoded()), None) + r.close() + assert captured["message_id"] == VALID_UUID + assert captured["last_event_id"] == 12 + + def test_query_param_fallback(self): + from application.api.answer.routes import messages as messages_module + + captured = {} + + def _fake_builder(message_id, last_event_id=None, **kwargs): + captured["last_event_id"] = last_event_id + yield ": connected\n\n" + + app = _make_app(decoded_token={"sub": "alice"}) + with patch.object( + messages_module, "_user_owns_message", return_value=True + ), patch.object( + messages_module, "build_message_event_stream", _fake_builder + ): + with app.test_client() as c: + r = c.get( + f"/api/messages/{VALID_UUID}/events?last_event_id=5" + ) + next(iter(r.iter_encoded()), None) + r.close() + assert captured["last_event_id"] == 5 + + def test_invalid_cursor_normalised_to_none(self): + from application.api.answer.routes import messages as messages_module + + captured = {} + + def _fake_builder(message_id, last_event_id=None, **kwargs): + captured["last_event_id"] = last_event_id + yield ": connected\n\n" + + app = _make_app(decoded_token={"sub": "alice"}) + with patch.object( + messages_module, "_user_owns_message", return_value=True + ), patch.object( + messages_module, "build_message_event_stream", _fake_builder + ): + with app.test_client() as c: + r = c.get( + f"/api/messages/{VALID_UUID}/events", + headers={"Last-Event-ID": "definitely-not-a-number"}, + ) + next(iter(r.iter_encoded()), None) + r.close() + assert captured["last_event_id"] is None diff --git a/tests/api/user/sources/test_upload_idempotency.py b/tests/api/user/sources/test_upload_idempotency.py index bd9eb45a..cade9be5 100644 --- a/tests/api/user/sources/test_upload_idempotency.py +++ b/tests/api/user/sources/test_upload_idempotency.py @@ -543,6 +543,84 @@ class TestRemoteIdempotency: assert response.status_code == 400 assert mock_apply.call_count == 0 + def test_no_header_returns_source_id_matching_worker_kwarg( + self, app, pg_conn, + ): + """Regression: without an ``Idempotency-Key``, the route must + still return a ``source_id`` AND pass that same id to the worker + as ``source_id`` so SSE envelopes line up with what the + frontend already has. Previously the route omitted ``source_id`` + entirely on the no-key path and the worker minted its own + random uuid, breaking push correlation for the default upload + flow. + """ + from application.api.user.sources.upload import UploadRemote + + apply_mock = _apply_async_mock() + with _patch_db(pg_conn), patch( + "application.api.user.sources.upload.ingest_remote.apply_async", + apply_mock, + ), app.test_request_context( + "/api/remote", method="POST", + data={ + "user": "u", "source": "github", "name": "g", + "data": json.dumps({"repo_url": "https://github.com/x/y"}), + }, + content_type="multipart/form-data", + ): + from flask import request + request.decoded_token = {"sub": "u"} + response = UploadRemote().post() + assert response.status_code == 200 + assert "source_id" in response.json + assert ( + apply_mock.call_args.kwargs["kwargs"]["source_id"] + == response.json["source_id"] + ) + + def test_no_header_connector_returns_source_id_matching_worker_kwarg( + self, app, pg_conn, + ): + """Same regression as above for the connector branch + (``ingest_connector_task``). The connector path took the + no-key gap independently of the plain remote path.""" + from application.api.user.sources.upload import UploadRemote + + apply_mock = _apply_async_mock() + # Pick any registered connector — the route only branches on + # ``ConnectorCreator.get_supported_connectors()``. + from application.parser.connectors.connector_creator import ( + ConnectorCreator, + ) + supported = ConnectorCreator.get_supported_connectors() + if not supported: + pytest.skip("no connectors registered in this build") + connector_source = next(iter(supported)) + + with _patch_db(pg_conn), patch( + "application.api.user.sources.upload.ingest_connector_task.apply_async", + apply_mock, + ), app.test_request_context( + "/api/remote", method="POST", + data={ + "user": "u", "source": connector_source, "name": "g", + "data": json.dumps({ + "session_token": "tok", + "file_ids": ["f1"], + }), + }, + content_type="multipart/form-data", + ): + from flask import request + request.decoded_token = {"sub": "u"} + response = UploadRemote().post() + assert response.status_code == 200 + assert "source_id" in response.json + assert ( + apply_mock.call_args.kwargs["kwargs"]["source_id"] + == response.json["source_id"] + ) + def _seed_source(pg_conn, user="u", **kw): from application.storage.db.repositories.sources import SourcesRepository @@ -673,10 +751,140 @@ class TestManageSourceFilesIdempotency: assert apply_mock.call_count == 1 # Loser's response carries the winner's task_id, not the # original 200-with-added_files payload. - assert second.json["task_id"] == first.json["reingest_task_id"] + # ``manage_source_files`` aliases ``task_id`` -> + # ``reingest_task_id`` in the cached payload so the dedup + # response shape matches the fresh-request response (FileTree + # keys reingest correlation on ``reingest_task_id`` / + # ``source_id``). + assert second.json["reingest_task_id"] == first.json["reingest_task_id"] + # Cached ``source_id`` must equal the real source row id (not + # the helper's uuid5-of-key) so FileTree's SSE correlation on + # ``event.scope.id === result.source_id`` keeps working. + assert second.json["source_id"] == first.json["source_id"] + assert second.json["source_id"] == str(src["id"]) # Confirm the loser never invoked the file-save path. assert fake_storage.save_file.call_count == 1 + def test_remove_same_key_second_post_returns_real_source_id( + self, app, pg_conn + ): + """Regression: the ``remove`` cached branch used to leave the + helper's synthetic ``source_id`` (uuid5 of the scoped key) in + place. The reingest worker publishes SSE events tagged with the + real source row id, so the cached response had to be patched to + match what the fresh response returns — otherwise FileTree's + SSE correlation silently fails on every idempotent retry and + the user never sees the directory refresh. + """ + from application.api.user.sources.upload import ManageSourceFiles + + user = "alice-mgr-rmrep" + src = _seed_source( + pg_conn, + user=user, + file_path="/data", + file_name_map={"a.txt": "a.txt"}, + ) + + fake_storage = MagicMock() + fake_storage.file_exists.return_value = True + apply_mock = _apply_async_mock() + + def _do_remove(): + return app.test_request_context( + "/api/manage_source_files", + method="POST", + data={ + "source_id": str(src["id"]), + "operation": "remove", + "file_paths": json.dumps(["a.txt"]), + }, + content_type="multipart/form-data", + headers={"Idempotency-Key": "mgr-rmrep"}, + ) + + with _patch_db(pg_conn), patch( + "application.api.user.sources.upload.StorageCreator.get_storage", + return_value=fake_storage, + ), patch( + "application.api.user.tasks.reingest_source_task.apply_async", + apply_mock, + ): + with _do_remove(): + from flask import request + request.decoded_token = {"sub": user} + first = ManageSourceFiles().post() + with _do_remove(): + from flask import request + request.decoded_token = {"sub": user} + second = ManageSourceFiles().post() + + assert first.status_code == 200 + assert second.status_code == 200 + assert apply_mock.call_count == 1 + assert second.json["reingest_task_id"] == first.json["reingest_task_id"] + # The contract under test: cached source_id matches the fresh + # response (the real source row id), not the helper's uuid5. + assert second.json["source_id"] == first.json["source_id"] + assert second.json["source_id"] == str(src["id"]) + + def test_remove_directory_same_key_second_post_returns_real_source_id( + self, app, pg_conn + ): + """Same regression as the ``remove`` test, for the + ``remove_directory`` branch. + """ + from application.api.user.sources.upload import ManageSourceFiles + + user = "alice-mgr-rmdir-rep" + src = _seed_source( + pg_conn, + user=user, + file_path="/data", + file_name_map={"sub/a.txt": "a.txt"}, + ) + + fake_storage = MagicMock() + fake_storage.is_directory.return_value = True + fake_storage.remove_directory.return_value = True + apply_mock = _apply_async_mock() + + def _do_remove_dir(): + return app.test_request_context( + "/api/manage_source_files", + method="POST", + data={ + "source_id": str(src["id"]), + "operation": "remove_directory", + "directory_path": "sub", + }, + content_type="multipart/form-data", + headers={"Idempotency-Key": "mgr-rmdir-rep"}, + ) + + with _patch_db(pg_conn), patch( + "application.api.user.sources.upload.StorageCreator.get_storage", + return_value=fake_storage, + ), patch( + "application.api.user.tasks.reingest_source_task.apply_async", + apply_mock, + ): + with _do_remove_dir(): + from flask import request + request.decoded_token = {"sub": user} + first = ManageSourceFiles().post() + with _do_remove_dir(): + from flask import request + request.decoded_token = {"sub": user} + second = ManageSourceFiles().post() + + assert first.status_code == 200 + assert second.status_code == 200 + assert apply_mock.call_count == 1 + assert second.json["reingest_task_id"] == first.json["reingest_task_id"] + assert second.json["source_id"] == first.json["source_id"] + assert second.json["source_id"] == str(src["id"]) + def test_concurrent_same_key_only_one_apply_async(self, app, pg_engine): """N parallel same-key POSTs → exactly one apply_async.""" from concurrent.futures import ThreadPoolExecutor diff --git a/tests/api/user/test_conversations.py b/tests/api/user/test_conversations.py index 31ea0b20..8667f6eb 100644 --- a/tests/api/user/test_conversations.py +++ b/tests/api/user/test_conversations.py @@ -506,6 +506,60 @@ class TestGetMessageTail: assert response.status_code == 404 + def test_streaming_row_returns_partial_from_journal(self, app, pg_conn): + """Mid-stream rows must rebuild from message_events, not return the placeholder.""" + from application.api.user.conversations.routes import GetMessageTail + from application.storage.db.repositories.message_events import ( + MessageEventsRepository, + ) + + owner = "user-tail-partial" + _, msg_id = self._seed_in_flight_message(pg_conn, owner) + events_repo = MessageEventsRepository(pg_conn) + events_repo.record(msg_id, 0, "message_id", {"type": "message_id"}) + events_repo.record(msg_id, 1, "answer", {"type": "answer", "answer": "Hello"}) + events_repo.record(msg_id, 2, "answer", {"type": "answer", "answer": ", world"}) + events_repo.record( + msg_id, 3, "source", {"type": "source", "source": [{"id": "s1"}]} + ) + + with _patch_conversations_db(pg_conn), app.test_request_context( + f"/api/messages/{msg_id}/tail" + ): + from flask import request + + request.decoded_token = {"sub": owner} + response = GetMessageTail().get(msg_id) + + assert response.status_code == 200 + assert response.json["status"] == "streaming" + assert response.json["response"] == "Hello, world" + assert response.json["sources"] == [{"id": "s1"}] + assert "terminated prior to completion" not in ( + response.json["response"] or "" + ) + + def test_streaming_row_with_empty_journal_returns_empty_response( + self, app, pg_conn + ): + """Empty journal returns empty response, not the placeholder.""" + from application.api.user.conversations.routes import GetMessageTail + + owner = "user-tail-empty" + _, msg_id = self._seed_in_flight_message(pg_conn, owner) + + with _patch_conversations_db(pg_conn), app.test_request_context( + f"/api/messages/{msg_id}/tail" + ): + from flask import request + + request.decoded_token = {"sub": owner} + response = GetMessageTail().get(msg_id) + + assert response.status_code == 200 + assert response.json["status"] == "streaming" + assert response.json["response"] == "" + class TestUpdateConversationNameHappy: def test_returns_401_unauthenticated(self, app): diff --git a/tests/api/user/test_tasks.py b/tests/api/user/test_tasks.py index 203d8383..a2792db0 100644 --- a/tests/api/user/test_tasks.py +++ b/tests/api/user/test_tasks.py @@ -33,7 +33,7 @@ class TestIngestTask: mock_worker.assert_called_once_with( ANY, "dir", ["pdf"], "job1", "/path", "file.pdf", "user1", - file_name_map=None, idempotency_key=None, + file_name_map=None, idempotency_key=None, source_id=None, ) assert result == {"status": "ok"} @@ -50,7 +50,7 @@ class TestIngestTask: mock_worker.assert_called_once_with( ANY, "dir", ["pdf"], "job1", "/path", "file.pdf", "user1", - file_name_map=name_map, idempotency_key=None, + file_name_map=name_map, idempotency_key=None, source_id=None, ) @@ -66,7 +66,7 @@ class TestIngestRemoteTask: mock_worker.assert_called_once_with( ANY, {"url": "http://x"}, "job1", "user1", "web", - idempotency_key=None, + idempotency_key=None, source_id=None, ) assert result == {"status": "ok"} @@ -169,6 +169,7 @@ class TestIngestConnectorTask: doc_id=None, sync_frequency="never", idempotency_key=None, + source_id=None, ) assert result == {"status": "ok"} @@ -207,6 +208,7 @@ class TestIngestConnectorTask: doc_id="doc1", sync_frequency="daily", idempotency_key=None, + source_id=None, ) assert result == {"status": "ok"} @@ -220,7 +222,7 @@ class TestSetupPeriodicTasks: setup_periodic_tasks(sender) - assert sender.add_periodic_task.call_count == 7 + assert sender.add_periodic_task.call_count == 8 calls = sender.add_periodic_task.call_args_list @@ -241,6 +243,9 @@ class TestSetupPeriodicTasks: assert calls[5][1].get("name") == "reconciliation" # version-check (every 7h) assert calls[6][0][0] == timedelta(hours=7) + # message_events retention sweep (24h) + assert calls[7][0][0] == timedelta(hours=24) + assert calls[7][1].get("name") == "cleanup-message-events" class TestMcpOauthTask: @@ -257,20 +262,6 @@ class TestMcpOauthTask: assert result == {"url": "http://auth"} -class TestMcpOauthStatusTask: - @pytest.mark.unit - @patch("application.api.user.tasks.mcp_oauth_status") - def test_calls_mcp_oauth_status(self, mock_worker): - from application.api.user.tasks import mcp_oauth_status_task - - mock_worker.return_value = {"status": "authorized"} - - result = mcp_oauth_status_task("task123") - - mock_worker.assert_called_once_with(ANY, "task123") - assert result == {"status": "authorized"} - - class TestDurableTaskRetryPolicy: """The long-running tasks share a uniform retry policy.""" @@ -302,7 +293,6 @@ class TestDurableTaskRetryPolicy: "schedule_syncs", "sync_source", "mcp_oauth_task", - "mcp_oauth_status_task", "cleanup_pending_tool_state", "reconciliation_task", "version_check_task", @@ -438,6 +428,93 @@ class TestCleanupPendingToolState: } +class TestCleanupMessageEventsTask: + """Retention janitor delegates to MessageEventsRepository.cleanup_older_than.""" + + @pytest.mark.unit + def test_skips_when_postgres_uri_missing(self, monkeypatch): + from application.api.user.tasks import cleanup_message_events + from application.core.settings import settings + + monkeypatch.setattr(settings, "POSTGRES_URI", None, raising=False) + + result = cleanup_message_events.run() + assert result == {"deleted": 0, "skipped": "POSTGRES_URI not set"} + + @pytest.mark.unit + def test_deletes_rows_past_retention_window(self, pg_conn, monkeypatch): + import uuid + + from sqlalchemy import text as _text + + from application.api.user.tasks import cleanup_message_events + from application.core.settings import settings + from application.storage.db.repositories.message_events import ( + MessageEventsRepository, + ) + + # Seed parent rows so the FK on message_events holds. + user_id = f"user-{uuid.uuid4().hex[:8]}" + conv_id = uuid.uuid4() + msg_id = uuid.uuid4() + pg_conn.execute( + _text("INSERT INTO users (user_id) VALUES (:u)"), + {"u": user_id}, + ) + pg_conn.execute( + _text( + "INSERT INTO conversations (id, user_id, name) " + "VALUES (:id, :u, 'test')" + ), + {"id": conv_id, "u": user_id}, + ) + pg_conn.execute( + _text( + "INSERT INTO conversation_messages (id, conversation_id, " + "user_id, position) VALUES (:id, :c, :u, 0)" + ), + {"id": msg_id, "c": conv_id, "u": user_id}, + ) + + repo = MessageEventsRepository(pg_conn) + repo.record(str(msg_id), 0, "answer", {"chunk": "stale"}) + repo.record(str(msg_id), 1, "answer", {"chunk": "fresh"}) + # Backdate seq=0 past the default 14-day retention so the + # janitor catches it; seq=1 stays at "now" and must survive. + pg_conn.execute( + _text( + "UPDATE message_events SET created_at = now() - interval '20 days' " + "WHERE message_id = CAST(:id AS uuid) AND sequence_no = 0" + ), + {"id": str(msg_id)}, + ) + + monkeypatch.setattr( + settings, "POSTGRES_URI", "postgresql://stub", raising=False + ) + + @contextmanager + def _fake_begin(): + yield pg_conn + + fake_engine = MagicMock() + fake_engine.begin = _fake_begin + + with patch( + "application.storage.db.engine.get_engine", + return_value=fake_engine, + ): + result = cleanup_message_events.run() + + assert result == { + "deleted": 1, + "ttl_days": settings.MESSAGE_EVENTS_RETENTION_DAYS, + } + # Only the fresh row survives. + rows = repo.read_after(str(msg_id)) + assert [r["sequence_no"] for r in rows] == [1] + + class TestIngestIdempotency: """Same short-circuit applies to the ingest task path.""" @@ -449,7 +526,7 @@ class TestIngestIdempotency: def _fake_worker(self, directory, formats, job_name, file_path, filename, user, file_name_map=None, - idempotency_key=None): + idempotency_key=None, source_id=None): worker_calls.append(filename) return {"status": "ok", "directory": directory} diff --git a/tests/api/user/test_tools_mcp_pg.py b/tests/api/user/test_tools_mcp_pg.py index 41796004..ecd98902 100644 --- a/tests/api/user/test_tools_mcp_pg.py +++ b/tests/api/user/test_tools_mcp_pg.py @@ -363,47 +363,6 @@ class TestMCPServerSave: assert response.status_code in (200, 201) -class TestMCPOAuthStatus: - def test_returns_pending_when_no_data(self, app): - from application.api.user.tools.mcp import MCPOAuthStatus - - fake_redis = MagicMock() - fake_redis.get.return_value = None - - with patch( - "application.api.user.tools.mcp.get_redis_instance", - return_value=fake_redis, - ), app.test_request_context("/api/mcp_oauth_status/t1"): - response = MCPOAuthStatus().get("t1") - assert response.status_code == 200 - assert response.json["status"] == "pending" - - def test_returns_status_from_redis(self, app): - from application.api.user.tools.mcp import MCPOAuthStatus - - fake_redis = MagicMock() - fake_redis.get.return_value = '{"status": "completed", "tools": [{"name": "t1", "description": "d"}]}' - - with patch( - "application.api.user.tools.mcp.get_redis_instance", - return_value=fake_redis, - ), app.test_request_context("/api/mcp_oauth_status/t1"): - response = MCPOAuthStatus().get("t1") - assert response.status_code == 200 - assert response.json["status"] == "completed" - assert response.json["tools"][0]["name"] == "t1" - - def test_redis_error_returns_500(self, app): - from application.api.user.tools.mcp import MCPOAuthStatus - - with patch( - "application.api.user.tools.mcp.get_redis_instance", - side_effect=RuntimeError("boom"), - ), app.test_request_context("/api/mcp_oauth_status/t1"): - response = MCPOAuthStatus().get("t1") - assert response.status_code == 500 - - class TestMCPOAuthCallback: def test_error_param_redirects_error(self, app): from application.api.user.tools.mcp import MCPOAuthCallback diff --git a/tests/api/v1/test_routes_extended.py b/tests/api/v1/test_routes_extended.py index a832ab57..c7a46eae 100644 --- a/tests/api/v1/test_routes_extended.py +++ b/tests/api/v1/test_routes_extended.py @@ -513,4 +513,73 @@ class TestChatCompletionsHappyPath: assert resp.status_code == 200 assert resp.mimetype == "text/event-stream" + def test_stream_handles_id_prefixed_chunks(self, pg_conn): + """``complete_stream`` emits ``id: \\n`` before each + ``data:`` line. The v1 streaming consumer must skip the id + header and the informational ``message_id`` event, not silently + drop every chunk. + """ + app = _build_app() + + def _fake_translate(data, api_key): + return {"question": "hi"} + + fake_processor = MagicMock() + fake_processor.decoded_token = {"sub": "u"} + fake_processor.conversation_id = "conv-1" + fake_processor.agent_config = {} + fake_processor.agent_id = None + fake_processor.model_id = "m" + + def _fake_helper_complete_stream(**kw): + # Mirror the new wire format: id-prefixed records, plus + # the informational message_id event the v1 path doesn't + # have an analog for. + yield 'id: 0\ndata: {"type": "message_id", "message_id": "m-1"}\n\n' + yield 'id: 1\ndata: {"type": "id", "id": "conv-1"}\n\n' + yield 'id: 2\ndata: {"type": "answer", "answer": "hi"}\n\n' + + translated_chunks: list = [] + + def _fake_translate_stream_event(event_data, completion_id, model_name): + translated_chunks.append(event_data) + return ['data: x\n\n'] + + fake_helper = MagicMock() + fake_helper.check_usage.return_value = None + fake_helper.complete_stream.side_effect = _fake_helper_complete_stream + + with _patch_v1_db(pg_conn), patch( + "application.api.v1.routes.translate_request", + side_effect=_fake_translate, + ), patch( + "application.api.v1.routes.StreamProcessor", + return_value=fake_processor, + ), patch( + "application.api.v1.routes._V1AnswerHelper", + return_value=fake_helper, + ), patch( + "application.api.v1.routes.translate_stream_event", + side_effect=_fake_translate_stream_event, + ): + with app.test_client() as c: + resp = c.post( + "/v1/chat/completions", + headers={"Authorization": "Bearer x"}, + json={ + "messages": [{"role": "user", "content": "Hi"}], + "stream": True, + }, + ) + # Drain the response so the generator runs to completion. + list(resp.iter_encoded()) + + assert resp.status_code == 200 + # message_id event is skipped (no v1 analog); id + answer are + # decoded and forwarded to the translator. + types_translated = [c.get("type") for c in translated_chunks] + assert "message_id" not in types_translated + assert "id" in types_translated + assert "answer" in types_translated + diff --git a/tests/conftest.py b/tests/conftest.py index 8670673f..e1c35531 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,9 +15,9 @@ their own conftest to point at a real, long-running Postgres instance tests and are marked with ``@pytest.mark.integration``. No mongomock. The ``mock_mongo_db`` fixture that used to live here was -removed as part of the Phase 4/5 Mongo→Postgres cutover. Tests that -still reference it will fail with "fixture not found" until the -corresponding route handler is migrated to a repository read. +removed as part of the Mongo→Postgres cutover. Tests that still +reference it will fail with "fixture not found" until the corresponding +route handler is migrated to a repository read. """ from __future__ import annotations diff --git a/tests/core/test_model_registry_yaml.py b/tests/core/test_model_registry_yaml.py index 169d74ba..326977a0 100644 --- a/tests/core/test_model_registry_yaml.py +++ b/tests/core/test_model_registry_yaml.py @@ -1,4 +1,4 @@ -"""Phase 1 regression tests for the YAML-driven ModelRegistry. +"""Regression tests for the YAML-driven ModelRegistry. These tests encode the contract that persisted agent / workflow / conversation references depend on: every model id and core capability diff --git a/tests/core/test_models_config_dir.py b/tests/core/test_models_config_dir.py index 30718ca8..d6c7abee 100644 --- a/tests/core/test_models_config_dir.py +++ b/tests/core/test_models_config_dir.py @@ -1,4 +1,4 @@ -"""Phase 3 tests: operator MODELS_CONFIG_DIR. +"""Tests for the operator MODELS_CONFIG_DIR. Covers the operator-supplied directory of model YAMLs that's loaded after the built-in catalog. Operators use this to add new diff --git a/tests/core/test_openai_compatible.py b/tests/core/test_openai_compatible.py index 75bb8da1..f8f52987 100644 --- a/tests/core/test_openai_compatible.py +++ b/tests/core/test_openai_compatible.py @@ -1,4 +1,4 @@ -"""Phase 2 tests for the openai_compatible provider. +"""Tests for the openai_compatible provider. Covers YAML loading from a temp directory, multiple coexisting catalogs (Mistral + Together), env-var-based credential resolution, the legacy diff --git a/tests/e2e/helpers/agents.ts b/tests/e2e/helpers/agents.ts index ac4f60a1..7897b522 100644 --- a/tests/e2e/helpers/agents.ts +++ b/tests/e2e/helpers/agents.ts @@ -1,6 +1,6 @@ /** - * Phase 2 helper — shared agent provisioning for specs that need a - * published agent (with a real api_key) for subsequent /stream or /search + * Shared agent provisioning for specs that need a published agent + * (with a real api_key) for subsequent /stream or /search * calls. A PUBLISHED classic agent requires name, description, chunks, * retriever, prompt_id AND a source — otherwise `/api/create_agent` * returns 400. diff --git a/tests/e2e/helpers/api.ts b/tests/e2e/helpers/api.ts index ead5db21..7a226a2d 100644 --- a/tests/e2e/helpers/api.ts +++ b/tests/e2e/helpers/api.ts @@ -1,5 +1,4 @@ /** - * Phase 1 helper — see e2e-plan.md §P1-B. * Pre-authenticated Playwright APIRequestContext pointed at the e2e Flask. */ diff --git a/tests/e2e/helpers/auth.ts b/tests/e2e/helpers/auth.ts index cd60be10..65e5c1a6 100644 --- a/tests/e2e/helpers/auth.ts +++ b/tests/e2e/helpers/auth.ts @@ -1,5 +1,4 @@ /** - * Phase 1 helper — see e2e-plan.md §P1-B. * JWT signing + per-test BrowserContext seeding for DocsGPT e2e. */ diff --git a/tests/e2e/helpers/db.ts b/tests/e2e/helpers/db.ts index 7df6f94f..b5831a3d 100644 --- a/tests/e2e/helpers/db.ts +++ b/tests/e2e/helpers/db.ts @@ -1,5 +1,4 @@ /** - * Phase 1 helper — see e2e-plan.md §P1-B. * Thin pg wrapper + typed row helpers for DB assertions in specs. */ diff --git a/tests/e2e/helpers/reset.ts b/tests/e2e/helpers/reset.ts index af922ce8..58414cca 100644 --- a/tests/e2e/helpers/reset.ts +++ b/tests/e2e/helpers/reset.ts @@ -1,5 +1,4 @@ /** - * Phase 1 helper — see e2e-plan.md §P1-B. * Per-test TRUNCATE — preserves `alembic_version`, wipes every other table. */ diff --git a/tests/e2e/helpers/streaming.ts b/tests/e2e/helpers/streaming.ts index a104578e..e2ff9b77 100644 --- a/tests/e2e/helpers/streaming.ts +++ b/tests/e2e/helpers/streaming.ts @@ -1,5 +1,5 @@ /** - * Phase 2 helper — shared streaming primitives. + * Shared streaming primitives for SSE specs. * * The backend exposes two streaming endpoints: * - POST /stream (answer_ns path="/" + route "/stream") — SSE body diff --git a/tests/e2e/helpers/uploads.ts b/tests/e2e/helpers/uploads.ts index f4c950bf..594fddfe 100644 --- a/tests/e2e/helpers/uploads.ts +++ b/tests/e2e/helpers/uploads.ts @@ -1,5 +1,5 @@ /** - * Phase 3 helper — upload / task_status polling primitives for Tier-B specs. + * Upload / task_status polling primitives shared across upload specs. * * These helpers wrap the two patterns that recur across B1/B3/B4/B5: * diff --git a/tests/e2e/playwright.config.ts b/tests/e2e/playwright.config.ts index f091fd96..d143c274 100644 --- a/tests/e2e/playwright.config.ts +++ b/tests/e2e/playwright.config.ts @@ -2,7 +2,7 @@ // mock LLM, Postgres reset) is handled by `scripts/e2e/up.sh` and // `scripts/e2e/down.sh`. Playwright's built-in `webServer` can only manage one // process and would fight with the four-service native setup this suite needs. -// See `e2e-plan.md` → "Phase 0 — Foundation" → P0-A for the orchestration +// See `e2e-plan.md` → "Foundation" → P0-A for the orchestration // contract. import { defineConfig, devices } from '@playwright/test'; diff --git a/tests/e2e/specs/auth/isolation.spec.ts b/tests/e2e/specs/auth/isolation.spec.ts index 9d72b841..d2b49134 100644 --- a/tests/e2e/specs/auth/isolation.spec.ts +++ b/tests/e2e/specs/auth/isolation.spec.ts @@ -1,8 +1,9 @@ /** - * Phase 1 — P1-C · cross-user isolation. + * P1-C · cross-user isolation. * - * Covers the Phase 1 acceptance criterion: two freshly-minted users (A and - * B) must not see each other's writes. User A creates a prompt; user B's + * Covers the cross-user isolation acceptance criterion: two + * freshly-minted users (A and B) must not see each other's writes. + * User A creates a prompt; user B's * `/api/get_prompts` must not include it, and the DB must report zero * `prompts` rows for user B. * diff --git a/tests/e2e/specs/auth/session_jwt.spec.ts b/tests/e2e/specs/auth/session_jwt.spec.ts index b9f22500..8276b524 100644 --- a/tests/e2e/specs/auth/session_jwt.spec.ts +++ b/tests/e2e/specs/auth/session_jwt.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 1 — P1-C · session_jwt auth flow. + * P1-C · session_jwt auth flow. * * Covers e2e-plan.md §7.2: with `AUTH_TYPE=session_jwt` the backend issues a * fresh token via `/api/generate_token` on first load, the frontend stashes diff --git a/tests/e2e/specs/auth/simple_jwt.spec.ts b/tests/e2e/specs/auth/simple_jwt.spec.ts index bf0b2c73..e90edae1 100644 --- a/tests/e2e/specs/auth/simple_jwt.spec.ts +++ b/tests/e2e/specs/auth/simple_jwt.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 1 — P1-C · simple_jwt auth flow. + * P1-C · simple_jwt auth flow. * * Covers e2e-plan.md §7.3: with `AUTH_TYPE=simple_jwt` the backend does NOT * issue tokens — the frontend shows `JWTModal`, the user pastes a diff --git a/tests/e2e/specs/tier-a/agent-pin.spec.ts b/tests/e2e/specs/tier-a/agent-pin.spec.ts index 56d9375c..4230dda6 100644 --- a/tests/e2e/specs/tier-a/agent-pin.spec.ts +++ b/tests/e2e/specs/tier-a/agent-pin.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 2 — P2-11 · agent pin/unpin. + * P2-11 · agent pin/unpin. * * Writes to `users.agent_preferences.pinned` (JSONB text[]) via * `POST /api/pin_agent?id=`. Reads via `GET /api/pinned_agents`. diff --git a/tests/e2e/specs/tier-a/agent-share-user.spec.ts b/tests/e2e/specs/tier-a/agent-share-user.spec.ts index 2e177c4e..38d9a450 100644 --- a/tests/e2e/specs/tier-a/agent-share-user.spec.ts +++ b/tests/e2e/specs/tier-a/agent-share-user.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 2 — P2-12 · Cross-user agent share. + * P2-12 · Cross-user agent share. * * Writes to `users.agent_preferences.shared_with_me` (JSONB) and * `agents.shared_token` (CITEXT UNIQUE). Exercises the full cross-user diff --git a/tests/e2e/specs/tier-a/agents.spec.ts b/tests/e2e/specs/tier-a/agents.spec.ts index 86b3bf88..cfefdca0 100644 --- a/tests/e2e/specs/tier-a/agents.spec.ts +++ b/tests/e2e/specs/tier-a/agents.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 2 — P2-10 · Agent CRUD (classic type). + * P2-10 · Agent CRUD (classic type). * * Writes to `agents`, reads `prompts`, touches `users`. Exercises the full * create→publish→update→delete lifecycle plus cross-tenant isolation and a diff --git a/tests/e2e/specs/tier-a/analytics.spec.ts b/tests/e2e/specs/tier-a/analytics.spec.ts index db00d11f..33d59ea3 100644 --- a/tests/e2e/specs/tier-a/analytics.spec.ts +++ b/tests/e2e/specs/tier-a/analytics.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 2 — P2-04 · Token usage / message / feedback analytics. + * P2-04 · Token usage / message / feedback analytics. * * Silent-break covered: analytics bucket keys use YYYY-MM-DD HH24:00 format * (Postgres `to_char`). A drift to the Python `%Y-%m-%d %H:00` strftime diff --git a/tests/e2e/specs/tier-a/attachments.spec.ts b/tests/e2e/specs/tier-a/attachments.spec.ts index f610b665..32eb2a57 100644 --- a/tests/e2e/specs/tier-a/attachments.spec.ts +++ b/tests/e2e/specs/tier-a/attachments.spec.ts @@ -1,7 +1,7 @@ /** - * Phase 2 — P2-14 · Attachment upload bound to conversation message. + * P2-14 · Attachment upload bound to conversation message. * - * Writes to the Tier-1 `attachments` table (via `/api/store_attachment` → Celery + * Writes to the `attachments` table (via `/api/store_attachment` → Celery * worker `attachment_worker` → `AttachmentsRepository.create`) and, via the * subsequent `/stream` call, to `conversation_messages.attachments[]` * (UUID[]). @@ -32,7 +32,7 @@ * flow is flakier here than a raw multipart POST, and the P2 brief explicitly * permits API-direct when the upload surface is Celery-dispatched). DB * assertions on BOTH `attachments` and `conversation_messages.attachments[]` - * are non-negotiable per the Phase 2 invariant. + * are non-negotiable per the migration invariant. */ import { readFile } from 'node:fs/promises'; diff --git a/tests/e2e/specs/tier-a/chat-turn.spec.ts b/tests/e2e/specs/tier-a/chat-turn.spec.ts index 480e6a33..b8108730 100644 --- a/tests/e2e/specs/tier-a/chat-turn.spec.ts +++ b/tests/e2e/specs/tier-a/chat-turn.spec.ts @@ -1,7 +1,7 @@ /** - * Phase 2 — P2-07 · chat turn save (the HOTTEST migration path). + * P2-07 · chat turn save (the HOTTEST migration path). * - * A single /stream call is responsible for writes to FOUR Tier-1 tables: + * A single /stream call is responsible for writes to FOUR core tables: * * - `conversations` (1 row per new conversation) * - `conversation_messages` (1 row per turn — prompt AND response in the @@ -186,7 +186,7 @@ test.describe('tier-a · chat turn save', () => { await resetDb(); }); - test('single chat turn via UI writes to all four Tier-1 tables and reload rehydrates the conversation', async ({ + test('single chat turn via UI writes to all four core tables and reload rehydrates the conversation', async ({ browser, }) => { const { context, sub } = await newUserContext(browser); diff --git a/tests/e2e/specs/tier-a/conversation-crud.spec.ts b/tests/e2e/specs/tier-a/conversation-crud.spec.ts index 3affc8d2..67800b97 100644 --- a/tests/e2e/specs/tier-a/conversation-crud.spec.ts +++ b/tests/e2e/specs/tier-a/conversation-crud.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 2 — P2-08 · Conversation CRUD (Tier-A migration-critical). + * P2-08 · Conversation CRUD (Tier-A migration-critical). * * Covers e2e-plan.md §P2 row P2-08: writes/deletes against `conversations` * and the cascading child tables (`conversation_messages`, diff --git a/tests/e2e/specs/tier-a/feedback.spec.ts b/tests/e2e/specs/tier-a/feedback.spec.ts index 0fdc5edf..e588569f 100644 --- a/tests/e2e/specs/tier-a/feedback.spec.ts +++ b/tests/e2e/specs/tier-a/feedback.spec.ts @@ -1,14 +1,14 @@ /** - * Phase 2 — P2-05 · Message feedback. + * P2-05 · Message feedback. * - * Writes to the JSONB `conversation_messages.feedback` column. The separate - * `feedback` table planned in early Phase 1 was dropped during Phase 3 - * hardening — see migration-postgres.md §2.2: - * "A dedicated `feedback` table was originally planned for this tier - * (and built in early Phase 1) but dropped during Phase 3 hardening: - * per-message feedback lives on `conversation_messages.feedback` (JSONB) - * via `ConversationsRepository.set_feedback`, and the standalone table - * was never populated or read." + * Writes to the JSONB `conversation_messages.feedback` column. A + * separate `feedback` table was considered earlier in the migration + * but dropped — see migration-postgres.md §2.2: + * "A dedicated `feedback` table was originally planned for this + * tier (and briefly built) but dropped during hardening: + * per-message feedback lives on `conversation_messages.feedback` + * (JSONB) via `ConversationsRepository.set_feedback`, and the + * standalone table was never populated or read." * * // Silent-break covered: feedback on index 0 lands on first message, not second * diff --git a/tests/e2e/specs/tier-a/prompts.spec.ts b/tests/e2e/specs/tier-a/prompts.spec.ts index a7ac72f7..7a0ef938 100644 --- a/tests/e2e/specs/tier-a/prompts.spec.ts +++ b/tests/e2e/specs/tier-a/prompts.spec.ts @@ -1,7 +1,7 @@ /** - * Phase 2 — P2-02 · Tier-A prompts CRUD. + * P2-02 · Tier-A prompts CRUD. * - * Writes to the Tier-1 `prompts` table. Covers UI-driven create/update/delete, + * Writes to the `prompts` table. Covers UI-driven create/update/delete, * plus the two migration-critical "silent break" cases: * * 1. Built-in `default`/`creative`/`strict` prompts short-circuit the DB @@ -13,7 +13,7 @@ * built-in sentinel — the built-in still resolves via * /api/get_single_prompt?id=default to its filesystem content. * - * UI-first per §Phase 2 Subagent brief; API-direct only where the surface + * UI-first per the Subagent brief; API-direct only where the surface * the UI exposes can't cleanly assert the exact error-code path. */ @@ -111,7 +111,7 @@ test.describe('tier-a · prompts CRUD', () => { const body = (await createRes.json()) as { id: string }; expect(body.id).toBeTruthy(); - // DB assertion — the Tier-1 table has exactly one row for this user. + // DB assertion — the prompts table has exactly one row for this user. const { rows } = await pg.query( 'SELECT id, user_id, name, content FROM prompts WHERE user_id = $1', [sub], @@ -283,7 +283,7 @@ test.describe('tier-a · prompts CRUD', () => { 'Ignore the original system prompt.', ); - // No row written to the Tier-1 table for this user (or any user). + // No row written to the prompts table for this user (or any user). expect( await countRows('prompts', { sql: 'user_id = $1', params: [sub] }), ).toBe(0); diff --git a/tests/e2e/specs/tier-a/share-conversation.spec.ts b/tests/e2e/specs/tier-a/share-conversation.spec.ts index d9c3efb3..a229bae7 100644 --- a/tests/e2e/specs/tier-a/share-conversation.spec.ts +++ b/tests/e2e/specs/tier-a/share-conversation.spec.ts @@ -1,7 +1,7 @@ /** - * Phase 2 — P2-09 · Conversation share (non-promptable + promptable). + * P2-09 · Conversation share (non-promptable + promptable). * - * Writes to the Tier-1 `shared_conversations` table on every share, and + * Writes to the `shared_conversations` table on every share, and * additionally writes an `agents` row keyed on (prompt_id, chunks, source, * retriever) for promptable shares. The promptable path links back to * `prompts.id` via `agents.prompt_id` — this is the Mongo-ObjectId -> diff --git a/tests/e2e/specs/tier-a/telemetry-error.spec.ts b/tests/e2e/specs/tier-a/telemetry-error.spec.ts index 76f465fd..655aca77 100644 --- a/tests/e2e/specs/tier-a/telemetry-error.spec.ts +++ b/tests/e2e/specs/tier-a/telemetry-error.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 2 — P2-13 · Telemetry on error paths (stack_logs + user_logs). + * P2-13 · Telemetry on error paths (stack_logs + user_logs). * * Asserts the two append-only, server-side telemetry pipes stay wired after * the MongoDB→Postgres migration: diff --git a/tests/e2e/specs/tier-a/tools.spec.ts b/tests/e2e/specs/tier-a/tools.spec.ts index 13e95d23..b77dab8f 100644 --- a/tests/e2e/specs/tier-a/tools.spec.ts +++ b/tests/e2e/specs/tier-a/tools.spec.ts @@ -1,7 +1,7 @@ /** - * Phase 2 — P2-03 · Tools CRUD + encrypted secrets. + * P2-03 · Tools CRUD + encrypted secrets. * - * Writes to the Tier-1 `user_tools` table via the `/api/create_tool`, + * Writes to the `user_tools` table via the `/api/create_tool`, * `/api/update_tool`, `/api/update_tool_status`, and `/api/delete_tool` * endpoints (the same calls the UI issues from `frontend/src/settings/Tools.tsx` * + `ToolConfig.tsx` + `AddToolModal.tsx` + `ConfigToolModal.tsx`). Each test diff --git a/tests/e2e/specs/tier-a/user-logs.spec.ts b/tests/e2e/specs/tier-a/user-logs.spec.ts index d39fc3d2..4b36d170 100644 --- a/tests/e2e/specs/tier-a/user-logs.spec.ts +++ b/tests/e2e/specs/tier-a/user-logs.spec.ts @@ -1,7 +1,7 @@ /** - * Phase 2 — P2-06 · User activity logs viewer. + * P2-06 · User activity logs viewer. * - * Reads from the Tier-1 append-only `user_logs` table via the + * Reads from the append-only `user_logs` table via the * `POST /api/get_user_logs` endpoint (the call the UI issues from * `frontend/src/settings/Logs.tsx` → `userService.getLogs`). Each log row is * written server-side by `complete_stream` in diff --git a/tests/e2e/specs/tier-b/agent-preview.spec.ts b/tests/e2e/specs/tier-b/agent-preview.spec.ts index fc50c274..0b86f0fd 100644 --- a/tests/e2e/specs/tier-b/agent-preview.spec.ts +++ b/tests/e2e/specs/tier-b/agent-preview.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 3 Tier-B · agent preview does NOT persist conversations + * Tier-B · agent preview does NOT persist conversations * * Covers B14. The preview flow in `frontend/src/agents/agentPreviewSlice.ts` * calls `handleFetchAnswerSteaming(..., save_conversation=false)` — see the diff --git a/tests/e2e/specs/tier-b/artifacts.spec.ts b/tests/e2e/specs/tier-b/artifacts.spec.ts index 701deb66..d9e88993 100644 --- a/tests/e2e/specs/tier-b/artifacts.spec.ts +++ b/tests/e2e/specs/tier-b/artifacts.spec.ts @@ -1,4 +1,4 @@ -// Phase 3 Tier-B · B15 artifact sidebar (todos / notes / memory) + legacy fallback. +// Tier-B · B15 artifact sidebar (todos / notes / memory) + legacy fallback. /** * The UI's artifact sidebar (`frontend/src/components/ArtifactSidebar.tsx`) * opens on a question that triggered a tool call that wrote a `note` or a diff --git a/tests/e2e/specs/tier-b/chunks.spec.ts b/tests/e2e/specs/tier-b/chunks.spec.ts index 3ec5e00d..844fb36b 100644 --- a/tests/e2e/specs/tier-b/chunks.spec.ts +++ b/tests/e2e/specs/tier-b/chunks.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 3 Tier-B · chunks (B3) — get / add / update / delete chunks. + * Tier-B · chunks (B3) — get / add / update / delete chunks. * * Exercises `application/api/user/sources/chunks.py`. All chunk endpoints go * through `get_vector_store(str(doc["id"]))` (application/api/user/base.py: diff --git a/tests/e2e/specs/tier-b/folders.spec.ts b/tests/e2e/specs/tier-b/folders.spec.ts index a071d681..62ce9527 100644 --- a/tests/e2e/specs/tier-b/folders.spec.ts +++ b/tests/e2e/specs/tier-b/folders.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 3 Tier-B · agent folders CRUD + nesting + move/bulk-move + * Tier-B · agent folders CRUD + nesting + move/bulk-move * * Covers B10: create/list/rename/delete folders, nest via parent_id, cycle * prevention, and single/bulk agent-move into folders. API-only — folder diff --git a/tests/e2e/specs/tier-b/images.spec.ts b/tests/e2e/specs/tier-b/images.spec.ts index 463efcf8..6c09e59d 100644 --- a/tests/e2e/specs/tier-b/images.spec.ts +++ b/tests/e2e/specs/tier-b/images.spec.ts @@ -1,4 +1,4 @@ -// Phase 3 Tier-B · B17 image serve endpoint. +// Tier-B · B17 image serve endpoint. /** * Exercises `GET /api/images/` in * `application/api/user/attachments/routes.py` — the static-ish endpoint diff --git a/tests/e2e/specs/tier-b/search.spec.ts b/tests/e2e/specs/tier-b/search.spec.ts index 2c7d277c..b2f38fcb 100644 --- a/tests/e2e/specs/tier-b/search.spec.ts +++ b/tests/e2e/specs/tier-b/search.spec.ts @@ -1,6 +1,6 @@ -// Phase 3 Tier-B · /api/search via agent api_key +// Tier-B · /api/search via agent api_key /** - * Phase 3 — B8 · `/api/search` via agent api_key. + * B8 · `/api/search` via agent api_key. * * `application/api/answer/routes/search.py` is the fast-path retrieval * endpoint used by embed/widget integrations: it takes a body with diff --git a/tests/e2e/specs/tier-b/source-files.spec.ts b/tests/e2e/specs/tier-b/source-files.spec.ts index 6c71e6c8..cd56a02d 100644 --- a/tests/e2e/specs/tier-b/source-files.spec.ts +++ b/tests/e2e/specs/tier-b/source-files.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 3 Tier-B · source-files (B5) — POST /api/manage_source_files. + * Tier-B · source-files (B5) — POST /api/manage_source_files. * * Exercises the three `operation` branches in * `application/api/user/sources/upload.py::ManageSourceFiles`: diff --git a/tests/e2e/specs/tier-b/source-sync.spec.ts b/tests/e2e/specs/tier-b/source-sync.spec.ts index 4d7854ea..aaf303fd 100644 --- a/tests/e2e/specs/tier-b/source-sync.spec.ts +++ b/tests/e2e/specs/tier-b/source-sync.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 3 Tier-B · source-sync (B4) — sync frequency + manual sync. + * Tier-B · source-sync (B4) — sync frequency + manual sync. * * Two endpoints: * diff --git a/tests/e2e/specs/tier-b/sources-list.spec.ts b/tests/e2e/specs/tier-b/sources-list.spec.ts index 215467cc..c6449621 100644 --- a/tests/e2e/specs/tier-b/sources-list.spec.ts +++ b/tests/e2e/specs/tier-b/sources-list.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 3 Tier-B · sources-list (B2) — list / paginate / search / delete / + * Tier-B · sources-list (B2) — list / paginate / search / delete / * directory-structure. * * Exercises the read-side endpoints the UI calls from diff --git a/tests/e2e/specs/tier-b/streaming-ux.spec.ts b/tests/e2e/specs/tier-b/streaming-ux.spec.ts index 37ed39ae..2c970d27 100644 --- a/tests/e2e/specs/tier-b/streaming-ux.spec.ts +++ b/tests/e2e/specs/tier-b/streaming-ux.spec.ts @@ -1,6 +1,6 @@ -// Phase 3 Tier-B · streaming UX (SSE delivery, research progress, tool approval resume) +// Tier-B · streaming UX (SSE delivery, research progress, tool approval resume) /** - * Phase 3 — B7 · streaming UX + tool-state plumbing. + * B7 · streaming UX + tool-state plumbing. * * Focus: the `/stream` SSE surface that `frontend/src/conversation/Conversation.tsx` * consumes. Unlike the migration-critical chat-turn spec (P2-07), these tests @@ -103,11 +103,19 @@ test.describe('tier-b · streaming UX', () => { isNoneDoc: true, }); expect(status).toBe(200); - // Every non-empty line in the body must be `data: ...` framed. + // Every non-empty line in the body must be a valid SSE field — + // ``data:`` for payload, ``id:`` for the per-chunk replay cursor + // added by the snapshot-tail branch, or ``event:`` for typed + // events. Anything else (e.g. a raw JSON line) is a wire-format + // bug. for (const line of text.split('\n')) { const trimmed = line.trim(); if (!trimmed) continue; - expect(trimmed.startsWith('data:')).toBe(true); + expect( + trimmed.startsWith('data:') || + trimmed.startsWith('id:') || + trimmed.startsWith('event:'), + ).toBe(true); } // At least one answer delta landed and the stream ended cleanly. expect(frames.length).toBeGreaterThan(1); @@ -160,14 +168,20 @@ test.describe('tier-b · streaming UX', () => { } }); - test('pending_tool_state round-trip: SQL-inserted row is cleared by a tool_actions resume /stream', async ({ + test('pending_tool_state round-trip: SQL-inserted row is claimed (resuming) or cleared by a tool_actions resume /stream', async ({ browser, }) => { // We model the tool-approval flow at the table level — the contract is // that the route path with `tool_actions` + `conversation_id` calls - // `StreamProcessor.resume_from_tool_actions`, which loads from PG and - // DELETEs the row after success (stream_processor.py:978-979). No real - // agent wiring is needed to test that contract. + // `StreamProcessor.resume_from_tool_actions`, which loads from PG, then + // `mark_resuming` flips status `pending`→`resuming` so the row cannot + // be re-claimed (see continuation_service.py + repositories/pending_tool_state.py). + // On clean completion the route layer DELETEs the row + // (routes/base.py: `delete_state` after stream success); on a crash the + // row stays in `resuming` for the janitor to revert after the grace + // window. Either outcome proves the replay footgun is closed — what + // must not happen is the row remaining in `pending` status. + // No real agent wiring is needed to test that contract. const { sub, token } = await newUserContext(browser); const api = await authedRequest(playwright, token); try { @@ -248,9 +262,9 @@ test.describe('tier-b · streaming UX', () => { }), ).toBe(1); - // 3. Fire the resume request. Even if the agent crashes partway, the - // state must be deleted — the route layer guarantees that by calling - // delete_state *before* invoking gen_continuation. + // 3. Fire the resume request. The route layer claims the row via + // `mark_resuming` before invoking gen_continuation, so even if the + // agent crashes mid-stream the row's status flips off `pending`. const resume = await api.post('/stream', { data: { conversation_id: convId, @@ -264,15 +278,15 @@ test.describe('tier-b · streaming UX', () => { expect(resume.status()).toBe(200); await resume.text(); - // 4. Row must be gone. Proves delete_state ran during the resume - // code path. If this fails, the resume is replayable — a footgun - // that was a Mongo-era bug #2088. - expect( - await countRows('pending_tool_state', { - sql: 'user_id = $1 AND conversation_id = CAST($2 AS uuid)', - params: [sub, convId], - }), - ).toBe(0); + // 4. Row must be either deleted (clean completion) or marked + // `resuming` (crash partway). What must not happen is the row + // remaining in `pending` — that would mean the resume is + // re-fireable, a footgun mirroring Mongo-era bug #2088. + const pendingCount = await countRows('pending_tool_state', { + sql: "user_id = $1 AND conversation_id = CAST($2 AS uuid) AND status = 'pending'", + params: [sub, convId], + }); + expect(pendingCount).toBe(0); } finally { await api.dispose(); } diff --git a/tests/e2e/specs/tier-b/templates.spec.ts b/tests/e2e/specs/tier-b/templates.spec.ts index 84d1f708..783d36ae 100644 --- a/tests/e2e/specs/tier-b/templates.spec.ts +++ b/tests/e2e/specs/tier-b/templates.spec.ts @@ -1,12 +1,13 @@ /** - * Phase 3 Tier-B · agent template gallery + adopt + * Tier-B · agent template gallery + adopt * * Covers B12: `GET /api/template_agents` lists rows owned by * `user_id = '__system__'`, and `POST /api/adopt_agent?id=…` creates a * user-owned copy with the same shape but a fresh `id`/`user_id`/`key`. * - * The e2e DB starts with zero template rows — Phase 0 only applies the - * schema, not template seeding. We insert a `__system__`-owned row via + * The e2e DB starts with zero template rows — the bootstrap only + * applies the schema, not template seeding. We insert a + * `__system__`-owned row via * direct SQL to simulate a seeded template. This matches the brief's * explicit guidance ("if empty, INSERT a fake __system__ template agent * via SQL for the test") and avoids coupling to a future seed script diff --git a/tests/e2e/specs/tier-b/upload.spec.ts b/tests/e2e/specs/tier-b/upload.spec.ts index c47e7f42..6c9a9ff6 100644 --- a/tests/e2e/specs/tier-b/upload.spec.ts +++ b/tests/e2e/specs/tier-b/upload.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 3 Tier-B · upload (B1) — core upload happy paths. + * Tier-B · upload (B1) — core upload happy paths. * * Exercises the two ingest entry points the UI drives from * `frontend/src/upload/Upload.tsx`: diff --git a/tests/e2e/specs/tier-b/v1-completions.spec.ts b/tests/e2e/specs/tier-b/v1-completions.spec.ts index 97e750e3..d14471c2 100644 --- a/tests/e2e/specs/tier-b/v1-completions.spec.ts +++ b/tests/e2e/specs/tier-b/v1-completions.spec.ts @@ -1,6 +1,6 @@ -// Phase 3 Tier-B · /v1/chat/completions + /v1/models (OpenAI-compatible) +// Tier-B · /v1/chat/completions + /v1/models (OpenAI-compatible) /** - * Phase 3 — B9 · OpenAI-compatible endpoints. + * B9 · OpenAI-compatible endpoints. * * `application/api/v1/routes.py` exposes `/v1/chat/completions` (POST) * and `/v1/models` (GET). Both read an agent API key from diff --git a/tests/e2e/specs/tier-b/webhook.spec.ts b/tests/e2e/specs/tier-b/webhook.spec.ts index e0a020e6..2c5fb5ea 100644 --- a/tests/e2e/specs/tier-b/webhook.spec.ts +++ b/tests/e2e/specs/tier-b/webhook.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 3 Tier-B · agent incoming webhook + * Tier-B · agent incoming webhook * * Covers B11: minting of `agents.incoming_webhook_token` via * `GET /api/agent_webhook?id=…`, POSTing the resolved URL triggers a diff --git a/tests/e2e/specs/tier-b/workflow-builder.spec.ts b/tests/e2e/specs/tier-b/workflow-builder.spec.ts index 3cc3784a..28134cce 100644 --- a/tests/e2e/specs/tier-b/workflow-builder.spec.ts +++ b/tests/e2e/specs/tier-b/workflow-builder.spec.ts @@ -1,5 +1,5 @@ /** - * Phase 3 Tier-B · workflow builder (nodes / edges / versioning) + * Tier-B · workflow builder (nodes / edges / versioning) * * Covers B13: `POST /api/workflows`, `GET /api/workflows/:id`, * `PUT /api/workflows/:id`, `DELETE /api/workflows/:id`, and the diff --git a/tests/e2e/specs/tier-c/ui-smoke.spec.ts b/tests/e2e/specs/tier-c/ui-smoke.spec.ts index 0fa1652a..8418a0c9 100644 --- a/tests/e2e/specs/tier-c/ui-smoke.spec.ts +++ b/tests/e2e/specs/tier-c/ui-smoke.spec.ts @@ -1,7 +1,7 @@ import * as playwright from '@playwright/test'; -// Phase 4 Tier-C · UI smoke (surface coverage, no DB assertions) +// Tier-C · UI smoke (surface coverage, no DB assertions) /** - * Phase 4 — P4 · Tier-C UI smoke. + * P4 · Tier-C UI smoke. * * Cheap surface coverage for UI-only behaviors. One combined spec, many * small tests. Scope: @@ -292,8 +292,10 @@ test.describe('tier-c · UI smoke', () => { const newChat = page.getByRole('link', { name: /new chat/i }).first(); await expect(newChat).toBeVisible(); await newChat.click(); - // Clicking "New Chat" must not navigate us off the root route. - await expect(page).toHaveURL(/127\.0\.0\.1:5179\/?$/); + // Clicking "New Chat" lands on the empty-conversation sentinel + // route. Root or /c/new are both acceptable — the SSE branch made + // /c/new the canonical fresh-chat URL. + await expect(page).toHaveURL(/127\.0\.0\.1:5179\/(c\/new)?$/); } finally { await context.close(); } @@ -305,7 +307,7 @@ test.describe('tier-c · UI smoke', () => { // them set (see scripts/e2e/env.sh), so App.tsx's `` is // never rendered and there is nothing observable to assert. // - // If/when we add a prod-bundle opt-in (Phase 5-C), revive this test with + // If/when we add a prod-bundle opt-in, revive this test with // those env vars set and check the banner + localStorage-dismiss flow. }); diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b1a04957..b12202d5 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -76,9 +76,11 @@ def pg_clean_users(pg_conn): The outer transaction rollback handles cleanup, but if a previous interrupted run left rows committed, this fixture removes them inside the transaction scope so they are invisible to the test. - ``DELETE`` rather than ``TRUNCATE`` because ``TRUNCATE`` in Postgres - cannot be rolled back within a nested transaction the way - ``DELETE`` can. + ``TRUNCATE CASCADE`` cascades through every FK to ``users`` (18 + tables and counting — agents, conversations, sources, user_logs, + etc.) so the test view is clean regardless of which dependent rows + a prior run left behind. PostgreSQL ``TRUNCATE`` is transactional, + so the outer rollback still restores everything. """ - pg_conn.execute(text("DELETE FROM users")) + pg_conn.execute(text("TRUNCATE TABLE users CASCADE")) return pg_conn diff --git a/tests/integration/test_mcp.py b/tests/integration/test_mcp.py index 2feaed41..95e5eef0 100644 --- a/tests/integration/test_mcp.py +++ b/tests/integration/test_mcp.py @@ -4,10 +4,13 @@ Integration tests for DocsGPT MCP (Model Context Protocol) server endpoints. Endpoints tested: - /api/mcp_server/callback (GET) - OAuth callback -- /api/mcp_server/oauth_status/{task_id} (GET) - OAuth status - /api/mcp_server/save (POST) - Save MCP server config - /api/mcp_server/test (POST) - Test MCP server connection +OAuth status (previously polled via /api/mcp_server/oauth_status/) +is now delivered exclusively through the per-user SSE pipe at +/api/events; see docs/runbooks/sse-notifications.md. + Usage: python tests/integration/test_mcp.py python tests/integration/test_mcp.py --base-url http://localhost:7091 @@ -62,75 +65,6 @@ class MCPTests(DocsGPTTestBase): self.record_result(test_name, False, str(e)) return False - # ------------------------------------------------------------------------- - # OAuth Status Tests - # ------------------------------------------------------------------------- - - def test_mcp_oauth_status(self) -> bool: - """Test getting MCP OAuth status.""" - test_name = "MCP OAuth status" - self.print_header(test_name) - - if not self.require_auth(test_name): - return True - - try: - response = self.get( - "/api/mcp_server/oauth_status/test-task-id-123", - timeout=10, - ) - - if response.status_code in [200, 404]: - self.print_success(f"OAuth status check: {response.status_code}") - if response.status_code == 200: - result = response.json() - self.print_info(f"Status: {result.get('status', 'N/A')}") - self.record_result(test_name, True, f"Status: {response.status_code}") - return True - - self.print_error(f"Unexpected status: {response.status_code}") - self.record_result(test_name, False, f"Status: {response.status_code}") - return False - - except Exception as e: - self.print_error(f"Exception: {e}") - self.record_result(test_name, False, str(e)) - return False - - def test_mcp_oauth_status_invalid_task(self) -> bool: - """Test OAuth status for invalid task ID.""" - test_name = "MCP OAuth status invalid" - self.print_header(test_name) - - if not self.require_auth(test_name): - return True - - try: - response = self.get( - "/api/mcp_server/oauth_status/nonexistent-task-xyz", - timeout=10, - ) - - if response.status_code in [404, 400]: - self.print_success(f"Correctly returned: {response.status_code}") - self.record_result(test_name, True, "Invalid task handled") - return True - elif response.status_code == 200: - result = response.json() - if result.get("status") in ["not_found", "unknown", None]: - self.print_success("Invalid task handled (status: not_found)") - self.record_result(test_name, True, "Invalid task handled") - return True - - self.print_warning(f"Status: {response.status_code}") - self.record_result(test_name, True, f"Status: {response.status_code}") - return True - - except Exception as e: - self.print_error(f"Exception: {e}") - self.record_result(test_name, False, str(e)) - return False - # ------------------------------------------------------------------------- # Save Tests # ------------------------------------------------------------------------- @@ -311,10 +245,6 @@ class MCPTests(DocsGPTTestBase): # Callback tests self.test_mcp_callback() - # OAuth status tests - self.test_mcp_oauth_status() - self.test_mcp_oauth_status_invalid_task() - # Save tests self.test_mcp_save() self.test_mcp_save_invalid() diff --git a/tests/integration/test_v1_api.py b/tests/integration/test_v1_api.py index 1561d183..c656cfb7 100644 --- a/tests/integration/test_v1_api.py +++ b/tests/integration/test_v1_api.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Integration tests for the /v1/ chat completions API (Phase 4). +Integration tests for the /v1/ chat completions API. Endpoints tested: - /v1/chat/completions (POST) - Standard chat completions (streaming & non-streaming) diff --git a/tests/storage/db/repositories/test_conversations.py b/tests/storage/db/repositories/test_conversations.py index 1a31051e..7f5dadbd 100644 --- a/tests/storage/db/repositories/test_conversations.py +++ b/tests/storage/db/repositories/test_conversations.py @@ -5,7 +5,10 @@ from __future__ import annotations from datetime import datetime, timezone -from application.storage.db.repositories.conversations import ConversationsRepository +from application.storage.db.repositories.conversations import ( + ConversationsRepository, + MessageUpdateOutcome, +) def _repo(conn) -> ConversationsRepository: @@ -478,10 +481,10 @@ class TestReserveAndFinalizeMessage: msg = repo.reserve_message( conv["id"], prompt="q", placeholder_response="ph", ) - ok = repo.update_message_by_id( + outcome = repo.update_message_by_id( msg["id"], {"response": "real answer", "status": "complete"}, ) - assert ok is True + assert outcome is MessageUpdateOutcome.UPDATED refreshed = repo.get_message_at(conv["id"], 0) assert refreshed["response"] == "real answer" assert refreshed["status"] == "complete" @@ -502,7 +505,61 @@ class TestReserveAndFinalizeMessage: def test_update_message_by_id_rejects_non_uuid(self, pg_conn): repo = _repo(pg_conn) - assert repo.update_message_by_id("not-a-uuid", {"status": "complete"}) is False + assert ( + repo.update_message_by_id("not-a-uuid", {"status": "complete"}) + is MessageUpdateOutcome.INVALID + ) + + def test_update_message_by_id_distinguishes_already_complete(self, pg_conn): + """When the row is already ``complete``, a subsequent + ``only_if_non_terminal=True`` finalize must report + ``ALREADY_COMPLETE`` — not ``UPDATED`` and not the generic + not-found case. The SSE abort handler relies on this to + journal ``end`` instead of a spurious ``error`` when the + normal-path finalize wins the race against a client + disconnect. + """ + repo = _repo(pg_conn) + conv = repo.create("user-r", "c") + msg = repo.reserve_message( + conv["id"], prompt="q", placeholder_response="ph", + ) + first = repo.update_message_by_id( + msg["id"], {"response": "ok", "status": "complete"}, + only_if_non_terminal=True, + ) + assert first is MessageUpdateOutcome.UPDATED + second = repo.update_message_by_id( + msg["id"], {"response": "ok again", "status": "complete"}, + only_if_non_terminal=True, + ) + assert second is MessageUpdateOutcome.ALREADY_COMPLETE + # The second attempt must NOT have overwritten anything. + refreshed = repo.get_message_at(conv["id"], 0) + assert refreshed["response"] == "ok" + + def test_update_message_by_id_distinguishes_already_failed(self, pg_conn): + repo = _repo(pg_conn) + conv = repo.create("user-r", "c") + msg = repo.reserve_message( + conv["id"], prompt="q", placeholder_response="ph", + ) + assert repo.update_message_by_id( + msg["id"], {"response": "boom", "status": "failed"}, + only_if_non_terminal=True, + ) is MessageUpdateOutcome.UPDATED + assert repo.update_message_by_id( + msg["id"], {"response": "late", "status": "complete"}, + only_if_non_terminal=True, + ) is MessageUpdateOutcome.ALREADY_FAILED + + def test_update_message_by_id_unknown_uuid_is_not_found(self, pg_conn): + repo = _repo(pg_conn) + assert repo.update_message_by_id( + "00000000-0000-0000-0000-000000000000", + {"status": "complete"}, + only_if_non_terminal=True, + ) is MessageUpdateOutcome.NOT_FOUND def test_update_message_status_only(self, pg_conn): repo = _repo(pg_conn) @@ -557,9 +614,9 @@ class TestReserveAndFinalizeMessage: class TestConcurrentAppend: """Two threads appending to the same conversation must not race on - ``position``. The plan (migration-postgres.md §Phase 3) explicitly - calls this out as the single trickiest invariant, so we exercise it - directly with two parallel connections.""" + ``position``. The migration plan explicitly calls this out as the + single trickiest invariant, so we exercise it directly with two + parallel connections.""" def test_concurrent_appends_get_distinct_positions(self, pg_engine, pg_conn): import threading diff --git a/tests/storage/db/repositories/test_message_events_repository.py b/tests/storage/db/repositories/test_message_events_repository.py new file mode 100644 index 00000000..0dcbef1f --- /dev/null +++ b/tests/storage/db/repositories/test_message_events_repository.py @@ -0,0 +1,301 @@ +"""Round-trip tests for ``MessageEventsRepository`` against a real +ephemeral Postgres (via the project's ``pg_engine`` fixture). + +Coverage: insert ordering, read_after cursor semantics, the composite +PK rejecting duplicate (message_id, sequence_no), cascade on parent +delete. +""" + +from __future__ import annotations + +import uuid + +import pytest +from sqlalchemy import text +from sqlalchemy.exc import IntegrityError + +from application.storage.db.repositories.message_events import ( + MessageEventsRepository, +) + + +def _seed_message(conn) -> str: + """Create a parent conversation + message and return the message_id.""" + user_id = f"user-{uuid.uuid4().hex[:8]}" + conv_id = uuid.uuid4() + msg_id = uuid.uuid4() + conn.execute( + text("INSERT INTO users (user_id) VALUES (:u)"), + {"u": user_id}, + ) + conn.execute( + text( + "INSERT INTO conversations (id, user_id, name) " + "VALUES (:id, :u, 'test')" + ), + {"id": conv_id, "u": user_id}, + ) + conn.execute( + text( + "INSERT INTO conversation_messages (id, conversation_id, user_id, position) " + "VALUES (:id, :c, :u, 0)" + ), + {"id": msg_id, "c": conv_id, "u": user_id}, + ) + return str(msg_id) + + +@pytest.mark.integration +class TestMessageEventsRepository: + def test_record_and_read_after_in_order(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + + repo.record(message_id, 0, "answer", {"chunk": "Hello"}) + repo.record(message_id, 1, "answer", {"chunk": " world"}) + repo.record(message_id, 2, "end", {}) + + rows = list(repo.read_after(message_id)) + assert [r["sequence_no"] for r in rows] == [0, 1, 2] + assert rows[0]["event_type"] == "answer" + assert rows[0]["payload"] == {"chunk": "Hello"} + assert rows[2]["event_type"] == "end" + + def test_read_after_filters_by_cursor(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + + for n in range(5): + repo.record(message_id, n, "answer", {"chunk": f"#{n}"}) + + # Cursor at 1 returns sequence_no in [2, 3, 4]. + rows = list(repo.read_after(message_id, last_sequence_no=1)) + assert [r["sequence_no"] for r in rows] == [2, 3, 4] + + # Cursor None replays the whole backlog. + rows = list(repo.read_after(message_id, last_sequence_no=None)) + assert [r["sequence_no"] for r in rows] == [0, 1, 2, 3, 4] + + # Cursor past the end returns empty. + rows = list(repo.read_after(message_id, last_sequence_no=99)) + assert rows == [] + + def test_duplicate_sequence_no_raises(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + + repo.record(message_id, 0, "answer", {"chunk": "first"}) + with pytest.raises(IntegrityError): + repo.record(message_id, 0, "answer", {"chunk": "second"}) + + def test_latest_sequence_no(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + # Empty journal returns None. + assert repo.latest_sequence_no(message_id) is None + repo.record(message_id, 0, "answer", {"chunk": "a"}) + repo.record(message_id, 7, "answer", {"chunk": "b"}) + assert repo.latest_sequence_no(message_id) == 7 + + def test_reconstruct_partial_concats_answer_and_thought(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + repo.record(message_id, 0, "message_id", {"type": "message_id"}) + repo.record(message_id, 1, "answer", {"type": "answer", "answer": "Hello"}) + repo.record(message_id, 2, "thought", {"type": "thought", "thought": "Hmm "}) + repo.record(message_id, 3, "answer", {"type": "answer", "answer": ", world"}) + repo.record(message_id, 4, "thought", {"type": "thought", "thought": "OK."}) + + partial = repo.reconstruct_partial(message_id) + assert partial["response"] == "Hello, world" + assert partial["thought"] == "Hmm OK." + assert partial["sources"] == [] + assert partial["tool_calls"] == [] + + def test_reconstruct_partial_last_wins_for_source_and_tool_calls(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + repo.record(message_id, 0, "source", {"type": "source", "source": [{"id": "a"}]}) + repo.record(message_id, 1, "tool_calls", {"type": "tool_calls", "tool_calls": [{"name": "x"}]}) + repo.record(message_id, 2, "source", {"type": "source", "source": [{"id": "a"}, {"id": "b"}]}) + repo.record(message_id, 3, "tool_calls", {"type": "tool_calls", "tool_calls": []}) + + partial = repo.reconstruct_partial(message_id) + assert partial["sources"] == [{"id": "a"}, {"id": "b"}] + assert partial["tool_calls"] == [] + + def test_reconstruct_partial_empty_journal(self, pg_conn): + message_id = _seed_message(pg_conn) + partial = MessageEventsRepository(pg_conn).reconstruct_partial(message_id) + assert partial == { + "response": "", + "thought": "", + "sources": [], + "tool_calls": [], + } + + def test_reconstruct_partial_skips_malformed_payload(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + repo.record(message_id, 0, "answer", {"type": "answer", "answer": "good"}) + # Non-string answer field — must be ignored, not crash. + repo.record(message_id, 1, "answer", {"type": "answer", "answer": 123}) + repo.record(message_id, 2, "answer", {"type": "answer", "answer": " more"}) + + partial = repo.reconstruct_partial(message_id) + assert partial["response"] == "good more" + + def test_cascade_on_message_delete(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + + repo.record(message_id, 0, "answer", {"chunk": "x"}) + pg_conn.execute( + text("DELETE FROM conversation_messages WHERE id = CAST(:id AS uuid)"), + {"id": message_id}, + ) + assert list(repo.read_after(message_id)) == [] + + def test_payload_default_empty_jsonb(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + repo.record(message_id, 0, "end") # no payload + rows = list(repo.read_after(message_id)) + assert rows[0]["payload"] == {} + + def test_record_with_list_payload_preserves_shape(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + repo.record(message_id, 0, "sources", [{"url": "a"}, {"url": "b"}]) + rows = repo.read_after(message_id) + assert rows[0]["payload"] == [{"url": "a"}, {"url": "b"}] + + def test_record_empty_event_type_raises(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + with pytest.raises(ValueError): + repo.record(message_id, 0, "") + + def test_record_failure_isolated_via_savepoint(self, pg_conn): + """The shared-connection caller must use ``begin_nested`` so a + single integrity error doesn't poison the whole transaction. + """ + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + + repo.record(message_id, 0, "answer", {"chunk": "first"}) + # Duplicate sequence_no — must not abort the outer transaction + # when wrapped in a SAVEPOINT. + try: + with pg_conn.begin_nested(): + repo.record(message_id, 0, "answer", {"chunk": "duplicate"}) + except IntegrityError: + pass + # Subsequent record on the same outer transaction still works. + repo.record(message_id, 1, "answer", {"chunk": "after"}) + rows = repo.read_after(message_id) + assert [r["sequence_no"] for r in rows] == [0, 1] + + def test_cleanup_older_than_deletes_only_aged_rows(self, pg_conn): + message_id = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + + repo.record(message_id, 0, "answer", {"chunk": "stale"}) + repo.record(message_id, 1, "answer", {"chunk": "fresh"}) + # Backdate row 0 past the retention window so the janitor catches + # it; row 1 stays at "now" and must survive. + pg_conn.execute( + text( + "UPDATE message_events SET created_at = now() - interval '20 days' " + "WHERE message_id = CAST(:id AS uuid) AND sequence_no = 0" + ), + {"id": message_id}, + ) + + deleted = repo.cleanup_older_than(ttl_days=14) + assert deleted == 1 + rows = list(repo.read_after(message_id)) + assert [r["sequence_no"] for r in rows] == [1] + + def test_cleanup_older_than_rejects_non_positive(self, pg_conn): + repo = MessageEventsRepository(pg_conn) + with pytest.raises(ValueError): + repo.cleanup_older_than(ttl_days=0) + with pytest.raises(ValueError): + repo.cleanup_older_than(ttl_days=-1) + + def test_message_id_isolation(self, pg_conn): + m1 = _seed_message(pg_conn) + m2 = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + + repo.record(m1, 0, "answer", {"chunk": "msg1"}) + repo.record(m2, 0, "answer", {"chunk": "msg2"}) + + rows1 = list(repo.read_after(m1)) + rows2 = list(repo.read_after(m2)) + assert len(rows1) == 1 and rows1[0]["payload"] == {"chunk": "msg1"} + assert len(rows2) == 1 and rows2[0]["payload"] == {"chunk": "msg2"} + + def test_bulk_record_inserts_all_rows_in_order(self, pg_conn): + """``bulk_record`` is the hot-path INSERT used by + ``BatchedJournalWriter``. All rows for one ``message_id`` land + in one statement; ``read_after`` returns them in seq order. + """ + msg = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + + events = [ + (0, "answer", {"chunk": "first"}), + (1, "answer", {"chunk": "second"}), + (2, "answer", {"chunk": "third"}), + (3, "end", {"type": "end"}), + ] + repo.bulk_record(msg, events) + + rows = list(repo.read_after(msg)) + assert [r["sequence_no"] for r in rows] == [0, 1, 2, 3] + assert [r["event_type"] for r in rows] == ["answer", "answer", "answer", "end"] + assert rows[0]["payload"] == {"chunk": "first"} + assert rows[3]["payload"] == {"type": "end"} + + def test_bulk_record_empty_list_is_no_op(self, pg_conn): + msg = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + repo.bulk_record(msg, []) + assert list(repo.read_after(msg)) == [] + + def test_bulk_record_collision_aborts_entire_batch(self, pg_conn): + """All-or-nothing contract: if any row in the batch collides + on the composite PK, Postgres aborts the whole INSERT. Caller + (``BatchedJournalWriter``) handles this by opening its own + short-lived ``db_session`` per flush, so the surrounding + transaction is isolated from the abort. Mirror that here by + wrapping the failing call in ``begin_nested`` (savepoint) so + the outer test transaction can keep running. + """ + from sqlalchemy.exc import IntegrityError + + msg = _seed_message(pg_conn) + repo = MessageEventsRepository(pg_conn) + # Seed a row at seq=1 so the bulk batch will collide on it. + repo.record(msg, 1, "answer", {"prev": True}) + + events = [ + (0, "answer", {"chunk": "a"}), + (1, "answer", {"chunk": "b"}), # collides with seeded row + (2, "answer", {"chunk": "c"}), + ] + try: + with pg_conn.begin_nested(): + repo.bulk_record(msg, events) + except IntegrityError: + pass + else: + pytest.fail("expected IntegrityError on duplicate seq") + + # The bulk INSERT aborted — none of the three rows landed. + # Only the original seed row remains. + rows = list(repo.read_after(msg)) + assert [r["sequence_no"] for r in rows] == [1] + assert rows[0]["payload"] == {"prev": True} diff --git a/tests/test_cache.py b/tests/test_cache.py index 411db4fe..74b4ae9c 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -157,7 +157,9 @@ class TestGetRedisInstance: assert result is mock_instance mock_from_url.assert_called_once_with( - "redis://localhost:6379/0", socket_connect_timeout=2 + "redis://localhost:6379/0", + socket_connect_timeout=2, + health_check_interval=10, ) @patch("application.cache.redis.Redis.from_url") diff --git a/tests/test_client_tools.py b/tests/test_client_tools.py index 10f30768..8778d9aa 100644 --- a/tests/test_client_tools.py +++ b/tests/test_client_tools.py @@ -1,4 +1,4 @@ -"""Tests for client-side tools (Phase 2). +"""Tests for client-side tools. Covers merge_client_tools, prepare_tools_for_llm with client tools, check_pause for client-side tools, and the full flow through the handler. diff --git a/tests/test_continuation.py b/tests/test_continuation.py index 0f3e19ea..e48a1cc0 100644 --- a/tests/test_continuation.py +++ b/tests/test_continuation.py @@ -1,4 +1,4 @@ -"""Tests for the continuation infrastructure (Phase 1). +"""Tests for the continuation infrastructure. Covers ContinuationService, ToolExecutor.check_pause, handler pause signaling, BaseAgent.gen_continuation, and request validation. diff --git a/tests/test_event_replay.py b/tests/test_event_replay.py new file mode 100644 index 00000000..db00a7b2 --- /dev/null +++ b/tests/test_event_replay.py @@ -0,0 +1,899 @@ +"""Unit tests for ``application/streaming/event_replay.py``. + +The replay generator is the snapshot+tail boundary the chat-stream +reconnect path lives on. The boundary correctness invariants worth +locking down: + +- Snapshot replay yields rows in ``sequence_no`` order with the SSE + ``id:`` header set to that sequence_no. +- Live tail dedupes pub/sub messages whose ``sequence_no`` is already + covered by the snapshot. +- A backlog read failure inside ``on_subscribe`` doesn't wedge the + generator — it surfaces a terminal ``error`` event (``code: + snapshot_failed``) and returns, freeing the WSGI thread instead of + pinning it on keepalives the client can't hear as terminal. +- Keepalive comments fire after the configured silence window. +- ``encode_pubsub_message`` round-trips with ``_decode_pubsub_message``. +""" + +from __future__ import annotations + +import json +from typing import Iterator +from unittest.mock import MagicMock, patch + +import pytest + +from application.streaming.event_replay import ( + _SSE_LINE_SPLIT_PATTERN, + _decode_pubsub_message, + build_message_event_stream, + encode_pubsub_message, + format_sse_event, +) + + +# ── format_sse_event ──────────────────────────────────────────────────── + + +@pytest.mark.unit +class TestFormatSseEvent: + def test_simple_payload(self): + out = format_sse_event({"type": "answer", "answer": "Hi"}, sequence_no=3) + # Body is one JSON line; SSE record terminated with blank line. + lines = out.rstrip("\n").split("\n") + assert lines[0] == "id: 3" + assert lines[1].startswith("data: ") + body = lines[1][len("data: "):] + assert json.loads(body) == {"type": "answer", "answer": "Hi"} + assert out.endswith("\n\n") + + def test_negative_sequence_for_synthetic_events(self): + out = format_sse_event( + {"type": "error", "code": "snapshot_failed"}, sequence_no=-1 + ) + assert "id: -1" in out + + def test_payload_with_embedded_newlines_splits_into_multiple_data_lines(self): + # A pathological payload (shouldn't happen for json.dumps output, + # but defensive against future callers). + out = format_sse_event({"type": "x", "text": "a\nb"}, sequence_no=0) + # The JSON itself escapes newlines, so this still produces one + # data line — the split logic kicks in only when the encoded + # body has literal newlines. + assert out.count("data: ") == 1 + + +# ── pub/sub encode/decode ─────────────────────────────────────────────── + + +@pytest.mark.unit +class TestPubsubEnvelope: + def test_encode_decode_roundtrip(self): + wire = encode_pubsub_message( + "msg-1", 7, "answer", {"type": "answer", "answer": "ok"} + ) + envelope = _decode_pubsub_message(wire.encode("utf-8")) + assert envelope == { + "message_id": "msg-1", + "sequence_no": 7, + "event_type": "answer", + "payload": {"type": "answer", "answer": "ok"}, + } + + def test_decode_returns_none_on_garbage(self): + assert _decode_pubsub_message(b"not json") is None + assert _decode_pubsub_message(b'"top-level-string"') is None + assert _decode_pubsub_message(b"null") is None + + def test_decode_accepts_str_input(self): + envelope = _decode_pubsub_message('{"sequence_no": 1, "payload": {}}') + assert envelope == {"sequence_no": 1, "payload": {}} + + +# ── build_message_event_stream ────────────────────────────────────────── + + +def _fake_topic_subscribe(messages: list, *, fire_callback: bool = True): + """Build a ``Topic.subscribe`` mock that fires ``on_subscribe`` then + yields the supplied bytes in order, then yields ``None`` ticks + indefinitely so the generator can keep running until the test + closes it. + """ + + def _impl(self, on_subscribe=None, poll_timeout=1.0): + if fire_callback and on_subscribe is not None: + on_subscribe() + for m in messages: + yield m + while True: + yield None + + return _impl + + +def _drain(gen: Iterator[str], *, max_items: int = 50) -> list[str]: + out = [] + try: + for _ in range(max_items): + out.append(next(gen)) + except StopIteration: + pass + finally: + gen.close() + return out + + +@pytest.mark.unit +class TestBuildMessageEventStream: + def test_yields_connected_prelude_first(self): + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = [] + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + first = next(gen) + gen.close() + assert first == ": connected\n\n" + + def test_snapshot_replays_in_sequence_order(self): + rows = [ + { + "sequence_no": 0, + "event_type": "answer", + "payload": {"type": "answer", "answer": "A"}, + }, + { + "sequence_no": 1, + "event_type": "answer", + "payload": {"type": "answer", "answer": "B"}, + }, + ] + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = rows + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = _drain(gen, max_items=4) + + # Expect: prelude, two snapshot frames, then keepalive ticks (None + # not yielded as a string — generator yields keepalive comments + # instead). With poll_timeout=1.0 and short test window we may + # only see the prelude + snapshot before close. + assert out[0] == ": connected\n\n" + assert "id: 0" in out[1] + assert "id: 1" in out[2] + # The repo was queried with the right cursor. + mock_repo_cls.return_value.read_after.assert_called_once_with( + "msg-1", last_sequence_no=None + ) + + def test_live_tail_dedupes_against_snapshot(self): + snapshot_rows = [ + { + "sequence_no": 5, + "event_type": "answer", + "payload": {"type": "answer", "answer": "snapshot"}, + }, + ] + live_envelope = encode_pubsub_message( + "msg-1", 5, "answer", {"type": "answer", "answer": "duplicate"} + ).encode("utf-8") + live_new_envelope = encode_pubsub_message( + "msg-1", 6, "answer", {"type": "answer", "answer": "fresh"} + ).encode("utf-8") + + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([live_envelope, live_new_envelope]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = snapshot_rows + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = _drain(gen, max_items=4) + + assert out[0] == ": connected\n\n" + assert "id: 5" in out[1] + assert '"answer": "snapshot"' in out[1] + # The duplicate live event (seq=5) is dropped. + assert "id: 6" in out[2] + assert '"answer": "fresh"' in out[2] + # No third frame past the fresh one (only None ticks). + + def test_live_tail_passes_through_when_seq_strictly_greater_than_replay(self): + """No snapshot rows; every live event is fresh and yielded.""" + live = encode_pubsub_message( + "msg-1", 0, "answer", {"type": "answer", "answer": "x"} + ).encode("utf-8") + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([live]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = [] + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = _drain(gen, max_items=3) + + assert out[0] == ": connected\n\n" + assert "id: 0" in out[1] + assert '"answer": "x"' in out[1] + + def test_snapshot_read_failure_surfaces_synthetic_event(self): + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.side_effect = RuntimeError("boom") + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + # Ask for more than the expected output so we'd notice a + # regression where the generator keeps emitting keepalives. + out = _drain(gen, max_items=10) + + assert out[0] == ": connected\n\n" + # Terminal ``error`` event with the snapshot-specific ``code`` so + # the frontend's existing end/error contract closes the stream. + assert '"type": "error"' in out[1] + assert '"code": "snapshot_failed"' in out[1] + assert "id: -1" in out[1] + # Generator must return after the synthetic — otherwise the + # client hangs on keepalives waiting for a terminal event that + # would never arrive. + assert len(out) == 2 + + def test_malformed_pubsub_message_dropped_silently(self): + bad = b"not-json" + good = encode_pubsub_message( + "msg-1", 0, "answer", {"type": "answer", "answer": "ok"} + ).encode("utf-8") + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([bad, good]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = [] + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = _drain(gen, max_items=3) + + assert out[0] == ": connected\n\n" + # Bad message dropped; good one yielded. + assert "id: 0" in out[1] + assert '"answer": "ok"' in out[1] + + def test_pubsub_envelope_with_non_int_sequence_dropped(self): + bad = json.dumps( + {"sequence_no": "not-int", "payload": {"type": "x"}} + ).encode("utf-8") + good = encode_pubsub_message( + "msg-1", 0, "answer", {"type": "answer", "answer": "ok"} + ).encode("utf-8") + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([bad, good]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = [] + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = _drain(gen, max_items=3) + + assert out[0] == ": connected\n\n" + assert "id: 0" in out[1] + + +@pytest.mark.unit +class TestDedupFloorSeededFromCursor: + """Regressions for the dedup-floor bugs the round-1 review flagged. + + With ``max_replayed_seq`` initialised to ``last_event_id``, an + empty snapshot still rejects republished live events the client + has already seen. Advancing on yield protects against republish + past the snapshot ceiling. + """ + + def test_empty_snapshot_dedups_against_last_event_id(self): + # No snapshot rows; live event with seq=3 (already seen by + # client at last_event_id=5) must be dropped. + live_dup = encode_pubsub_message( + "msg-1", 3, "answer", {"type": "answer", "answer": "stale"} + ).encode("utf-8") + live_fresh = encode_pubsub_message( + "msg-1", 6, "answer", {"type": "answer", "answer": "fresh"} + ).encode("utf-8") + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([live_dup, live_fresh]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = [] + + gen = build_message_event_stream( + "msg-1", + last_event_id=5, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = _drain(gen, max_items=3) + + assert out[0] == ": connected\n\n" + # Stale live event dropped; only the fresh one yielded. + assert "id: 6" in out[1] + assert '"answer": "fresh"' in out[1] + + def test_yielded_live_event_advances_dedup_floor(self): + """A republish of an already-yielded seq must be dropped.""" + live_first = encode_pubsub_message( + "msg-1", 0, "answer", {"type": "answer", "answer": "first"} + ).encode("utf-8") + live_dup = encode_pubsub_message( + "msg-1", 0, "answer", {"type": "answer", "answer": "duplicate"} + ).encode("utf-8") + live_third = encode_pubsub_message( + "msg-1", 1, "answer", {"type": "answer", "answer": "third"} + ).encode("utf-8") + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([live_first, live_dup, live_third]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = [] + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = _drain(gen, max_items=4) + + assert out[0] == ": connected\n\n" + assert "id: 0" in out[1] + assert '"answer": "first"' in out[1] + # Duplicate seq=0 dropped. + assert "id: 1" in out[2] + assert '"answer": "third"' in out[2] + + +@pytest.mark.unit +class TestSnapshotWhenSubscribeUnavailable: + """Regression for C1: when Redis is down, ``Topic.subscribe`` exits + immediately without firing ``on_subscribe``. The snapshot is in + Postgres and must still be served. + """ + + def test_snapshot_served_when_subscribe_returns_immediately(self): + rows = [ + { + "sequence_no": 0, + "event_type": "answer", + "payload": {"type": "answer", "answer": "from snapshot"}, + }, + ] + # Topic.subscribe yields nothing (Redis-down behaviour). + def _empty_subscribe(self, on_subscribe=None, poll_timeout=1.0): + return + yield # pragma: no cover (make the function a generator) + + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _empty_subscribe, + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = rows + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = list(gen) + + assert out[0] == ": connected\n\n" + # Snapshot row served via the post-subscribe fallback path. + assert "id: 0" in out[1] + assert '"answer": "from snapshot"' in out[1] + + def test_callback_fired_then_subscribe_dies_does_not_duplicate(self): + """Regression: if ``on_subscribe`` ran and populated the buffer, + a subsequent inner-generator failure (e.g. transient Redis + ``get_message`` exception between SUBSCRIBE-ack and the first + poll) must not trigger a second snapshot read. Re-reading would + append the same rows twice and double the answer chunks on the + client (the per-message reconnect dispatcher does not dedup + by ``id``). + """ + rows = [ + { + "sequence_no": 0, + "event_type": "answer", + "payload": {"type": "answer", "answer": "first"}, + }, + { + "sequence_no": 1, + "event_type": "answer", + "payload": {"type": "answer", "answer": "second"}, + }, + ] + + # Mimic the broadcast_channel race: SUBSCRIBE acks, on_subscribe + # runs, then the inner ``get_message`` raises and the generator + # returns without ever yielding. + def _subscribe_dies_after_callback( + self, on_subscribe=None, poll_timeout=1.0 + ): + if on_subscribe is not None: + on_subscribe() + return + yield # pragma: no cover (make the function a generator) + + repo_mock = MagicMock() + repo_mock.read_after.return_value = rows + + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository", + return_value=repo_mock, + ), patch( + "application.streaming.event_replay.Topic.subscribe", + _subscribe_dies_after_callback, + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = list(gen) + + # The snapshot must have been read exactly once — re-reading + # would have re-appended the rows behind the originals. + assert repo_mock.read_after.call_count == 1 + assert out[0] == ": connected\n\n" + # Each row appears exactly once, in order. + assert "id: 0" in out[1] + assert '"answer": "first"' in out[1] + assert "id: 1" in out[2] + assert '"answer": "second"' in out[2] + # Nothing past the snapshot (no duplicates). + assert len(out) == 3 + + +@pytest.mark.unit +class TestTerminalEventClosesStream: + """Regression: ``/api/messages//events`` is a live tail that + keeps emitting keepalives. Without explicit close-on-terminal the + client's drain promise never resolves and the WSGI thread is + pinned waiting for events that won't come for an already-finished + stream. + """ + + def test_terminal_in_snapshot_closes_after_flush(self): + rows = [ + { + "sequence_no": 0, + "event_type": "answer", + "payload": {"type": "answer", "answer": "A"}, + }, + { + "sequence_no": 1, + "event_type": "end", + "payload": {"type": "end"}, + }, + ] + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = rows + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + # Drain to exhaustion. Without the close-on-terminal fix + # this would hang; with it, the generator returns after + # flushing the snapshot. + out = list(gen) + + assert out[0] == ": connected\n\n" + assert "id: 0" in out[1] + assert "id: 1" in out[2] + # No keepalives or further frames after the terminal. + assert all("keepalive" not in line for line in out[3:]) + + def test_terminal_falls_back_to_event_type_when_payload_lacks_type(self): + # Belt-and-suspenders: a journal write that records ``end`` only + # in the column (e.g. an abort handler that didn't seed + # ``payload.type``) must still terminate the replay. + rows = [ + { + "sequence_no": 0, + "event_type": "answer", + "payload": {"type": "answer", "answer": "partial"}, + }, + { + "sequence_no": 1, + "event_type": "end", + "payload": {}, + }, + ] + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = rows + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = list(gen) + + assert out[0] == ": connected\n\n" + assert "id: 0" in out[1] + assert "id: 1" in out[2] + assert all("keepalive" not in line for line in out[3:]) + + def test_terminal_in_live_tail_closes(self): + live_answer = encode_pubsub_message( + "msg-1", 0, "answer", {"type": "answer", "answer": "x"} + ).encode("utf-8") + live_end = encode_pubsub_message( + "msg-1", 1, "end", {"type": "end"} + ).encode("utf-8") + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([live_answer, live_end]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = [] + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = list(gen) + + assert out[0] == ": connected\n\n" + assert "id: 0" in out[1] and '"answer": "x"' in out[1] + assert "id: 1" in out[2] and '"type": "end"' in out[2] + assert all("keepalive" not in line for line in out[3:]) + + def test_error_event_also_closes(self): + """The agent's catch-all path emits ``error`` with no trailing + ``end`` — treating ``error`` as terminal closes that path too. + """ + live_err = encode_pubsub_message( + "msg-1", 0, "error", {"type": "error", "error": "boom"} + ).encode("utf-8") + with patch( + "application.streaming.event_replay.db_readonly" + ) as mock_readonly, patch( + "application.streaming.event_replay.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.event_replay.Topic.subscribe", + _fake_topic_subscribe([live_err]), + create=False, + ): + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.read_after.return_value = [] + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=0.05, + poll_timeout_seconds=0.01, + ) + out = list(gen) + + assert out[0] == ": connected\n\n" + assert "id: 0" in out[1] and '"type": "error"' in out[1] + assert all("keepalive" not in line for line in out[2:]) + + +@pytest.mark.unit +def test_sse_line_split_pattern_handles_all_terminators(): + assert _SSE_LINE_SPLIT_PATTERN.split("a\rb\nc\r\nd") == ["a", "b", "c", "d"] + + +@pytest.mark.unit +class TestWatchdogClosesIdleReconnect: + """Without the watchdog a reconnect stream with a non-terminal + snapshot and a dead producer (worker crash between chunks and + finalize) would emit keepalives forever — the live tail blocks + on a pub/sub that nobody publishes to and the frontend's drain + promise never resolves. The watchdog periodically inspects + ``conversation_messages`` and closes the stream with a terminal + SSE event when the row has gone terminal in the DB or the + producer's heartbeat has gone stale. + """ + + @staticmethod + def _patch_subscribe_idle_forever(): + """Build a fake ``Topic.subscribe`` that fires ``on_subscribe`` + and then yields ``None`` ticks indefinitely (i.e. the producer + is gone). Returns the generator function and a list collecting + the ``on_subscribe`` calls so tests can assert ordering. + """ + + def _impl(self, on_subscribe=None, poll_timeout=1.0): + if on_subscribe is not None: + on_subscribe() + while True: + yield None + + return _impl + + def _mock_liveness_row(self, status, err=None, is_stale=False): + """Build the ``conn.execute(...).first()`` return value the + watchdog SQL expects — ``(status, err, is_stale)``. + """ + return (status, err, is_stale) + + def _build_gen_with_liveness( + self, + liveness_row, + *, + keepalive_seconds=99.0, + watchdog_interval_seconds=0.0, + producer_idle_seconds=999.0, + ): + """Wire up the patches the watchdog tests share. + + The snapshot read goes through the patched + ``MessageEventsRepository`` (returns empty), and the watchdog + liveness check goes through ``conn.execute(...).first()`` on the + same ``db_readonly``-yielded ``MagicMock`` connection. + """ + mock_conn = MagicMock() + mock_conn.execute.return_value.first.return_value = liveness_row + + readonly_patch = patch( + "application.streaming.event_replay.db_readonly" + ) + repo_patch = patch( + "application.streaming.event_replay.MessageEventsRepository" + ) + subscribe_patch = patch( + "application.streaming.event_replay.Topic.subscribe", + self._patch_subscribe_idle_forever(), + create=False, + ) + + mock_readonly = readonly_patch.start() + mock_repo_cls = repo_patch.start() + subscribe_patch.start() + + mock_readonly.return_value.__enter__.return_value = mock_conn + mock_repo_cls.return_value.read_after.return_value = [] + + gen = build_message_event_stream( + "msg-1", + last_event_id=None, + keepalive_seconds=keepalive_seconds, + poll_timeout_seconds=0.001, + watchdog_interval_seconds=watchdog_interval_seconds, + producer_idle_seconds=producer_idle_seconds, + ) + return gen, [readonly_patch, repo_patch, subscribe_patch] + + def test_watchdog_emits_synthetic_end_when_status_complete(self): + """A row that flipped to ``complete`` after the snapshot read + (e.g. finalize ran on another worker, journal write lost) must + be surfaced as ``end`` so the client closes cleanly. + """ + gen, patches = self._build_gen_with_liveness( + self._mock_liveness_row("complete") + ) + try: + out = _drain(gen, max_items=5) + finally: + for p in patches: + p.stop() + + assert out[0] == ": connected\n\n" + # Watchdog synthetic: id:-1, ``{"type": "end"}`` + terminal = [s for s in out if '"type": "end"' in s] + assert len(terminal) == 1 + assert "id: -1" in terminal[0] + + def test_watchdog_emits_synthetic_error_when_status_failed(self): + gen, patches = self._build_gen_with_liveness( + self._mock_liveness_row( + "failed", err="RuntimeError: upstream blew up" + ) + ) + try: + out = _drain(gen, max_items=5) + finally: + for p in patches: + p.stop() + + assert out[0] == ": connected\n\n" + terminal = [s for s in out if '"type": "error"' in s] + assert len(terminal) == 1 + assert '"code": "producer_failed"' in terminal[0] + # The stashed error from metadata is surfaced verbatim so the + # UI can show the real reason instead of a generic message. + assert "RuntimeError: upstream blew up" in terminal[0] + + def test_watchdog_emits_synthetic_error_when_producer_stale(self): + """Non-terminal status + heartbeat older than the threshold ⇒ + producing worker is presumed dead. Without this, the live tail + would hang on keepalives until the proxy idle-timeout fires. + """ + gen, patches = self._build_gen_with_liveness( + self._mock_liveness_row("streaming", is_stale=True), + producer_idle_seconds=1.0, + ) + try: + out = _drain(gen, max_items=5) + finally: + for p in patches: + p.stop() + + assert out[0] == ": connected\n\n" + terminal = [s for s in out if '"type": "error"' in s] + assert len(terminal) == 1 + assert '"code": "producer_stale"' in terminal[0] + + def test_watchdog_does_not_fire_while_producer_alive(self): + """A non-terminal row with a fresh heartbeat is healthy; the + watchdog must keep silent (yield keepalives instead) so a + slow-but-alive stream isn't prematurely terminated. + """ + gen, patches = self._build_gen_with_liveness( + self._mock_liveness_row("streaming", is_stale=False), + keepalive_seconds=0.01, + ) + try: + out = _drain(gen, max_items=5) + finally: + for p in patches: + p.stop() + + assert out[0] == ": connected\n\n" + # No synthetic terminal — the rest of the output is keepalives. + assert all( + '"type": "end"' not in s and '"type": "error"' not in s + for s in out + ) + assert any("keepalive" in s for s in out) + + def test_watchdog_handles_missing_row_as_terminal(self): + """If the message row got deleted out from under us mid-tail, + the watchdog must close the stream rather than tail forever + on a row that no longer exists. + """ + gen, patches = self._build_gen_with_liveness(None) + try: + out = _drain(gen, max_items=5) + finally: + for p in patches: + p.stop() + + assert out[0] == ": connected\n\n" + terminal = [s for s in out if '"code": "message_missing"' in s] + assert len(terminal) == 1 diff --git a/tests/test_events_substrate.py b/tests/test_events_substrate.py new file mode 100644 index 00000000..7523f456 --- /dev/null +++ b/tests/test_events_substrate.py @@ -0,0 +1,390 @@ +"""Unit tests for the SSE substrate (publisher + Topic + route helpers). + +Round-trip integration tests against a real / fake Redis live in +``tests/test_events_integration.py``. The tests here lock in the +closure-mutated, race-sensitive bits the route relies on so future +refactors can't regress them silently. +""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from application.api.events.routes import ( + _SSE_LINE_SPLIT, + _format_sse, + _normalize_last_event_id, +) +from application.events.keys import ( + connection_counter_key, + stream_id_compare, + stream_key, + topic_name, +) +from application.events.publisher import publish_user_event +from application.streaming.broadcast_channel import Topic + + +# ── keys ──────────────────────────────────────────────────────────────── + + +@pytest.mark.unit +class TestKeys: + def test_stream_key(self): + assert stream_key("alice") == "user:alice:stream" + + def test_topic_name(self): + assert topic_name("alice") == "user:alice" + + def test_connection_counter_key(self): + assert connection_counter_key("alice") == "user:alice:sse_count" + + +# ── stream_id_compare ─────────────────────────────────────────────────── + + +@pytest.mark.unit +class TestStreamIdCompare: + def test_equal(self): + assert stream_id_compare("1234-0", "1234-0") == 0 + + def test_seq_ordering(self): + assert stream_id_compare("1234-0", "1234-1") == -1 + assert stream_id_compare("1234-1", "1234-0") == 1 + + def test_ms_ordering(self): + assert stream_id_compare("1234-0", "5678-0") == -1 + assert stream_id_compare("5678-0", "1234-0") == 1 + + def test_digit_count_does_not_break_int_compare(self): + # String compare would say "9" > "100"; integer compare must not. + assert stream_id_compare("9-0", "100-0") == -1 + assert stream_id_compare("100-0", "9-0") == 1 + + def test_missing_seq_treated_as_zero(self): + assert stream_id_compare("1234", "1234-0") == 0 + + def test_malformed_input_raises(self): + # Callers must pre-validate; the function refuses to silently + # lex-compare garbage because a malformed id that sorts + # lex-greater would pin dedup forever. + with pytest.raises(ValueError): + stream_id_compare("foo", "bar") + with pytest.raises(ValueError): + stream_id_compare("123-0", "foo") + with pytest.raises(ValueError): + stream_id_compare("foo", "123-0") + + +# ── _format_sse ───────────────────────────────────────────────────────── + + +@pytest.mark.unit +class TestFormatSse: + def test_simple_payload(self): + assert _format_sse("hello") == "data: hello\n\n" + + def test_with_event_id(self): + assert _format_sse("hello", event_id="42-0") == "id: 42-0\ndata: hello\n\n" + + def test_lf_split(self): + assert _format_sse("a\nb") == "data: a\ndata: b\n\n" + + def test_crlf_split(self): + assert _format_sse("a\r\nb") == "data: a\ndata: b\n\n" + + def test_cr_only_split(self): + # WHATWG SSE treats CR alone as a line terminator. Must match. + assert _format_sse("a\rb") == "data: a\ndata: b\n\n" + + def test_mixed_terminators(self): + # Each variant in one payload — all should split. + out = _format_sse("a\rb\nc\r\nd") + assert out == "data: a\ndata: b\ndata: c\ndata: d\n\n" + + def test_empty_payload(self): + # WHATWG: empty data field is dispatched as a message with data="". + assert _format_sse("") == "data: \n\n" + + def test_terminator_regex_compiles(self): + # Defensive: ensure the regex itself is well-formed. + assert _SSE_LINE_SPLIT.split("a\r\nb\rc\nd") == ["a", "b", "c", "d"] + + +# ── _normalize_last_event_id ──────────────────────────────────────────── + + +@pytest.mark.unit +class TestNormalizeLastEventId: + def test_none(self): + assert _normalize_last_event_id(None) is None + + def test_empty_string(self): + assert _normalize_last_event_id("") is None + + def test_whitespace_only(self): + assert _normalize_last_event_id(" ") is None + + def test_stripped_valid_id(self): + assert _normalize_last_event_id(" 1234-0 ") == "1234-0" + + def test_ms_only(self): + assert _normalize_last_event_id("1234567890") == "1234567890" + + def test_ms_seq(self): + assert _normalize_last_event_id("1234567890-0") == "1234567890-0" + + def test_high_seq(self): + assert _normalize_last_event_id("1-99999") == "1-99999" + + def test_garbage_rejected(self): + for bad in ("foo", "foo-bar", "1234-foo", "-1", "1234-", "1-2-3", "0x1234"): + assert _normalize_last_event_id(bad) is None, bad + + +# ── publisher ─────────────────────────────────────────────────────────── + + +@pytest.mark.unit +class TestPublishUserEvent: + def setup_method(self): + # Reset the cache singleton so the patched mock gets used. + import application.cache as cache_mod + + cache_mod._redis_instance = None + cache_mod._redis_creation_failed = False + + def teardown_method(self): + import application.cache as cache_mod + + cache_mod._redis_instance = None + cache_mod._redis_creation_failed = False + + def test_returns_none_on_missing_user_id(self): + assert publish_user_event("", "x.y", {}) is None + + def test_returns_none_on_missing_event_type(self): + assert publish_user_event("alice", "", {}) is None + + @patch("application.events.publisher.get_redis_instance") + @patch("application.events.publisher.settings") + def test_returns_none_when_push_disabled(self, mock_settings, mock_redis): + mock_settings.ENABLE_SSE_PUSH = False + mock_settings.EVENTS_STREAM_MAXLEN = 1000 + result = publish_user_event("alice", "src.ingest.progress", {"current": 1}) + assert result is None + # Must not even reach Redis when the master switch is off. + mock_redis.assert_not_called() + + @patch("application.events.publisher.get_redis_instance") + @patch("application.events.publisher.settings") + def test_returns_none_when_redis_unavailable(self, mock_settings, mock_redis): + mock_settings.ENABLE_SSE_PUSH = True + mock_settings.EVENTS_STREAM_MAXLEN = 1000 + mock_redis.return_value = None + assert publish_user_event("alice", "x.y", {}) is None + + @patch("application.events.publisher.get_redis_instance") + @patch("application.events.publisher.settings") + def test_returns_none_on_unserializable_payload( + self, mock_settings, mock_redis + ): + # Set object can't be JSON-encoded; the publisher must catch + # this *before* hitting Redis, with a single warning log. + mock_settings.ENABLE_SSE_PUSH = True + mock_settings.EVENTS_STREAM_MAXLEN = 1000 + result = publish_user_event( + "alice", "x.y", {"bad": {1, 2, 3}} # type: ignore[arg-type] + ) + assert result is None + mock_redis.assert_not_called() + + @patch("application.events.publisher.Topic") + @patch("application.events.publisher.get_redis_instance") + @patch("application.events.publisher.settings") + def test_xadd_and_publish_both_invoked_on_happy_path( + self, mock_settings, mock_redis, mock_topic_cls + ): + mock_settings.ENABLE_SSE_PUSH = True + mock_settings.EVENTS_STREAM_MAXLEN = 750 + mock_client = MagicMock() + mock_client.xadd.return_value = b"1735682400000-0" + mock_redis.return_value = mock_client + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + result = publish_user_event( + "alice", + "source.ingest.progress", + {"current": 30, "total": 100}, + scope={"kind": "source", "id": "abc"}, + ) + + assert result == "1735682400000-0" + + # XADD: stream key, MAXLEN, approximate — and the envelope before + # the id was known. + mock_client.xadd.assert_called_once() + args, kwargs = mock_client.xadd.call_args + assert args[0] == "user:alice:stream" + assert kwargs.get("maxlen") == 750 + assert kwargs.get("approximate") is True + # The envelope inside the XADD does NOT carry an ``id`` yet (it + # only exists after Redis returns), so the publish-time envelope + # is the source of truth for ``id``. + stored_event = json.loads(args[1]["event"]) + assert "id" not in stored_event + assert stored_event["type"] == "source.ingest.progress" + assert stored_event["topic"] == "user:alice" + assert stored_event["payload"] == {"current": 30, "total": 100} + assert stored_event["scope"] == {"kind": "source", "id": "abc"} + + # PUBLISH: same envelope plus the ``id`` from XADD. + mock_topic_cls.assert_called_once_with("user:alice") + mock_topic.publish.assert_called_once() + published = json.loads(mock_topic.publish.call_args[0][0]) + assert published["id"] == "1735682400000-0" + assert published["type"] == "source.ingest.progress" + + @patch("application.events.publisher.Topic") + @patch("application.events.publisher.get_redis_instance") + @patch("application.events.publisher.settings") + def test_xadd_failure_skips_live_publish( + self, mock_settings, mock_redis, mock_topic_cls + ): + """If the durable journal write fails there is no canonical id + to ship. Publishing an id-less envelope would put a record on + the wire that bypasses the SSE route's dedup floor and breaks + ``Last-Event-ID`` semantics for any reconnect — so we drop the + live publish too. Best-effort delivery means dropping + consistently, not delivering inconsistent state. + """ + mock_settings.ENABLE_SSE_PUSH = True + mock_settings.EVENTS_STREAM_MAXLEN = 1000 + mock_client = MagicMock() + mock_client.xadd.side_effect = Exception("redis went down") + mock_redis.return_value = mock_client + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + result = publish_user_event("alice", "x.y", {"k": 1}) + + # Backlog write failed → publisher returns None and skips the + # live publish so subscribers never see an id-less envelope. + assert result is None + mock_topic.publish.assert_not_called() + + +# ── Topic ─────────────────────────────────────────────────────────────── + + +@pytest.mark.unit +class TestTopic: + @patch("application.streaming.broadcast_channel.get_redis_instance") + def test_publish_returns_zero_when_redis_unavailable(self, mock_redis): + mock_redis.return_value = None + assert Topic("user:alice").publish("hi") == 0 + + @patch("application.streaming.broadcast_channel.get_redis_instance") + def test_publish_calls_redis_publish(self, mock_redis): + client = MagicMock() + client.publish.return_value = 3 + mock_redis.return_value = client + result = Topic("user:alice").publish("hi") + assert result == 3 + client.publish.assert_called_once_with("user:alice", "hi") + + @patch("application.streaming.broadcast_channel.get_redis_instance") + def test_publish_swallows_exceptions(self, mock_redis): + client = MagicMock() + client.publish.side_effect = Exception("boom") + mock_redis.return_value = client + # Must not raise. + assert Topic("user:alice").publish("hi") == 0 + + @patch("application.streaming.broadcast_channel.get_redis_instance") + def test_subscribe_returns_immediately_when_redis_unavailable( + self, mock_redis + ): + mock_redis.return_value = None + # Generator should produce nothing, not raise. + assert list(Topic("user:alice").subscribe(poll_timeout=0.01)) == [] + + @patch("application.streaming.broadcast_channel.get_redis_instance") + def test_subscribe_yields_none_on_poll_timeout(self, mock_redis): + client = MagicMock() + pubsub = MagicMock() + # Two timeouts (None) then a real message — caller breaks after. + pubsub.get_message.side_effect = [None, None, {"type": "message", "data": b"x"}] + client.pubsub.return_value = pubsub + mock_redis.return_value = client + + gen = Topic("user:alice").subscribe(poll_timeout=0.01) + first = next(gen) + second = next(gen) + third = next(gen) + assert first is None + assert second is None + assert third == b"x" + gen.close() + + @patch("application.streaming.broadcast_channel.get_redis_instance") + def test_subscribe_fires_on_subscribe_after_ack(self, mock_redis): + client = MagicMock() + pubsub = MagicMock() + pubsub.get_message.side_effect = [ + {"type": "subscribe", "data": 1}, + {"type": "message", "data": b"hi"}, + ] + client.pubsub.return_value = pubsub + mock_redis.return_value = client + + callback_calls = [] + + def cb(): + callback_calls.append(1) + + gen = Topic("user:alice").subscribe(on_subscribe=cb, poll_timeout=0.01) + # The "subscribe" message is consumed without yielding; the next + # value yielded is the actual data message. + msg = next(gen) + assert msg == b"hi" + assert callback_calls == [1] + gen.close() + + @patch("application.streaming.broadcast_channel.get_redis_instance") + def test_subscribe_cleans_up_on_generator_close(self, mock_redis): + client = MagicMock() + pubsub = MagicMock() + pubsub.get_message.side_effect = [ + {"type": "subscribe", "data": 1}, + None, # one tick so we stay in the loop + ] + client.pubsub.return_value = pubsub + mock_redis.return_value = client + + gen = Topic("user:alice").subscribe(poll_timeout=0.01) + next(gen) # advance into the subscribe-confirmation + gen.close() # client disconnect + + pubsub.unsubscribe.assert_called_once_with("user:alice") + pubsub.close.assert_called_once() + + @patch("application.streaming.broadcast_channel.get_redis_instance") + def test_subscribe_skips_unsubscribe_if_subscribe_never_acked( + self, mock_redis + ): + client = MagicMock() + pubsub = MagicMock() + pubsub.subscribe.side_effect = Exception("connection lost") + client.pubsub.return_value = pubsub + mock_redis.return_value = client + + # The generator must not yield, must not raise, and must not call + # unsubscribe (we never confirmed a subscription). + list(Topic("user:alice").subscribe(poll_timeout=0.01)) + pubsub.unsubscribe.assert_not_called() + pubsub.close.assert_called_once() diff --git a/tests/test_message_journal.py b/tests/test_message_journal.py new file mode 100644 index 00000000..250776f2 --- /dev/null +++ b/tests/test_message_journal.py @@ -0,0 +1,696 @@ +"""Unit tests for ``application/streaming/message_journal.py``. + +The journal hook is best-effort by contract — its failure modes are +the most important thing to lock down so a streaming hiccup never +crashes ``complete_stream``. +""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from application.streaming.message_journal import ( + BatchedJournalWriter, + record_event, +) + + +@pytest.mark.unit +class TestRecordEvent: + def test_invalid_args_return_false(self): + assert record_event("", 0, "answer") is False + assert record_event("msg-1", 0, "") is False + + def test_happy_path_writes_and_publishes(self): + with patch( + "application.streaming.message_journal.db_session" + ) as mock_session, patch( + "application.streaming.message_journal.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.message_journal.Topic" + ) as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo_cls.return_value = mock_repo + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + result = record_event( + "msg-1", 5, "answer", {"type": "answer", "answer": "ok"} + ) + + assert result is True + mock_repo.record.assert_called_once_with( + "msg-1", 5, "answer", {"type": "answer", "answer": "ok"} + ) + mock_topic_cls.assert_called_once_with("channel:msg-1") + mock_topic.publish.assert_called_once() + wire = mock_topic.publish.call_args[0][0] + envelope = json.loads(wire) + assert envelope["sequence_no"] == 5 + assert envelope["event_type"] == "answer" + assert envelope["payload"] == {"type": "answer", "answer": "ok"} + + def test_publish_attempted_even_when_journal_fails(self): + """A DB hiccup must not stop the live tail — currently-attached + subscribers should still receive the live event so their UI is + live even if a future reconnect's snapshot is missing this row. + """ + with patch( + "application.streaming.message_journal.db_session" + ) as mock_session, patch( + "application.streaming.message_journal.MessageEventsRepository" + ), patch( + "application.streaming.message_journal.Topic" + ) as mock_topic_cls: + mock_session.return_value.__enter__.side_effect = RuntimeError("pg down") + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + result = record_event( + "msg-1", 1, "answer", {"type": "answer", "answer": "x"} + ) + + assert result is False + mock_topic.publish.assert_called_once() + + def test_publish_failure_does_not_raise(self): + with patch( + "application.streaming.message_journal.db_session" + ) as mock_session, patch( + "application.streaming.message_journal.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.message_journal.Topic" + ) as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value.record = MagicMock() + mock_topic = MagicMock() + mock_topic.publish.side_effect = RuntimeError("redis down") + mock_topic_cls.return_value = mock_topic + + # Must not raise. + result = record_event("msg-1", 0, "answer", {"answer": "y"}) + assert result is True # Journal still committed. + + def test_payload_none_treated_as_empty_dict(self): + with patch( + "application.streaming.message_journal.db_session" + ) as mock_session, patch( + "application.streaming.message_journal.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.message_journal.Topic" + ) as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo_cls.return_value = mock_repo + mock_topic_cls.return_value = MagicMock() + + record_event("msg-1", 0, "end", None) + mock_repo.record.assert_called_once_with("msg-1", 0, "end", {}) + + def test_payload_non_dict_rejected_at_gate(self): + """Contract: payload must be a dict (or None). Lists, strings, + ints, and other shapes are rejected without writing or + publishing. + + Background: the live path (``base.py::_emit``) and the replay + path (``event_replay``) previously reconstructed non-dicts + differently — ``{"value": payload}`` live vs. + ``{"type": event_type}`` on replay — so a reconnecting client + would receive a different envelope than the one originally + streamed. Rejecting at this gate keeps the two paths + byte-identical. + """ + with patch( + "application.streaming.message_journal.db_session" + ) as mock_session, patch( + "application.streaming.message_journal.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.message_journal.Topic" + ) as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo_cls.return_value = mock_repo + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + result = record_event("msg-1", 0, "end", ["unexpected", "list"]) + + assert result is False + mock_repo.record.assert_not_called() + mock_topic.publish.assert_not_called() + + def test_integrity_error_retries_with_seq_plus_one(self): + """Composite-PK collision on (message_id, sequence_no) is + recovered by one retry against ``latest_sequence_no + 1`` — + the most likely cause is a stale seq seed on a continuation + retry, where the route read MAX(seq) from a separate + connection before another writer committed past it. + + On success the live pubsub publish uses the retried seq so + the journal row and the live frame agree, even though the + caller's original POST stream still carries the pre-retry id. + """ + from sqlalchemy.exc import IntegrityError + + # Two repo instances: one per ``with db_session()`` block. + # repo_first.record raises IntegrityError; the readonly + # session returns latest=7; repo_retry.record succeeds at 8. + repo_first = MagicMock(name="repo_first") + repo_first.record.side_effect = IntegrityError("stmt", {}, Exception()) + repo_readonly = MagicMock(name="repo_readonly") + repo_readonly.latest_sequence_no.return_value = 7 + repo_retry = MagicMock(name="repo_retry") + + repo_instances = iter([repo_first, repo_readonly, repo_retry]) + + with patch( + "application.streaming.message_journal.db_session" + ) as mock_session, patch( + "application.streaming.message_journal.db_readonly" + ) as mock_readonly, patch( + "application.streaming.message_journal.MessageEventsRepository", + side_effect=lambda conn: next(repo_instances), + ), patch( + "application.streaming.message_journal.Topic" + ) as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + result = record_event("msg-1", 3, "answer", {"text": "hi"}) + + assert result is True + # First INSERT attempted at the caller's seq=3. + repo_first.record.assert_called_once_with( + "msg-1", 3, "answer", {"text": "hi"} + ) + # Latest probed for the retry seed. + repo_readonly.latest_sequence_no.assert_called_once_with("msg-1") + # Retry INSERT at latest+1 = 8. + repo_retry.record.assert_called_once_with( + "msg-1", 8, "answer", {"text": "hi"} + ) + # Live publish uses the materialised seq so the wire and + # the journal row stay in lockstep on the retry path. + mock_topic.publish.assert_called_once() + wire = json.loads(mock_topic.publish.call_args[0][0]) + assert wire["sequence_no"] == 8 + + def test_integrity_error_retry_failure_drops_silently(self): + """If the retry collides again (truly concurrent writers in + lockstep) the journal write is dropped but the function still + returns ``False`` without raising — the streaming loop must + not be killed by a journal hiccup. + """ + from sqlalchemy.exc import IntegrityError + + repo_first = MagicMock(name="repo_first") + repo_first.record.side_effect = IntegrityError("stmt", {}, Exception()) + repo_readonly = MagicMock(name="repo_readonly") + repo_readonly.latest_sequence_no.return_value = 3 + repo_retry = MagicMock(name="repo_retry") + repo_retry.record.side_effect = IntegrityError("stmt", {}, Exception()) + + repo_instances = iter([repo_first, repo_readonly, repo_retry]) + + with patch( + "application.streaming.message_journal.db_session" + ) as mock_session, patch( + "application.streaming.message_journal.db_readonly" + ) as mock_readonly, patch( + "application.streaming.message_journal.MessageEventsRepository", + side_effect=lambda conn: next(repo_instances), + ), patch( + "application.streaming.message_journal.Topic" + ) as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_readonly.return_value.__enter__.return_value = MagicMock() + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + result = record_event("msg-1", 0, "answer", {"text": "hi"}) + + assert result is False + # Both INSERT attempts fired; the second raised too. + assert repo_first.record.call_count == 1 + assert repo_retry.record.call_count == 1 + # Live publish still fires on the materialised seq path — + # subscribers downgrade to keepalives if they were waiting + # on this event, which is correct since the journal can't + # serve it on a future reconnect anyway. + mock_topic.publish.assert_called_once() + + def test_payload_with_null_byte_stripped_before_insert(self): + """``\\x00`` in a string value would crash JSONB INSERT — strip + at the journal boundary so the row lands and reconnecting + clients see the chunk. + """ + with patch( + "application.streaming.message_journal.db_session" + ) as mock_session, patch( + "application.streaming.message_journal.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.message_journal.Topic" + ) as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo_cls.return_value = mock_repo + mock_topic_cls.return_value = MagicMock() + + record_event( + "msg-1", 0, "answer", {"answer": "hel\x00lo", "type": "answer"} + ) + + mock_repo.record.assert_called_once_with( + "msg-1", 0, "answer", {"answer": "hello", "type": "answer"} + ) + + def test_payload_with_nested_null_bytes_stripped_recursively(self): + """Strip walks nested dicts and lists — and string keys, to be + safe against an LLM emitting structured data with NULs. + """ + with patch( + "application.streaming.message_journal.db_session" + ) as mock_session, patch( + "application.streaming.message_journal.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.message_journal.Topic" + ) as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo_cls.return_value = mock_repo + mock_topic_cls.return_value = MagicMock() + + record_event( + "msg-1", + 0, + "answer", + { + "outer": {"inner\x00key": "a\x00b"}, + "items": ["x\x00y", {"k": "v\x00"}], + "clean": "ok", + }, + ) + + mock_repo.record.assert_called_once_with( + "msg-1", + 0, + "answer", + { + "outer": {"innerkey": "ab"}, + "items": ["xy", {"k": "v"}], + "clean": "ok", + }, + ) + + def test_payload_without_null_bytes_unchanged(self): + """Common-case: no NUL, no allocation churn — payload passed + through identically. + """ + with patch( + "application.streaming.message_journal.db_session" + ) as mock_session, patch( + "application.streaming.message_journal.MessageEventsRepository" + ) as mock_repo_cls, patch( + "application.streaming.message_journal.Topic" + ) as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo_cls.return_value = mock_repo + mock_topic_cls.return_value = MagicMock() + + payload = {"a": "1", "nested": {"b": [1, 2, "three"]}} + record_event("msg-1", 0, "answer", payload) + + mock_repo.record.assert_called_once_with( + "msg-1", 0, "answer", payload + ) + + +@pytest.mark.unit +class TestBatchedJournalWriter: + """``BatchedJournalWriter`` amortizes per-emit PG commits. + + Tests cover the four flush triggers (size, time, lifecycle, + explicit), the bulk→per-row IntegrityError fallback, and the + live-publish-stays-synchronous invariant. + """ + + def _patch_io(self): + """Standard patch set: stub PG sessions and Redis pubsub. + + Returns a tuple ``(session_cm, readonly_cm, repo_factory, topic)``. + The repo_factory is a list — each ``MessageEventsRepository(...)`` + call inside the writer pops one element off the front. Tests + push fakes onto it before triggering the path that needs them. + """ + from unittest.mock import patch as _patch + + session_cm = _patch("application.streaming.message_journal.db_session") + readonly_cm = _patch( + "application.streaming.message_journal.db_readonly" + ) + repo_cls = _patch( + "application.streaming.message_journal.MessageEventsRepository" + ) + topic_cls = _patch("application.streaming.message_journal.Topic") + return session_cm, readonly_cm, repo_cls, topic_cls + + def test_size_trigger_flushes_at_batch_size(self): + """Buffer reaches ``batch_size`` → one bulk_record call covers + all rows; publishes fire after the bulk commit, one per row. + """ + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo_cls.return_value = mock_repo + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + writer = BatchedJournalWriter( + "msg-1", batch_size=3, batch_interval_ms=10_000 + ) + # First two records: buffered, no bulk_record yet, no publish. + writer.record(0, "answer", {"text": "a"}) + writer.record(1, "answer", {"text": "b"}) + mock_repo.bulk_record.assert_not_called() + assert mock_topic.publish.call_count == 0 + # Third record: triggers a size-based flush. + writer.record(2, "answer", {"text": "c"}) + mock_repo.bulk_record.assert_called_once() + args = mock_repo.bulk_record.call_args + assert args.args[0] == "msg-1" + buffered = args.args[1] + assert len(buffered) == 3 + assert [seq for seq, _, _ in buffered] == [0, 1, 2] + # Publishes fire after the bulk commit, one per row in order. + assert mock_topic.publish.call_count == 3 + published_seqs = [ + json.loads(call.args[0])["sequence_no"] + for call in mock_topic.publish.call_args_list + ] + assert published_seqs == [0, 1, 2] + + def test_time_trigger_flushes_after_interval(self): + """When the elapsed time since the last flush exceeds + ``batch_interval_ms``, the next ``record()`` flushes — even if + the buffer is well below ``batch_size``. Drives reconnect + visibility for slow producers. + """ + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls, patch( + "application.streaming.message_journal.time.monotonic" + ) as mock_mono: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo_cls.return_value = mock_repo + mock_topic_cls.return_value = MagicMock() + + # First call from __init__ at t=0; subsequent calls drive + # the time-trigger check inside ``_should_flush``. + # t=0.000 → __init__ snapshots last_flush + # t=0.001 → record(0) — 1ms elapsed → no flush + # t=0.001 → after record, last_flush snapshot when flush called (n/a) + # t=0.200 → record(1) — 200ms elapsed → flush + # t=0.200 → flush updates last_flush + mock_mono.side_effect = iter([0.000, 0.001, 0.200, 0.200]) + + writer = BatchedJournalWriter( + "msg-1", batch_size=100, batch_interval_ms=100 + ) + writer.record(0, "answer", {"text": "a"}) + mock_repo.bulk_record.assert_not_called() + writer.record(1, "answer", {"text": "b"}) + mock_repo.bulk_record.assert_called_once() + + def test_close_drains_remaining_buffer(self): + """``close()`` is the lifecycle flush — at end of stream every + buffered event must commit before the writer goes silent. + Idempotent so it's safe in multiple finally clauses. + """ + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo_cls.return_value = mock_repo + mock_topic_cls.return_value = MagicMock() + + writer = BatchedJournalWriter( + "msg-1", batch_size=100, batch_interval_ms=100_000 + ) + writer.record(0, "answer", {"text": "a"}) + writer.record(1, "end", {"type": "end"}) + mock_repo.bulk_record.assert_not_called() + + writer.close() + mock_repo.bulk_record.assert_called_once() + assert len(mock_repo.bulk_record.call_args.args[1]) == 2 + + # Second close is a no-op. + writer.close() + assert mock_repo.bulk_record.call_count == 1 + + def test_record_after_close_returns_false(self): + """Writing past ``close()`` must not silently land in a stale + buffer — return False so the caller's flow surfaces the bug. + """ + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value = MagicMock() + mock_topic_cls.return_value = MagicMock() + + writer = BatchedJournalWriter("msg-1", batch_size=100) + writer.close() + assert writer.record(0, "answer", {"text": "a"}) is False + + def test_record_rejects_non_dict_payload(self): + """Same contract as ``record_event``: non-dict payloads are + rejected to keep live and replay paths byte-identical. + """ + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value = MagicMock() + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + writer = BatchedJournalWriter("msg-1") + assert writer.record(0, "answer", ["bad", "list"]) is False + # Nothing publishes either — the gate fires before the wire. + mock_topic.publish.assert_not_called() + + def test_bulk_collision_falls_back_to_per_row(self): + """Bulk INSERT fails with IntegrityError → writer retries each + row individually via the legacy ``record_event`` retry path, + so a single colliding seq doesn't drop the whole batch. + """ + from sqlalchemy.exc import IntegrityError + + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm as mock_readonly, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_readonly.return_value.__enter__.return_value = MagicMock() + + # First call: bulk_record raises. Subsequent calls: per-row + # record succeeds. Use a single fake whose first + # ``bulk_record`` call raises and whose ``record`` succeeds. + bulk_repo = MagicMock(name="bulk_repo") + bulk_repo.bulk_record.side_effect = IntegrityError( + "stmt", {}, Exception() + ) + per_row_repo = MagicMock(name="per_row_repo") + # MessageEventsRepository(conn) is called once per session + # opened. Bulk path = 1; per-row fallback = 2 rows × 1 each. + mock_repo_cls.side_effect = [bulk_repo, per_row_repo, per_row_repo] + + mock_topic_cls.return_value = MagicMock() + + writer = BatchedJournalWriter("msg-1", batch_size=2) + writer.record(0, "answer", {"text": "a"}) + writer.record(1, "answer", {"text": "b"}) + + bulk_repo.bulk_record.assert_called_once() + # Per-row fallback wrote each row in its own session. + assert per_row_repo.record.call_count == 2 + assert per_row_repo.record.call_args_list[0].args[1] == 0 + assert per_row_repo.record.call_args_list[1].args[1] == 1 + + def test_flush_clears_buffer_even_on_total_failure(self): + """A flush that fails with a non-IntegrityError exception must + still clear the buffer — leaving rows would grow memory + unbounded across the remainder of the stream. Degraded UX + (missing snapshot rows) beats runaway memory. + """ + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo.bulk_record.side_effect = RuntimeError("PG gone") + mock_repo_cls.return_value = mock_repo + mock_topic_cls.return_value = MagicMock() + + writer = BatchedJournalWriter("msg-1", batch_size=2) + writer.record(0, "answer", {"text": "a"}) + writer.record(1, "answer", {"text": "b"}) + # Bulk failed, but the buffer is cleared so a subsequent + # close() is a no-op and memory stays bounded. + assert writer._buffer == [] + writer.close() + # Only the one flush we forced — no double-attempt on + # close after the buffer was already drained. + assert mock_repo.bulk_record.call_count == 1 + + def test_record_does_not_publish_synchronously(self): + """Regression guard: ``record()`` must not publish before the + journal INSERT commits. Buffering an event with no flush yet + means zero pubsub frames on the wire — otherwise a reconnect + snapshot could miss a row that subscribers already received. + """ + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo_cls.return_value = MagicMock() + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + writer = BatchedJournalWriter("msg-1", batch_size=100) + for i in range(5): + writer.record(i, "answer", {"text": str(i)}) + mock_topic.publish.assert_not_called() + + def test_flush_publishes_each_buffered_event_after_bulk_insert_in_order( + self, + ): + """Happy path: flush issues one bulk INSERT, then publishes each + buffered frame in the order it was recorded. + """ + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo_cls.return_value = mock_repo + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + writer = BatchedJournalWriter( + "msg-1", batch_size=100, batch_interval_ms=100_000 + ) + for i in range(4): + writer.record(i, "answer", {"text": str(i)}) + mock_topic.publish.assert_not_called() + + writer.flush() + mock_repo.bulk_record.assert_called_once() + assert mock_topic.publish.call_count == 4 + published_seqs = [ + json.loads(call.args[0])["sequence_no"] + for call in mock_topic.publish.call_args_list + ] + assert published_seqs == [0, 1, 2, 3] + + def test_flush_bulk_exception_drops_both_row_and_publish(self): + """Non-IntegrityError from bulk_record → both the journal row + and the live publish are dropped together. Matches the + publish_user_event "drop live publish when durable write fails" + contract. + """ + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo.bulk_record.side_effect = RuntimeError("PG gone") + mock_repo_cls.return_value = mock_repo + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + writer = BatchedJournalWriter("msg-1", batch_size=2) + writer.record(0, "answer", {"text": "a"}) + writer.record(1, "answer", {"text": "b"}) + + mock_repo.bulk_record.assert_called_once() + mock_topic.publish.assert_not_called() + + def test_flush_per_row_publishes_on_commit_skips_on_persistent_drop(self): + """IntegrityError-after-retry path: per-row fallback publishes + each row that lands in PG (including those rewritten at + latest+1), and skips publish for any row dropped after the + retry collision. + """ + from sqlalchemy.exc import IntegrityError + + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm as mock_readonly, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_readonly.return_value.__enter__.return_value = MagicMock() + + # Bulk attempt collides. Per-row: row 0 commits at its seq; + # row 1 collides, readonly probe returns latest=9, retry at + # 10 collides too → dropped without publish. + bulk_repo = MagicMock(name="bulk_repo") + bulk_repo.bulk_record.side_effect = IntegrityError( + "stmt", {}, Exception() + ) + row0_repo = MagicMock(name="row0_repo") + row1_first_repo = MagicMock(name="row1_first_repo") + row1_first_repo.record.side_effect = IntegrityError( + "stmt", {}, Exception() + ) + row1_readonly_repo = MagicMock(name="row1_readonly_repo") + row1_readonly_repo.latest_sequence_no.return_value = 9 + row1_retry_repo = MagicMock(name="row1_retry_repo") + row1_retry_repo.record.side_effect = IntegrityError( + "stmt", {}, Exception() + ) + mock_repo_cls.side_effect = [ + bulk_repo, + row0_repo, + row1_first_repo, + row1_readonly_repo, + row1_retry_repo, + ] + + mock_topic = MagicMock() + mock_topic_cls.return_value = mock_topic + + writer = BatchedJournalWriter("msg-1", batch_size=2) + writer.record(0, "answer", {"text": "a"}) + writer.record(1, "answer", {"text": "b"}) + + # Only row 0 publishes — row 1 was dropped after the retry + # collision, so no live frame reaches the wire for it. + assert mock_topic.publish.call_count == 1 + wire = json.loads(mock_topic.publish.call_args[0][0]) + assert wire["sequence_no"] == 0 + assert wire["payload"] == {"text": "a"} + + def test_record_strips_null_bytes_before_buffer(self): + """Null-byte sanitization applies at the writer boundary too — + otherwise a NUL-bearing chunk would land in the buffer, the + bulk INSERT would fail with ``DataError``, and the per-row + fallback would re-attempt the same broken row. Strip at the + gate so the buffered tuple is JSONB-safe. + """ + session_cm, readonly_cm, repo_cls_p, topic_cls_p = self._patch_io() + with session_cm as mock_session, readonly_cm, repo_cls_p as mock_repo_cls, topic_cls_p as mock_topic_cls: + mock_session.return_value.__enter__.return_value = MagicMock() + mock_repo = MagicMock() + mock_repo_cls.return_value = mock_repo + mock_topic_cls.return_value = MagicMock() + + writer = BatchedJournalWriter("msg-1", batch_size=2) + writer.record(0, "answer", {"answer": "hel\x00lo"}) + writer.record( + 1, "answer", {"nested": {"k\x00": ["a\x00b", "ok"]}} + ) + + mock_repo.bulk_record.assert_called_once() + buffered = mock_repo.bulk_record.call_args.args[1] + assert buffered[0][2] == {"answer": "hello"} + assert buffered[1][2] == {"nested": {"k": ["ab", "ok"]}} diff --git a/tests/test_tool_approval.py b/tests/test_tool_approval.py index a9e95b0b..0d166712 100644 --- a/tests/test_tool_approval.py +++ b/tests/test_tool_approval.py @@ -1,4 +1,4 @@ -"""Tests for tool approval (Phase 3). +"""Tests for tool approval. Covers require_approval flag, check_pause for approval, the handler pause/resume flow, and gen_continuation with approved/denied actions. diff --git a/tests/test_v1_translator.py b/tests/test_v1_translator.py index 6974935c..91157e59 100644 --- a/tests/test_v1_translator.py +++ b/tests/test_v1_translator.py @@ -1,4 +1,4 @@ -"""Tests for the v1 API translator (Phase 4). +"""Tests for the v1 API translator. Covers request translation, response translation, streaming event translation, continuation detection, and history conversion. diff --git a/tests/worker/conftest.py b/tests/worker/conftest.py index eac8137a..26953e67 100644 --- a/tests/worker/conftest.py +++ b/tests/worker/conftest.py @@ -40,7 +40,12 @@ def patch_worker_db(pg_conn, monkeypatch): def task_self(): """Minimal stand-in for the Celery task ``self`` passed to workers. - Only ``update_state`` is ever exercised in the happy paths we cover - here, so a MagicMock is more than enough. + ``update_state`` and ``request.retries`` are the only attributes the + worker bodies touch in our tests. Defaulting ``retries`` to 0 makes + every fixture instance behave like a fresh first attempt, which is + what the queued-event publishes are gated on. Tests covering the + retry path override this to a positive int. """ - return MagicMock(name="celery_task_self") + task = MagicMock(name="celery_task_self") + task.request.retries = 0 + return task diff --git a/tests/worker/test_worker_sse_publishes.py b/tests/worker/test_worker_sse_publishes.py new file mode 100644 index 00000000..5dd9f107 --- /dev/null +++ b/tests/worker/test_worker_sse_publishes.py @@ -0,0 +1,1018 @@ +"""SSE publish wiring tests for ``application.worker``. + +Each worker function emits ``publish_user_event`` envelopes at its +queued / progress / completed / failed boundaries. The SSE frontend's +upload-toast, reingest-toast, attachment-toast, and MCP-OAuth toast all +depend on this exact emit sequence, so a regression that silently drops +a publish leaves the UI wedged on a stale "training" state until the +polling fallback rescues it. + +These tests patch ``application.worker.publish_user_event`` with a +capture list and assert the ordered call args per worker. Broader +worker behaviour (PG side effects, pipeline correctness) is covered by +the per-task test files in this directory; here we focus narrowly on +the publish contract. +""" + +from __future__ import annotations + +import uuid +from io import BytesIO +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from application.parser.schema.base import Document +from application.storage.db.repositories.sources import SourcesRepository + + +# ── helpers ────────────────────────────────────────────────────────────── + + +class _PublishCapture: + """Drop-in replacement for ``publish_user_event``. + + Records every call as ``(user_id, event_type, payload, scope)`` so + tests can assert ordering and envelope shape without caring about + the Redis/journal side effects (covered separately in + ``tests/test_events_substrate.py``). + """ + + def __init__(self) -> None: + self.calls: list[tuple[str, str, dict, dict | None]] = [] + + def __call__( + self, + user_id: str, + event_type: str, + payload: dict[str, Any], + *, + scope: dict[str, Any] | None = None, + ) -> None: + # Defensive copy so mutations to the payload after publish (the + # worker reuses some dicts) don't retro-edit our snapshot. + self.calls.append((user_id, event_type, dict(payload), scope)) + + def types(self) -> list[str]: + return [call[1] for call in self.calls] + + +@pytest.fixture +def publishes(monkeypatch): + """Patch ``publish_user_event`` in the worker module and yield the capture.""" + from application import worker + + cap = _PublishCapture() + monkeypatch.setattr(worker, "publish_user_event", cap) + return cap + + +def _patch_ingest_pipeline_min(monkeypatch, *, raise_in_pipeline: bool = False): + """Minimal stubs so ``ingest_worker`` runs to completion in-process. + + Mirrors ``tests/worker/test_ingest_worker.py::_patch_ingest_pipeline`` + but exposes a knob to force a mid-pipeline exception so the + ``failed`` branch can be reached without otherwise rewriting the + function. + """ + from application import worker + + fake_storage = MagicMock(name="storage") + fake_storage.is_directory.return_value = False + fake_storage.get_file.return_value = BytesIO(b"hello") + monkeypatch.setattr( + worker.StorageCreator, "get_storage", lambda: fake_storage + ) + + fake_reader = MagicMock(name="reader") + fake_reader.load_data.return_value = [ + Document(text="hello body", extra_info={"source": "a.txt"}) + ] + fake_reader.directory_structure = { + "a.txt": {"type": "text/plain", "size_bytes": 5, "token_count": 2} + } + monkeypatch.setattr( + worker, "SimpleDirectoryReader", lambda *a, **kw: fake_reader + ) + + if raise_in_pipeline: + def _boom(*a, **kw): + raise RuntimeError("pipeline kaboom") + + monkeypatch.setattr(worker, "embed_and_store_documents", _boom) + else: + monkeypatch.setattr( + worker, + "embed_and_store_documents", + lambda docs, full_path, source_id, task, **kw: None, + ) + + monkeypatch.setattr( + worker, "upload_index", lambda full_path, file_data: None + ) + + +# ── ingest_worker ──────────────────────────────────────────────────────── + + +@pytest.mark.unit +class TestIngestWorkerPublishes: + def test_happy_path_emits_queued_then_completed( + self, patch_worker_db, task_self, monkeypatch, publishes + ): + from application import worker + + _patch_ingest_pipeline_min(monkeypatch) + caller_source_id = str(uuid.uuid4()) + + worker.ingest_worker( + task_self, + directory="inputs", + formats=[".txt"], + job_name="job1", + file_path="inputs/eve/job1/a.txt", + filename="a.txt", + user="eve", + retriever="classic", + source_id=caller_source_id, + ) + + assert publishes.types() == [ + "source.ingest.queued", + "source.ingest.completed", + ] + + user, etype, payload, scope = publishes.calls[0] + assert user == "eve" + assert etype == "source.ingest.queued" + assert payload["filename"] == "a.txt" + assert payload["job_name"] == "job1" + assert payload["operation"] == "upload" + # ``source_id`` in the queued payload MUST match the value the + # HTTP route returned to the frontend, otherwise the upload + # toast cannot match the event back to its task. + assert payload["source_id"] == caller_source_id + assert scope == {"kind": "source", "id": caller_source_id} + + completed = publishes.calls[1][2] + assert completed["source_id"] == caller_source_id + assert completed["operation"] == "upload" + assert "tokens" in completed + # Forward-looking contract: ``limited`` rides on every + # completed envelope so the frontend slice can route a future + # token-cap detection to the token-limit toast without further + # wire changes. Always False today. + assert completed["limited"] is False + + def test_pipeline_failure_emits_queued_then_failed( + self, patch_worker_db, task_self, monkeypatch, publishes + ): + from application import worker + + _patch_ingest_pipeline_min(monkeypatch, raise_in_pipeline=True) + + with pytest.raises(RuntimeError, match="pipeline kaboom"): + worker.ingest_worker( + task_self, + directory="inputs", + formats=[".txt"], + job_name="job1", + file_path="inputs/eve/job1/a.txt", + filename="a.txt", + user="eve", + retriever="classic", + idempotency_key="key-A", + ) + + assert publishes.types() == [ + "source.ingest.queued", + "source.ingest.failed", + ] + queued_payload = publishes.calls[0][2] + failed_payload = publishes.calls[1][2] + # Same source_id across the queued/failed pair so the frontend + # toast can correlate them. + assert queued_payload["source_id"] == failed_payload["source_id"] + # ``error`` is truncated to 1024 chars but our error message is + # well under that. + assert "pipeline kaboom" in failed_payload["error"] + + +# ── reingest_source_worker ─────────────────────────────────────────────── + + +def _seed_source_for_reingest(pg_conn, *, user_id: str, name: str = "doc-set"): + return SourcesRepository(pg_conn).create( + name, + user_id=user_id, + type="local", + retriever="classic", + file_path=f"inputs/{user_id}/{name}", + directory_structure={ + "stale.txt": { + "type": "text/plain", + "size_bytes": 10, + "token_count": 5, + } + }, + ) + + +def _stub_reingest_storage_and_vectorstore(monkeypatch): + from application import worker + + fake_storage = MagicMock(name="storage") + fake_storage.is_directory.return_value = True + fake_storage.list_files.return_value = [] + monkeypatch.setattr( + worker.StorageCreator, "get_storage", lambda: fake_storage + ) + + fake_store = MagicMock(name="vector_store") + fake_store.get_chunks.return_value = [] + monkeypatch.setattr( + "application.vectorstore.vector_creator.VectorCreator.create_vectorstore", + lambda *a, **kw: fake_store, + ) + return fake_store + + +@pytest.mark.unit +class TestReingestSourceWorkerPublishes: + def test_with_changes_emits_queued_then_completed_with_chunk_counts( + self, pg_conn, patch_worker_db, task_self, monkeypatch, publishes + ): + from application import worker + + src = _seed_source_for_reingest(pg_conn, user_id="alice") + source_id = str(src["id"]) + + _stub_reingest_storage_and_vectorstore(monkeypatch) + + fake_reader = MagicMock(name="reader") + fake_reader.load_data.return_value = [] + fake_reader.directory_structure = { + "fresh.md": { + "type": "text/markdown", + "size_bytes": 42, + "token_count": 17, + } + } + fake_reader.file_token_counts = {"fresh.md": 17} + monkeypatch.setattr( + worker, "SimpleDirectoryReader", lambda *a, **kw: fake_reader + ) + + worker.reingest_source_worker(task_self, source_id, "alice") + + assert publishes.types() == [ + "source.ingest.queued", + "source.ingest.completed", + ] + queued = publishes.calls[0][2] + assert queued["operation"] == "reingest" + assert queued["source_id"] == source_id + assert queued["name"] == "doc-set" + + completed = publishes.calls[1][2] + assert completed["operation"] == "reingest" + # ``chunks_added`` / ``chunks_deleted`` are integer counters + # the toast displays — keep them on the contract. + assert "chunks_added" in completed + assert "chunks_deleted" in completed + # ``no_changes`` is only set on the early-return branch; the + # full reingest path must not synthesize it. + assert "no_changes" not in completed + + def test_no_changes_branch_still_emits_completed( + self, pg_conn, patch_worker_db, task_self, monkeypatch, publishes + ): + """The early-return ``No changes detected`` path is the only + place a reingest can finish without going through the + chunk-diff loop. If it didn't publish a terminal event the + toast would hang on ``training`` forever. + """ + from application import worker + + # Seed with the same directory_structure the reader will report, + # so ``added_files`` and ``removed_files`` are both empty. + repo = SourcesRepository(pg_conn) + same_structure = { + "unchanged.txt": { + "type": "text/plain", + "size_bytes": 7, + "token_count": 3, + } + } + src = repo.create( + "stable-set", + user_id="alice", + type="local", + retriever="classic", + file_path="inputs/alice/stable-set", + directory_structure=same_structure, + ) + source_id = str(src["id"]) + + _stub_reingest_storage_and_vectorstore(monkeypatch) + + fake_reader = MagicMock(name="reader") + fake_reader.load_data.return_value = [] + fake_reader.directory_structure = same_structure + fake_reader.file_token_counts = {"unchanged.txt": 3} + monkeypatch.setattr( + worker, "SimpleDirectoryReader", lambda *a, **kw: fake_reader + ) + + worker.reingest_source_worker(task_self, source_id, "alice") + + assert publishes.types() == [ + "source.ingest.queued", + "source.ingest.completed", + ] + completed = publishes.calls[1][2] + assert completed.get("no_changes") is True + assert completed["chunks_added"] == 0 + assert completed["chunks_deleted"] == 0 + + def test_failure_before_lookup_emits_failed_without_queued( + self, patch_worker_db, task_self, monkeypatch, publishes + ): + """If the source lookup fails (row missing, PG hiccup) the + ``queued`` publish has not yet happened — but the ``failed`` + publish at the outer except must still fire so the frontend + toast doesn't wedge. The worker re-raises after publishing, + which is the documented contract for Celery retry handling. + """ + from application import worker + + with pytest.raises(ValueError, match="not found"): + worker.reingest_source_worker( + task_self, + "00000000-0000-0000-0000-000000000000", + "ghost", + ) + + # Lookup raises ValueError before the ``queued`` publish at + # line ~787; the outer except fires the ``failed`` publish. + assert publishes.types() == ["source.ingest.failed"] + failed = publishes.calls[0][2] + assert failed["operation"] == "reingest" + # Pre-lookup, ``source_name`` is empty; payload should still + # carry a ``name`` field (empty string) so the frontend doesn't + # KeyError on the toast template. + assert "name" in failed + assert "error" in failed + + +# ── remote_worker (upload mode) ────────────────────────────────────────── + + +def _stub_remote_pipeline(monkeypatch, *, raise_in_pipeline: bool = False): + from application import worker + + fake_loader = MagicMock(name="remote_loader") + fake_loader.load_data.return_value = [ + Document( + text="page body", + extra_info={"file_path": "guides/setup.md", "title": "setup"}, + doc_id="d1", + ) + ] + monkeypatch.setattr( + worker.RemoteCreator, "create_loader", lambda loader: fake_loader + ) + + if raise_in_pipeline: + def _boom(*a, **kw): + raise RuntimeError("remote-pipeline kaboom") + + monkeypatch.setattr(worker, "embed_and_store_documents", _boom) + else: + monkeypatch.setattr( + worker, + "embed_and_store_documents", + lambda docs, full_path, source_id, task, **kw: None, + ) + monkeypatch.setattr( + worker, "upload_index", lambda full_path, file_data: None + ) + + +@pytest.mark.unit +class TestRemoteWorkerPublishes: + def test_upload_happy_path( + self, tmp_path, task_self, monkeypatch, publishes + ): + from application import worker + + _stub_remote_pipeline(monkeypatch) + + worker.remote_worker( + task_self, + {"urls": ["http://example.com"]}, + "my-remote", + "bob", + "crawler", + directory=str(tmp_path / "temp"), + operation_mode="upload", + ) + + assert publishes.types() == [ + "source.ingest.queued", + "source.ingest.completed", + ] + queued = publishes.calls[0][2] + completed = publishes.calls[1][2] + assert queued["operation"] == "upload" + assert queued["loader"] == "crawler" + assert completed["operation"] == "upload" + # ``source_id`` is the same across both envelopes. + assert queued["source_id"] == completed["source_id"] + # Forward-looking ``limited`` contract — see ingest_worker + # happy-path test. Always False today; rides on the wire so a + # future token-cap detection can flip it without re-plumbing. + assert completed["limited"] is False + + def test_upload_failure_emits_failed( + self, tmp_path, task_self, monkeypatch, publishes + ): + from application import worker + + _stub_remote_pipeline(monkeypatch, raise_in_pipeline=True) + + with pytest.raises(RuntimeError, match="remote-pipeline kaboom"): + worker.remote_worker( + task_self, + {"urls": ["http://example.com"]}, + "my-remote", + "bob", + "crawler", + directory=str(tmp_path / "temp"), + operation_mode="upload", + ) + + assert publishes.types() == [ + "source.ingest.queued", + "source.ingest.failed", + ] + failed = publishes.calls[1][2] + assert "remote-pipeline kaboom" in failed["error"] + + +# ── ingest_connector ───────────────────────────────────────────────────── + + +def _stub_connector_pipeline( + monkeypatch, *, files_downloaded: int = 1, empty_result: bool = False +): + from application import worker + + fake_connector = MagicMock(name="connector") + fake_connector.download_to_directory.return_value = { + "files_downloaded": files_downloaded, + "empty_result": empty_result, + } + monkeypatch.setattr( + worker.ConnectorCreator, + "is_supported", + staticmethod(lambda s: True), + ) + monkeypatch.setattr( + worker.ConnectorCreator, + "create_connector", + staticmethod(lambda source_type, session_token: fake_connector), + ) + + fake_reader = MagicMock(name="reader") + fake_reader.load_data.return_value = [ + Document( + text="connector body", + extra_info={ + "source": "connector/file.md", + "file_path": "file.md", + }, + ) + ] + fake_reader.directory_structure = { + "file.md": { + "type": "text/markdown", + "size_bytes": 12, + "token_count": 3, + } + } + monkeypatch.setattr( + worker, "SimpleDirectoryReader", lambda *a, **kw: fake_reader + ) + + monkeypatch.setattr( + worker, + "embed_and_store_documents", + lambda docs, full_path, source_id, task, **kw: None, + ) + monkeypatch.setattr( + worker, "upload_index", lambda full_path, file_data: None + ) + + +@pytest.mark.unit +class TestIngestConnectorPublishes: + def test_upload_happy_path(self, task_self, monkeypatch, publishes): + from application import worker + + _stub_connector_pipeline(monkeypatch) + + worker.ingest_connector( + task_self, + "gdrive-folder", + "dave", + "google_drive", + session_token="tok", + file_ids=["f1"], + folder_ids=[], + operation_mode="upload", + ) + + assert publishes.types() == [ + "source.ingest.queued", + "source.ingest.completed", + ] + completed = publishes.calls[1][2] + assert completed["operation"] == "upload" + # Loader name is the provider, not the worker function name. + assert completed["loader"] == "google_drive" + # ``no_changes`` is only set on the early-return branch. + assert "no_changes" not in completed + + def test_no_files_downloaded_still_emits_completed( + self, task_self, monkeypatch, publishes + ): + """The "connector returned no files" early-return path is + otherwise silent; without the publish the toast wedges on + ``training`` until polling rescues it. + """ + from application import worker + + _stub_connector_pipeline( + monkeypatch, files_downloaded=0, empty_result=True + ) + + worker.ingest_connector( + task_self, + "gdrive-folder", + "dave", + "google_drive", + session_token="tok", + file_ids=["f1"], + folder_ids=[], + operation_mode="upload", + ) + + assert publishes.types() == [ + "source.ingest.queued", + "source.ingest.completed", + ] + completed = publishes.calls[1][2] + assert completed.get("no_changes") is True + assert completed["tokens"] == 0 + + +# ── attachment_worker ──────────────────────────────────────────────────── + + +@pytest.mark.unit +class TestAttachmentWorkerPublishes: + def test_happy_path_emits_full_progress_sequence( + self, pg_conn, patch_worker_db, task_self, monkeypatch, publishes + ): + from application import worker + + fake_doc = Document( + text="hello world", + extra_info={"transcript_language": "en"}, + ) + fake_storage = MagicMock(name="storage") + fake_storage.process_file.return_value = fake_doc + monkeypatch.setattr( + worker.StorageCreator, "get_storage", lambda: fake_storage + ) + monkeypatch.setattr( + worker, "get_default_file_extractor", lambda ocr_enabled=False: {} + ) + + file_info = { + "filename": "notes.txt", + "attachment_id": "507f1f77bcf86cd799439011", + "path": "uploads/user1/notes.txt", + "metadata": {"source": "chat"}, + } + worker.attachment_worker(task_self, file_info, "user1") + + # Four publishes: queued + two progress + completed. The + # progress events feed the percent-driven toast UI, so + # missing one would leave the UI frozen at the prior stage. + assert publishes.types() == [ + "attachment.queued", + "attachment.progress", + "attachment.progress", + "attachment.completed", + ] + + progress_a = publishes.calls[1][2] + progress_b = publishes.calls[2][2] + assert progress_a["current"] == 30 + assert progress_a["stage"] == "processing" + assert progress_b["current"] == 80 + assert progress_b["stage"] == "storing" + + completed = publishes.calls[3][2] + assert completed["filename"] == "notes.txt" + assert completed["token_count"] > 0 + assert completed["mime_type"] # populated from mimetypes + + # Scope ids are all the attachment_id verbatim. + for _, _, _, scope in publishes.calls: + assert scope == { + "kind": "attachment", + "id": file_info["attachment_id"], + } + + def test_processing_failure_emits_failed_with_partial_progress( + self, task_self, monkeypatch, publishes + ): + """``process_file`` raises mid-flow, AFTER the worker has + already emitted the ``current=30`` progress envelope. The + terminal ``failed`` must still arrive so the toast unwedges, + even though no ``current=80`` progress will follow. + """ + from application import worker + + fake_storage = MagicMock(name="storage") + fake_storage.process_file.side_effect = RuntimeError("parse boom") + monkeypatch.setattr( + worker.StorageCreator, "get_storage", lambda: fake_storage + ) + monkeypatch.setattr( + worker, "get_default_file_extractor", lambda ocr_enabled=False: {} + ) + + file_info = { + "filename": "notes.txt", + "attachment_id": "507f1f77bcf86cd799439011", + "path": "uploads/user1/notes.txt", + "metadata": {"source": "chat"}, + } + with pytest.raises(RuntimeError, match="parse boom"): + worker.attachment_worker(task_self, file_info, "user1") + + # queued + the first progress envelope (current=30) + failed. + # The current=80 publish never fires because ``process_file`` + # raises before reaching it. + assert publishes.types() == [ + "attachment.queued", + "attachment.progress", + "attachment.failed", + ] + assert publishes.calls[1][2]["current"] == 30 + failed = publishes.calls[2][2] + assert "parse boom" in failed["error"] + assert failed["filename"] == "notes.txt" + + +# ── mcp_oauth ──────────────────────────────────────────────────────────── + + +def _make_fake_mcp_tool_class( + *, + tools: list[str] | None = None, + discovery_raises: Exception | None = None, + authorization_url: str | None = "https://idp.example.com/authorize?state=xyz", +): + """Build a class that stands in for ``MCPTool`` in mcp_oauth. + + The worker constructs ``MCPTool(tool_config, user_id)`` then calls: + + - ``_client`` (attribute, falsy → triggers ``_setup_client``) + - ``_setup_client()`` (sync, sets up the client) + - ``_execute_with_client("list_tools")`` (async, returns anything) + - ``get_actions_metadata()`` (sync, returns the tools list) + + The fake mirrors the real handshake: ``__init__`` lifts the + ``oauth_redirect_publish`` callback out of ``tool_config`` and + ``_setup_client`` invokes it with ``authorization_url`` — the same + spot where the real ``DocsGPTOAuth.redirect_handler`` would fire it + once the upstream OAuth provider mints the URL. + + The ``discovery_raises`` knob simulates a mid-flow failure inside + the ``run_oauth_discovery`` coroutine. Setting + ``authorization_url=None`` suppresses the redirect publish so the + ``no user_id`` test path can assert the publisher is fully silent. + """ + resolved_tools = list(tools) if tools is not None else ["tool_a", "tool_b"] + + class _FakeMCPTool: + def __init__(self, tool_config: dict, user_id: str | None = None): + self._client = None + self._tool_config = tool_config + self._user_id = user_id + # Real ``MCPTool.__init__`` ``pop``s this key off the config + # so it does not leak into the persisted tool config blob. + # Mirror that behaviour so the test catches a regression + # where the worker stops passing the callback through. + self._oauth_redirect_publish = tool_config.pop( + "oauth_redirect_publish", None + ) + + def _setup_client(self) -> None: + self._client = object() + # Simulate the real OAuth flow: ``DocsGPTOAuth.redirect_handler`` + # invokes the worker-provided callback the moment the + # authorization URL is known. Doing it here keeps the + # observable publish ordering identical to the real path: + # ``in_progress`` → ``awaiting_redirect(url)`` → terminal. + if ( + authorization_url is not None + and callable(self._oauth_redirect_publish) + ): + self._oauth_redirect_publish(authorization_url) + + async def _execute_with_client(self, name: str): + if discovery_raises is not None: + raise discovery_raises + return None + + def get_actions_metadata(self): + return resolved_tools + + return _FakeMCPTool + + +@pytest.fixture +def _task_self_with_request_id(): + """``mcp_oauth`` reads ``self.request.id``; supply a stable one.""" + task = MagicMock(name="celery_task_self") + task.request.id = "task-xyz" + return task + + +def _patch_mcp_tool(monkeypatch, fake_tool_class) -> None: + """Replace the ``MCPTool`` symbol the worker imports at call time. + + The worker runs ``from application.agents.tools.mcp_tool import MCPTool`` + inside the function body. Importing the real module is a circular + dependency in this test process (it pulls in the user routes, + which pull in the MCP namespace, which pulls in + ``mcp_tool``), so we inject a stub module into ``sys.modules`` + BEFORE the import line is reached. Python will then resolve the + inline import against our stub instead of evaluating the real + file. + """ + import sys + import types + + stub = types.ModuleType("application.agents.tools.mcp_tool") + stub.MCPTool = fake_tool_class + monkeypatch.setitem( + sys.modules, "application.agents.tools.mcp_tool", stub + ) + + +@pytest.mark.unit +class TestMcpOauthPublishes: + def test_happy_path_emits_progress_sequence_and_completed( + self, monkeypatch, publishes, _task_self_with_request_id + ): + from application import worker + + auth_url = "https://idp.example.com/authorize?state=happy-path" + _patch_mcp_tool( + monkeypatch, + _make_fake_mcp_tool_class( + tools=["alpha", "beta"], authorization_url=auth_url, + ), + ) + + result = worker.mcp_oauth( + _task_self_with_request_id, + {"command": "node", "args": []}, + user_id="zoe", + ) + + assert result["success"] is True + assert result["tools"] == ["alpha", "beta"] + + # Ordered emit: in_progress → awaiting_redirect → completed. + # Any drift here changes the toast state machine. + assert publishes.types() == [ + "mcp.oauth.in_progress", + "mcp.oauth.awaiting_redirect", + "mcp.oauth.completed", + ] + awaiting = publishes.calls[1][2] + # The frontend opens the OAuth popup straight from this URL. + # Without it the popup never opens and the user retries; the + # test guards the contract that the SSE envelope is now the + # source of truth for the authorization URL. + assert awaiting["authorization_url"] == auth_url + completed = publishes.calls[2][2] + assert completed["tools"] == ["alpha", "beta"] + assert completed["tools_count"] == 2 + # Every envelope carries the task_id so the toast can correlate + # it to the original POST that kicked the OAuth off. + for _, _, payload, scope in publishes.calls: + assert payload["task_id"] == "task-xyz" + assert scope == {"kind": "mcp_oauth", "id": "task-xyz"} + + def test_finish_oauth_failure_emits_failed( + self, monkeypatch, publishes, _task_self_with_request_id + ): + """A failure inside the OAuth completion step is the most + common failure mode (user cancels, provider 4xx). Must surface + as a ``mcp.oauth.failed`` envelope so the toast unwedges. + """ + from application import worker + + _patch_mcp_tool( + monkeypatch, + _make_fake_mcp_tool_class( + discovery_raises=RuntimeError("oauth user cancelled") + ), + ) + + result = worker.mcp_oauth( + _task_self_with_request_id, + {"command": "node", "args": []}, + user_id="zoe", + ) + + assert result["success"] is False + # in_progress + awaiting_redirect + failed; no completed. + assert publishes.types() == [ + "mcp.oauth.in_progress", + "mcp.oauth.awaiting_redirect", + "mcp.oauth.failed", + ] + failed = publishes.calls[-1][2] + assert "oauth user cancelled" in failed["error"] + + def test_skips_publishes_when_user_id_missing( + self, monkeypatch, publishes, _task_self_with_request_id + ): + """Legacy callers can invoke ``mcp_oauth`` without a user_id; + the worker must not blow up but also must NOT publish to a + synthetic / shared channel. Polling-based status remains the + path of record in that case. + """ + from application import worker + + _patch_mcp_tool(monkeypatch, _make_fake_mcp_tool_class()) + + result = worker.mcp_oauth( + _task_self_with_request_id, + {"command": "node", "args": []}, + user_id=None, + ) + + assert result["success"] is True + # ``publish_oauth`` short-circuits when ``user_id`` is falsy. + assert publishes.calls == [] + + +# ── retry-gate: queued must NOT re-emit on Celery retries ─────────────── + + +@pytest.fixture +def task_self_retry(): + """``self`` with ``request.retries == 1`` — second attempt of the task.""" + task = MagicMock(name="celery_task_self_retry") + task.request.retries = 1 + return task + + +@pytest.mark.unit +class TestQueuedEventRetryGate: + """``DURABLE_TASK`` retries the same body on transient embed failures. + + Without the first-attempt gate the toast oscillates + ``queued → progress → failed → queued → progress → completed``. + These tests pin the gate behaviour at all four ingest sites. + """ + + def test_ingest_worker_skips_queued_on_retry( + self, patch_worker_db, task_self_retry, monkeypatch, publishes + ): + from application import worker + + _patch_ingest_pipeline_min(monkeypatch) + + worker.ingest_worker( + task_self_retry, + directory="inputs", + formats=[".txt"], + job_name="job1", + file_path="inputs/eve/job1/a.txt", + filename="a.txt", + user="eve", + retriever="classic", + source_id=str(uuid.uuid4()), + ) + + # Retry attempt: only ``completed`` fires; the ``queued`` from the + # original attempt has already been published and acked. + assert publishes.types() == ["source.ingest.completed"] + + def test_remote_worker_skips_queued_on_retry( + self, tmp_path, task_self_retry, monkeypatch, publishes + ): + from application import worker + + _stub_remote_pipeline(monkeypatch) + + worker.remote_worker( + task_self_retry, + {"urls": ["http://example.com"]}, + "my-remote", + "bob", + "crawler", + directory=str(tmp_path / "temp"), + operation_mode="upload", + ) + + assert publishes.types() == ["source.ingest.completed"] + + def test_ingest_connector_skips_queued_on_retry( + self, task_self_retry, monkeypatch, publishes + ): + from application import worker + + _stub_connector_pipeline(monkeypatch) + + worker.ingest_connector( + task_self_retry, + "gdrive-folder", + "dave", + "google_drive", + session_token="tok", + file_ids=["f1"], + folder_ids=[], + operation_mode="upload", + ) + + assert publishes.types() == ["source.ingest.completed"] + + def test_reingest_source_worker_skips_queued_on_retry( + self, pg_conn, patch_worker_db, task_self_retry, monkeypatch, publishes + ): + from application import worker + + src = _seed_source_for_reingest(pg_conn, user_id="alice") + source_id = str(src["id"]) + + _stub_reingest_storage_and_vectorstore(monkeypatch) + + fake_reader = MagicMock(name="reader") + fake_reader.load_data.return_value = [] + fake_reader.directory_structure = { + "fresh.md": { + "type": "text/markdown", + "size_bytes": 42, + "token_count": 17, + } + } + fake_reader.file_token_counts = {"fresh.md": 17} + monkeypatch.setattr( + worker, "SimpleDirectoryReader", lambda *a, **kw: fake_reader + ) + + worker.reingest_source_worker(task_self_retry, source_id, "alice") + + # Only the terminal ``completed`` — no ``queued`` re-emit. + assert publishes.types() == ["source.ingest.completed"] + + def test_ingest_worker_first_attempt_still_emits_queued( + self, patch_worker_db, task_self, monkeypatch, publishes + ): + """Sanity counterpart: ``retries == 0`` keeps the original + queued+completed pair so the no-retry path is unchanged. + """ + from application import worker + + _patch_ingest_pipeline_min(monkeypatch) + + worker.ingest_worker( + task_self, + directory="inputs", + formats=[".txt"], + job_name="job1", + file_path="inputs/eve/job1/a.txt", + filename="a.txt", + user="eve", + retriever="classic", + source_id=str(uuid.uuid4()), + ) + + assert publishes.types() == [ + "source.ingest.queued", + "source.ingest.completed", + ]