Merge pull request #2637 from arc53/fix/docling-parse-error-and-schedule-status

Fix/docling parse error and schedule status

application/agents/headless_runner.py

@@ -29,6 +29,34 @@ def _resolve_agent_id(agent_config: Dict[str, Any]) -> Optional[str]:
     return str(raw) if raw else None
 
 
+def _workflow_kwargs(agent_config: Dict[str, Any], owner: str) -> Dict[str, Any]:
+    """Bind a workflow agent to its graph, mirroring ``StreamProcessor``.
+
+    A ``WorkflowAgent`` built without one of these loads no graph and its
+    entire run is a single "Failed to load workflow configuration." error, so
+    a scheduled or webhook-fired workflow agent never does anything. The PG
+    ``agents`` row stores a UUID under ``workflow_id``; the legacy Mongo shape
+    used ``workflow``, which also carried an embedded graph.
+    """
+    kwargs: Dict[str, Any] = {"workflow_owner": owner}
+    embedded = agent_config.get("workflow")
+    if isinstance(embedded, dict):
+        kwargs["workflow"] = embedded
+        saved_id = agent_config.get("workflow_id")
+        if saved_id:
+            kwargs["workflow_id"] = str(saved_id)
+        return kwargs
+    wf_ref = agent_config.get("workflow_id") or embedded
+    if wf_ref:
+        kwargs["workflow_id"] = str(wf_ref)
+    else:
+        logger.warning(
+            "Workflow agent %s has no workflow reference; the run will load no graph.",
+            _resolve_agent_id(agent_config),
+        )
+    return kwargs
+
+
 def run_agent_headless(
     agent_config: Dict[str, Any],
     query: str,
@@ -133,22 +161,24 @@ def run_agent_headless(
     if conversation_id:
         tool_executor.conversation_id = str(conversation_id)
 
-    agent = AgentCreator.create_agent(
-        agent_type,
-        endpoint=endpoint,
-        llm_name=provider or settings.LLM_PROVIDER,
-        model_id=model_id,
-        api_key=system_api_key,
-        agent_id=agent_id,
-        user_api_key=user_api_key,
-        prompt=prompt,
-        chat_history=chat_history or [],
-        retrieved_docs=retrieved_docs,
-        decoded_token=decoded_token,
-        attachments=[],
-        json_schema=json_schema,
-        tool_executor=tool_executor,
-    )
+    agent_kwargs: Dict[str, Any] = {
+        "endpoint": endpoint,
+        "llm_name": provider or settings.LLM_PROVIDER,
+        "model_id": model_id,
+        "api_key": system_api_key,
+        "agent_id": agent_id,
+        "user_api_key": user_api_key,
+        "prompt": prompt,
+        "chat_history": chat_history or [],
+        "retrieved_docs": retrieved_docs,
+        "decoded_token": decoded_token,
+        "attachments": [],
+        "json_schema": json_schema,
+        "tool_executor": tool_executor,
+    }
+    if agent_type == "workflow":
+        agent_kwargs.update(_workflow_kwargs(agent_config, owner))
+    agent = AgentCreator.create_agent(agent_type, **agent_kwargs)
     if conversation_id:
         agent.conversation_id = str(conversation_id)
 
@@ -156,9 +186,28 @@ def run_agent_headless(
     thought = ""
     sources_log: List[Dict[str, Any]] = []
     tool_calls: List[Dict[str, Any]] = []
+    stream_error: Optional[str] = None
+    steps_completed = 0
     for event in agent.gen(query=query):
         if not isinstance(event, dict):
             continue
+        # ``Agent.gen`` reports a failed stream with an error event rather than
+        # by raising. Dropping it here (as this loop used to) makes a broken run
+        # indistinguishable from one that simply had nothing to say, and the
+        # caller records it as a success. Mirrors the sentinel in
+        # ``application/logging.py`` so an error carrying no message is still
+        # truthy instead of reading as "ok".
+        if event.get("type") == "error":
+            stream_error = str(event.get("error") or "")[:500] or "unspecified"
+            continue
+        # A workflow's work is its nodes: its tool calls stay in the engine's
+        # execution log and its node agents own their LLMs, so neither
+        # ``tool_calls`` nor the token tally below sees them. Counting
+        # completed steps is the only evidence a quiet workflow ran at all.
+        if event.get("type") == "workflow_step":
+            if event.get("status") == "completed":
+                steps_completed += 1
+            continue
         if "answer" in event:
             answer_full += str(event["answer"])
         elif "sources" in event:
@@ -169,7 +218,22 @@ def run_agent_headless(
             thought += str(event["thought"])
 
     denied = list(getattr(tool_executor, "headless_denials", []))
-    error_type = "tool_not_allowed" if denied and not answer_full.strip() else None
+    error: Optional[str] = None
+    if denied and not answer_full.strip():
+        error_type = "tool_not_allowed"
+        blocked = ", ".join(
+            str(d.get("tool_name") or d.get("action_name") or "?") for d in denied
+        )
+        error = f"headless allowlist blocked required tool: {blocked}"[:500]
+    elif stream_error:
+        error_type = "stream_error"
+        error = stream_error
+    else:
+        error_type = None
+    if stream_error:
+        logger.warning(
+            "Headless run for agent %s failed mid-stream: %s", agent_id, stream_error
+        )
 
     # Use the LLM accumulator (gen_token_usage / stream_token_usage decorators);
     # current_token_count is a context-size sentinel, not a usage tally.
@@ -186,5 +250,7 @@ def run_agent_headless(
         "generated_tokens": generated_tokens,
         "denied": denied,
         "error_type": error_type,
+        "error": error,
+        "steps_completed": steps_completed,
         "model_id": model_id,
     }

application/alembic/versions/0027_schedule_run_error_types.py

@@ -0,0 +1,61 @@
+"""0027 schedule_runs.error_type — allow 'stream_error' and 'empty_output'.
+
+``schedule_runs.status`` could only ever be written ``success`` for a run whose
+stream failed: the headless runner dropped the agent's error event, so the
+worker saw an ordinary return with an empty answer and no error type. With that
+fixed the worker now classifies two further outcomes, and both need to be
+accepted by the CHECK constraint added in 0010:
+
+* ``stream_error``  — the agent reported a mid-stream failure (provider error,
+  failed fallback) rather than raising.
+* ``empty_output``  — backstop: the run produced no answer, no tool call and no
+  completion tokens, so it did no work even though nothing raised.
+
+Constraint-only change; no data is rewritten and existing rows stay valid.
+Idempotent both ways.
+
+Revision ID: 0027_schedule_run_error_types
+Revises: 0026_stack_logs_agent_id
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+revision: str = "0027_schedule_run_error_types"
+down_revision: Union[str, None] = "0026_stack_logs_agent_id"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+_OLD = (
+    "'auth_expired', 'tool_not_allowed', 'budget_exceeded', "
+    "'timeout', 'agent_error', 'internal', 'missed', 'overlap'"
+)
+_NEW = _OLD + ", 'stream_error', 'empty_output'"
+
+
+def _recreate(values: str) -> None:
+    op.execute(
+        "ALTER TABLE schedule_runs "
+        "DROP CONSTRAINT IF EXISTS schedule_runs_error_type_chk;"
+    )
+    op.execute(
+        "ALTER TABLE schedule_runs ADD CONSTRAINT schedule_runs_error_type_chk "
+        f"CHECK (error_type IS NULL OR error_type IN ({values}));"
+    )
+
+
+def upgrade() -> None:
+    _recreate(_NEW)
+
+
+def downgrade() -> None:
+    # Rows written by the new classifier would violate the narrower constraint,
+    # so retire those values before restoring it.
+    op.execute(
+        "UPDATE schedule_runs SET error_type = 'internal' "
+        "WHERE error_type IN ('stream_error', 'empty_output');"
+    )
+    _recreate(_OLD)

application/api/user/scheduler_worker.py

@@ -290,14 +290,14 @@ def execute_scheduled_run_body(run_id: str, celery_task_id: Optional[str]) -> Di
 
     finished = datetime.now(timezone.utc)
 
-    # Headless denial with no usable output → tool_not_allowed.
-    if (
-        error_type is None
-        and (outcome.get("denied") or [])
-        and not (outcome.get("answer") or "").strip()
-    ):
-        error_type = "tool_not_allowed"
-        error_text = "headless allowlist blocked required tool"
+    # Honour the outcome the runner reported: it classifies headless denials
+    # (``tool_not_allowed``) and mid-stream failures (``stream_error``) alike,
+    # and owns the message. Re-deriving the denial case here only duplicated
+    # the runner's rule while leaving a failed stream — which returns normally,
+    # with an empty answer — to fall through to "success".
+    if error_type is None and outcome.get("error_type"):
+        error_type = str(outcome["error_type"])
+        error_text = str(outcome.get("error") or "") or error_type
 
     prompt_tokens = int(outcome.get("prompt_tokens", 0) or 0)
     generated_tokens = int(outcome.get("generated_tokens", 0) or 0)
@@ -313,6 +313,22 @@ def execute_scheduled_run_body(run_id: str, celery_task_id: Optional[str]) -> Di
             f"{schedule['token_budget']}"
         )
 
+    # Backstop for silent failures that raise nothing and emit no error event:
+    # a run that produced no answer, did no work and burned no completion
+    # tokens did not do its job, whatever the absence of an exception suggests.
+    # "Work" is deliberately broad — tool calls for a chat agent, completed
+    # nodes for a workflow — so a schedule whose whole purpose is a side effect
+    # (post a message, file a ticket) is never flagged for staying quiet.
+    if (
+        error_type is None
+        and not (outcome.get("answer") or "").strip()
+        and not (outcome.get("tool_calls") or [])
+        and not int(outcome.get("steps_completed") or 0)
+        and generated_tokens == 0
+    ):
+        error_type = "empty_output"
+        error_text = "run produced no answer, tool calls or completion tokens"
+
     answer = outcome.get("answer") or ""
     truncated = False
     if len(answer) > _OUTPUT_CAP_CHARS:

application/api/user/tasks.py

@@ -5,6 +5,7 @@ from sqlalchemy.exc import DataError
 
 from application.api.user.idempotency import with_idempotency
 from application.celery_init import celery
+from application.parser.file.base_parser import DocumentParseError
 from application.worker import (
     AttachmentRejectedError,
     agent_webhook_worker,
@@ -69,7 +70,10 @@ def _emit_ingest_poison_event(task_name, bound):
     )
 
 
-@celery.task(**DURABLE_TASK)
+# ``dont_autoretry_for``: a file that cannot be converted to text fails
+# identically on every attempt, so retrying only multiplies the log noise
+# before the same failure — go straight to the poison/failure path.
+@celery.task(**DURABLE_TASK, dont_autoretry_for=(DocumentParseError,))
 @with_idempotency(task_name="ingest", on_poison=_emit_ingest_poison_event)
 def ingest(
     self,
@@ -237,10 +241,14 @@ def _emit_attachment_poison_event(task_name, bound):
 
 
 # ``dont_autoretry_for``: a DataError (poison payload, e.g. NUL bytes or an
-# over-long value) or an AttachmentRejectedError (zip bomb) is deterministic —
-# retrying re-fails identically and multiplies log noise, so it goes straight
-# to the failure path.
-@celery.task(**DURABLE_TASK, dont_autoretry_for=(DataError, AttachmentRejectedError))
+# over-long value), an AttachmentRejectedError (zip bomb) or a
+# DocumentParseError (the file cannot be converted to text at all) is
+# deterministic — retrying re-fails identically and multiplies log noise, so it
+# goes straight to the failure path.
+@celery.task(
+    **DURABLE_TASK,
+    dont_autoretry_for=(DataError, AttachmentRejectedError, DocumentParseError),
+)
 @with_idempotency(
     task_name="store_attachment", on_poison=_emit_attachment_poison_event,
 )

application/parser/file/base_parser.py

@@ -5,6 +5,22 @@ from pathlib import Path
 from typing import Dict, List, Optional, Union
 
 
+class DocumentParseError(Exception):
+    """A file could not be converted to text.
+
+    Raised instead of returning the failure message as the document body: a
+    parser that hands back its own traceback produces an attachment/source
+    whose "content" is an error string, which then reaches the LLM as if it
+    were the document. Callers should treat this as a failed upload and tell
+    the user, rather than storing anything.
+
+    Deterministic for a given input, so it is marked non-retryable on the
+    Celery tasks that parse (alongside ``DataError`` and
+    ``AttachmentRejectedError``) — a poison file fails once instead of
+    re-failing identically on every retry.
+    """
+
+
 class BaseParser:
     """Base class for all parsers."""
 

application/parser/file/bulk.py

@@ -1,10 +1,10 @@
 """Simple reader that reads files of different formats from a directory."""
 import logging
 from pathlib import Path
-from typing import Callable, Dict, List, Optional, Union
+from typing import Callable, Dict, List, Optional, Tuple, Union
 
 from application.parser.file.base import BaseReader
-from application.parser.file.base_parser import BaseParser
+from application.parser.file.base_parser import BaseParser, DocumentParseError
 from application.parser.file.docs_parser import DocxParser, PDFParser
 from application.parser.file.epub_parser import EpubParser
 from application.parser.file.html_parser import HTMLParser
@@ -177,6 +177,8 @@ class SimpleDirectoryReader(BaseReader):
 
         self.file_extractor = file_extractor or DEFAULT_FILE_EXTRACTOR
         self.file_metadata = file_metadata
+        # (path, message) per file skipped by ``load_data`` as unparseable.
+        self.failed_files: List[Tuple[Path, str]] = []
 
     def _add_files(self, input_dir: Path) -> List[Path]:
         """Add files."""
@@ -230,27 +232,51 @@ class SimpleDirectoryReader(BaseReader):
 
         Returns:
             List[Document]: A list of documents.
+
+        Raises:
+            DocumentParseError: if no input file could be parsed. Individual
+                unreadable files are skipped and recorded in ``failed_files``
+                so one corrupt document does not cost the caller the rest of
+                the batch; a single-file read (the attachment path) still
+                raises, since skipping there would only defer the failure to
+                an empty result.
         """
         data: Union[str, List[str]] = ""
         data_list: List[str] = []
         metadata_list = []
         self.file_token_counts = {}
+        self.failed_files = []
 
         total_files = len(self.input_files)
+
+        def report_progress(files_done: int) -> None:
+            if progress_callback is None:
+                return
+            try:
+                progress_callback(files_done, total_files)
+            except Exception:
+                logging.warning("load_data progress callback failed", exc_info=True)
+
         for file_index, input_file in enumerate(self.input_files):
             suffix_lower = input_file.suffix.lower()
             parser_metadata = {}
-            if suffix_lower in self.file_extractor:
-                parser = self.file_extractor[suffix_lower]
-                if not parser.parser_config_set:
-                    parser.init_parser()
-                data = parser.parse_file(input_file, errors=self.errors)
-                parser_metadata = parser.get_file_metadata(input_file)
-            else:
-                # do standard read
-                with open(input_file, "r", errors=self.errors) as f:
-                    data = f.read()
-            
+            try:
+                if suffix_lower in self.file_extractor:
+                    parser = self.file_extractor[suffix_lower]
+                    if not parser.parser_config_set:
+                        parser.init_parser()
+                    data = parser.parse_file(input_file, errors=self.errors)
+                    parser_metadata = parser.get_file_metadata(input_file)
+                else:
+                    # do standard read
+                    with open(input_file, "r", errors=self.errors) as f:
+                        data = f.read()
+            except DocumentParseError as e:
+                logging.warning(f"Skipping unreadable file {input_file.name}: {e}")
+                self.failed_files.append((input_file, str(e)))
+                report_progress(file_index + 1)
+                continue
+
             # Calculate token count for this file
             if isinstance(data, List):
                 file_tokens = sum(num_tokens_from_string(str(d)) for d in data)
@@ -288,13 +314,23 @@ class SimpleDirectoryReader(BaseReader):
                 data_list.append(str(data))
                 metadata_list.append(base_metadata)
 
-            if progress_callback is not None:
-                try:
-                    progress_callback(file_index + 1, total_files)
-                except Exception:
-                    logging.warning(
-                        "load_data progress callback failed", exc_info=True
-                    )
+            report_progress(file_index + 1)
+
+        # Every file failed: there is nothing to ingest, so this is a failed
+        # read rather than an empty one. Callers (the attachment worker, the
+        # ingest tasks) treat it as terminal and tell the user.
+        if self.failed_files and not data_list:
+            if len(self.failed_files) == 1:
+                # Single-file read (the attachment path): the parser's own
+                # message reaches the user verbatim, so don't wrap it in
+                # "None of the 1 file(s)…".
+                raise DocumentParseError(self.failed_files[0][1])
+            names = ", ".join(p.name for p, _ in self.failed_files[:5])
+            raise DocumentParseError(
+                f"None of the {len(self.failed_files)} files could be parsed: "
+                f"{names}{'…' if len(self.failed_files) > 5 else ''} "
+                f"({self.failed_files[0][1]})"
+            )
 
         # Build directory structure if input_dir is provided
         if hasattr(self, 'input_dir'):

application/parser/file/docling_parser.py

@@ -14,7 +14,7 @@ import zipfile
 from pathlib import Path
 from typing import Dict, List, Optional, Union
 
-from application.parser.file.base_parser import BaseParser
+from application.parser.file.base_parser import BaseParser, DocumentParseError
 from application.utils import truncate_to_line_boundary
 
 logger = logging.getLogger(__name__)
@@ -346,9 +346,14 @@ class DoclingParser(BaseParser):
 
         except Exception as e:
             logger.error(f"Error parsing file with docling: {e}", exc_info=True)
-            if errors == "ignore":
-                return f"[Error parsing file with docling: {str(e)}]"
-            raise
+            # ``errors`` governs *decoding* leniency, not whether a total
+            # conversion failure may be substituted for the document. Returning
+            # the message here (the old ``errors == "ignore"`` branch) made the
+            # caller store the traceback as the file's text and report the
+            # upload successful — the model then read the error as the document.
+            raise DocumentParseError(
+                f"Failed to parse {Path(file).name} with docling: {e}"
+            ) from e
 
 
 class DoclingPDFParser(DoclingParser):

application/requirements.txt

@@ -12,7 +12,7 @@ cryptography==46.0.7
 dataclasses-json==0.6.7
 daytona==0.190.1
 defusedxml==0.7.1
-docling>=2.16.0
+docling==2.84.0
 rapidocr>=1.4.0
 onnxruntime>=1.19.0
 docx2txt==0.9
@@ -112,4 +112,4 @@ yarl==1.23.0
 markdownify==1.2.2
 tldextract==5.3.1
 websockets==16.0
-websocket-client==1.9.0
+websocket-client==1.9.0