diff --git a/application/agents/base.py b/application/agents/base.py index f84bd38d..8001f53a 100644 --- a/application/agents/base.py +++ b/application/agents/base.py @@ -36,6 +36,7 @@ class BaseAgent(ABC): json_schema_strict: bool = True, json_object: bool = False, llm_params: Optional[Dict] = None, + multimodal_content: Optional[List] = None, limited_token_mode: Optional[bool] = False, token_limit: Optional[int] = settings.DEFAULT_AGENT_LIMITS["token_limit"], limited_request_mode: Optional[bool] = False, @@ -119,6 +120,9 @@ class BaseAgent(ABC): # OpenAI sampling params forwarded from the request (temperature, # max_tokens, top_p, ...). Empty when the caller sent none. self.llm_params = llm_params or {} + # Full OpenAI content array (text + image_url parts) for the current + # user turn, when the request was multimodal; None otherwise. + self.multimodal_content = multimodal_content self.limited_token_mode = limited_token_mode self.token_limit = token_limit self.limited_request_mode = limited_request_mode @@ -512,7 +516,15 @@ class BaseAgent(ABC): "tool_call_id": call_id, "content": result_str, }) - messages.append({"role": "user", "content": query}) + # When the request was multimodal, send the full content array (text + + # image_url parts) so images reach the model; the text-only `query` above + # is used only for token budgeting / retrieval. + user_content = ( + self.multimodal_content + if getattr(self, "multimodal_content", None) + else query + ) + messages.append({"role": "user", "content": user_content}) return messages def _truncate_history_to_fit( diff --git a/application/api/answer/services/stream_processor.py b/application/api/answer/services/stream_processor.py index ab1d94e6..e7ebec6d 100644 --- a/application/api/answer/services/stream_processor.py +++ b/application/api/answer/services/stream_processor.py @@ -1326,6 +1326,7 @@ class StreamProcessor: "json_schema_strict": self.agent_config.get("json_schema_strict", True), "json_object": self.agent_config.get("json_object", False), "llm_params": self.data.get("llm_params") or {}, + "multimodal_content": self.data.get("multimodal_content"), "compressed_summary": self.compressed_summary, "llm": llm, "llm_handler": llm_handler, diff --git a/application/api/v1/translator.py b/application/api/v1/translator.py index ccb4b23a..f55f7c3c 100644 --- a/application/api/v1/translator.py +++ b/application/api/v1/translator.py @@ -120,6 +120,28 @@ def extract_conversation_id(messages: List[Dict]) -> Optional[str]: return None +def content_to_text(content: Any) -> str: + """Flatten an OpenAI message ``content`` to plain text. + + ``content`` may be a string or a list of typed parts + (``{"type":"text",...}`` / ``{"type":"image_url",...}`` / ...). Only text + parts contribute; image/other parts are dropped here. The full content + array is preserved separately (see ``multimodal_content``) so images still + reach the model in the final user message. + """ + if isinstance(content, str): + return content + if isinstance(content, list): + out = [] + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + out.append(part.get("text", "") or "") + elif isinstance(part, str): + out.append(part) + return "\n".join(out) + return "" if content is None else str(content) + + def extract_system_prompt(messages: List[Dict]) -> Optional[str]: """Extract the first system message content from the messages array. @@ -127,7 +149,7 @@ def extract_system_prompt(messages: List[Dict]) -> Optional[str]: """ for msg in messages: if msg.get("role") == "system": - return msg.get("content", "") + return content_to_text(msg.get("content", "")) return None @@ -147,9 +169,9 @@ def convert_history(messages: List[Dict]) -> List[Dict]: if msg.get("role") == "user": # Look ahead for assistant response if i + 1 < len(messages) and messages[i + 1].get("role") == "assistant": - content = messages[i + 1].get("content") or "" + content = content_to_text(messages[i + 1].get("content") or "") history.append({ - "prompt": msg.get("content", ""), + "prompt": content_to_text(msg.get("content", "")), "response": content, }) i += 2 @@ -253,12 +275,16 @@ def translate_request( result["json_object"] = True return result - # Normal request — extract question from last user message - question = "" + # Normal request — extract the question (text) from the last user message, + # and keep its full content array (text + image_url parts) when multimodal so + # images still reach the model in the final user message. + last_user_content = None for msg in reversed(messages): if msg.get("role") == "user": - question = msg.get("content", "") + last_user_content = msg.get("content") break + question = content_to_text(last_user_content) + multimodal_content = last_user_content if isinstance(last_user_content, list) else None history = convert_history(messages) system_prompt_override = extract_system_prompt(messages) @@ -292,6 +318,8 @@ def translate_request( result["json_object"] = True if sampling_params: result["llm_params"] = sampling_params + if multimodal_content is not None: + result["multimodal_content"] = multimodal_content return result